projects/linux·blob master · refs

3386 lines · 85.4 KiB

1// SPDX-License-Identifier: GPL-2.0
2#include <dirent.h>
3#include <errno.h>
4#include <inttypes.h>
5#include <regex.h>
6#include <stdlib.h>
7#include <string.h>
8#include "callchain.h"
9#include "debug.h"
10#include "dso.h"
11#include "env.h"
12#include "dwarf-regs.h"
13#include "event.h"
14#include "evsel.h"
15#include "hist.h"
16#include "machine.h"
17#include "map.h"
18#include "map_symbol.h"
19#include "branch.h"
20#include "mem-events.h"
21#include "mem-info.h"
22#include "path.h"
23#include "srcline.h"
24#include "symbol.h"
25#include "synthetic-events.h"
26#include "sort.h"
27#include "strlist.h"
28#include "target.h"
29#include "thread.h"
30#include "util.h"
31#include "vdso.h"
32#include <stdbool.h>
33#include <sys/types.h>
34#include <sys/stat.h>
35#include <unistd.h>
36#include "unwind.h"
37#include "linux/hash.h"
38#include "asm/bug.h"
39#include "bpf-event.h"
40#include <internal/lib.h> // page_size
41#include "cgroup.h"
42#include "arm64-frame-pointer-unwind-support.h"
43#include <api/io_dir.h>
44
45#include <linux/ctype.h>
46#include <symbol/kallsyms.h>
47#include <linux/mman.h>
48#include <linux/string.h>
49#include <linux/zalloc.h>
50
51static struct dso *machine__kernel_dso(struct machine *machine)
52{
53 return map__dso(machine->vmlinux_map);
54}
55
56static int machine__set_mmap_name(struct machine *machine)
57{
58 if (machine__is_host(machine))
59 machine->mmap_name = strdup("[kernel.kallsyms]");
60 else if (machine__is_default_guest(machine))
61 machine->mmap_name = strdup("[guest.kernel.kallsyms]");
62 else if (asprintf(&machine->mmap_name, "[guest.kernel.kallsyms.%d]",
63 machine->pid) < 0)
64 machine->mmap_name = NULL;
65
66 return machine->mmap_name ? 0 : -ENOMEM;
67}
68
69static void thread__set_guest_comm(struct thread *thread, pid_t pid)
70{
71 char comm[64];
72
73 snprintf(comm, sizeof(comm), "[guest/%d]", pid);
74 thread__set_comm(thread, comm, 0);
75}
76
77int machine__init(struct machine *machine, const char *root_dir, pid_t pid)
78{
79 int err = -ENOMEM;
80
81 memset(machine, 0, sizeof(*machine));
82 RB_CLEAR_NODE(&machine->rb_node);
83 dsos__init(&machine->dsos);
84 threads__init(&machine->threads);
85
86 machine->kmaps = maps__new(machine);
87 if (machine->kmaps == NULL)
88 goto out;
89
90 machine->vdso_info = NULL;
91 machine->env = NULL;
92
93 machine->pid = pid;
94
95 machine->id_hdr_size = 0;
96 machine->kptr_restrict_warned = false;
97 machine->comm_exec = false;
98 machine->kernel_start = 0;
99 machine->vmlinux_map = NULL;
100 /* There is no initial context switch in, so we start at 1. */
101 machine->parallelism = 1;
102
103 machine->root_dir = strdup(root_dir);
104 if (machine->root_dir == NULL)
105 goto out;
106
107 if (machine__set_mmap_name(machine))
108 goto out;
109
110 if (pid != HOST_KERNEL_ID) {
111 struct thread *thread = machine__findnew_thread(machine, -1,
112 pid);
113
114 if (thread == NULL)
115 goto out;
116
117 thread__set_guest_comm(thread, pid);
118 thread__put(thread);
119 }
120
121 machine->current_tid = NULL;
122 err = 0;
123
124out:
125 if (err) {
126 maps__zput(machine->kmaps);
127 zfree(&machine->root_dir);
128 zfree(&machine->mmap_name);
129 }
130 return err;
131}
132
133static struct machine *__machine__new_host(struct perf_env *host_env, bool kernel_maps)
134{
135 struct machine *machine = malloc(sizeof(*machine));
136
137 if (!machine)
138 return NULL;
139
140 if (machine__init(machine, "", HOST_KERNEL_ID) != 0) {
141 free(machine);
142 return NULL;
143 }
144
145 if (kernel_maps && machine__create_kernel_maps(machine) < 0) {
146 free(machine);
147 return NULL;
148 }
149 machine->env = host_env;
150 return machine;
151}
152
153struct machine *machine__new_host(struct perf_env *host_env)
154{
155 return __machine__new_host(host_env, /*kernel_maps=*/true);
156}
157
158static int mmap_handler(const struct perf_tool *tool __maybe_unused,
159 union perf_event *event,
160 struct perf_sample *sample,
161 struct machine *machine)
162{
163 return machine__process_mmap2_event(machine, event, sample);
164}
165
166static int machine__init_live(struct machine *machine, pid_t pid)
167{
168 union perf_event event;
169
170 memset(&event, 0, sizeof(event));
171 return perf_event__synthesize_mmap_events(NULL, &event, pid, pid,
172 mmap_handler, machine, true);
173}
174
175struct machine *machine__new_live(struct perf_env *host_env, bool kernel_maps, pid_t pid)
176{
177 struct machine *machine = __machine__new_host(host_env, kernel_maps);
178
179 if (!machine)
180 return NULL;
181
182 if (machine__init_live(machine, pid)) {
183 machine__delete(machine);
184 return NULL;
185 }
186 return machine;
187}
188
189struct machine *machine__new_kallsyms(struct perf_env *host_env)
190{
191 struct machine *machine = machine__new_host(host_env);
192 /*
193 * FIXME:
194 * 1) We should switch to machine__load_kallsyms(), i.e. not explicitly
195 * ask for not using the kcore parsing code, once this one is fixed
196 * to create a map per module.
197 */
198 if (machine && machine__load_kallsyms(machine, "/proc/kallsyms") <= 0) {
199 machine__delete(machine);
200 machine = NULL;
201 }
202
203 return machine;
204}
205
206void machine__delete_threads(struct machine *machine)
207{
208 threads__remove_all_threads(&machine->threads);
209}
210
211void machine__exit(struct machine *machine)
212{
213 if (machine == NULL)
214 return;
215
216 machine__destroy_kernel_maps(machine);
217 maps__zput(machine->kmaps);
218 dsos__exit(&machine->dsos);
219 machine__exit_vdso(machine);
220 zfree(&machine->root_dir);
221 zfree(&machine->mmap_name);
222 zfree(&machine->current_tid);
223 zfree(&machine->kallsyms_filename);
224
225 threads__exit(&machine->threads);
226}
227
228void machine__delete(struct machine *machine)
229{
230 if (machine) {
231 machine__exit(machine);
232 free(machine);
233 }
234}
235
236int machines__init(struct machines *machines)
237{
238 int err = machine__init(&machines->host, "", HOST_KERNEL_ID);
239
240 machines->guests = RB_ROOT_CACHED;
241 return err;
242}
243
244void machines__exit(struct machines *machines)
245{
246 machine__exit(&machines->host);
247 /* XXX exit guest */
248}
249
250struct machine *machines__add(struct machines *machines, pid_t pid,
251 const char *root_dir)
252{
253 struct rb_node **p = &machines->guests.rb_root.rb_node;
254 struct rb_node *parent = NULL;
255 struct machine *pos, *machine = malloc(sizeof(*machine));
256 bool leftmost = true;
257
258 if (machine == NULL)
259 return NULL;
260
261 if (machine__init(machine, root_dir, pid) != 0) {
262 free(machine);
263 return NULL;
264 }
265
266 while (*p != NULL) {
267 parent = *p;
268 pos = rb_entry(parent, struct machine, rb_node);
269 if (pid < pos->pid)
270 p = &(*p)->rb_left;
271 else {
272 p = &(*p)->rb_right;
273 leftmost = false;
274 }
275 }
276
277 rb_link_node(&machine->rb_node, parent, p);
278 rb_insert_color_cached(&machine->rb_node, &machines->guests, leftmost);
279
280 machine->machines = machines;
281
282 return machine;
283}
284
285void machines__set_comm_exec(struct machines *machines, bool comm_exec)
286{
287 struct rb_node *nd;
288
289 machines->host.comm_exec = comm_exec;
290
291 for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
292 struct machine *machine = rb_entry(nd, struct machine, rb_node);
293
294 machine->comm_exec = comm_exec;
295 }
296}
297
298struct machine *machines__find(struct machines *machines, pid_t pid)
299{
300 struct rb_node **p = &machines->guests.rb_root.rb_node;
301 struct rb_node *parent = NULL;
302 struct machine *machine;
303 struct machine *default_machine = NULL;
304
305 if (pid == HOST_KERNEL_ID)
306 return &machines->host;
307
308 while (*p != NULL) {
309 parent = *p;
310 machine = rb_entry(parent, struct machine, rb_node);
311 if (pid < machine->pid)
312 p = &(*p)->rb_left;
313 else if (pid > machine->pid)
314 p = &(*p)->rb_right;
315 else
316 return machine;
317 if (!machine->pid)
318 default_machine = machine;
319 }
320
321 return default_machine;
322}
323
324struct machine *machines__findnew(struct machines *machines, pid_t pid)
325{
326 char path[PATH_MAX];
327 const char *root_dir = "";
328 struct machine *machine = machines__find(machines, pid);
329
330 if (machine && (machine->pid == pid))
331 goto out;
332
333 if ((pid != HOST_KERNEL_ID) &&
334 (pid != DEFAULT_GUEST_KERNEL_ID) &&
335 (symbol_conf.guestmount)) {
336 if (snprintf(path, sizeof(path), "%s/%d",
337 symbol_conf.guestmount, pid) >= (int)sizeof(path)) {
338 pr_err("Guest path too long for pid %d\n", pid);
339 machine = NULL;
340 goto out;
341 }
342 if (access(path, R_OK)) {
343 static struct strlist *seen;
344
345 if (!seen)
346 seen = strlist__new(NULL, NULL);
347
348 if (!seen || !strlist__has_entry(seen, path)) {
349 pr_err("Can't access file %s\n", path);
350 if (seen)
351 strlist__add(seen, path);
352 }
353 machine = NULL;
354 goto out;
355 }
356 root_dir = path;
357 }
358
359 machine = machines__add(machines, pid, root_dir);
360out:
361 return machine;
362}
363
364struct machine *machines__find_guest(struct machines *machines, pid_t pid)
365{
366 struct machine *machine = machines__find(machines, pid);
367
368 if (!machine)
369 machine = machines__findnew(machines, DEFAULT_GUEST_KERNEL_ID);
370 return machine;
371}
372
373/*
374 * A common case for KVM test programs is that the test program acts as the
375 * hypervisor, creating, running and destroying the virtual machine, and
376 * providing the guest object code from its own object code. In this case,
377 * the VM is not running an OS, but only the functions loaded into it by the
378 * hypervisor test program, and conveniently, loaded at the same virtual
379 * addresses.
380 *
381 * Normally to resolve addresses, MMAP events are needed to map addresses
382 * back to the object code and debug symbols for that object code.
383 *
384 * Currently, there is no way to get such mapping information from guests
385 * but, in the scenario described above, the guest has the same mappings
386 * as the hypervisor, so support for that scenario can be achieved.
387 *
388 * To support that, copy the host thread's maps to the guest thread's maps.
389 * Note, we do not discover the guest until we encounter a guest event,
390 * which works well because it is not until then that we know that the host
391 * thread's maps have been set up.
392 *
393 * This function returns the guest thread. Apart from keeping the data
394 * structures sane, using a thread belonging to the guest machine, instead
395 * of the host thread, allows it to have its own comm (refer
396 * thread__set_guest_comm()).
397 */
398static struct thread *findnew_guest_code(struct machine *machine,
399 struct machine *host_machine,
400 pid_t pid)
401{
402 struct thread *host_thread;
403 struct thread *thread;
404 int err;
405
406 if (!machine)
407 return NULL;
408
409 thread = machine__findnew_thread(machine, -1, pid);
410 if (!thread)
411 return NULL;
412
413 /* Assume maps are set up if there are any */
414 if (!maps__empty(thread__maps(thread)))
415 return thread;
416
417 host_thread = machine__find_thread(host_machine, -1, pid);
418 if (!host_thread)
419 goto out_err;
420
421 thread__set_guest_comm(thread, pid);
422
423 /*
424 * Guest code can be found in hypervisor process at the same address
425 * so copy host maps.
426 */
427 err = maps__copy_from(thread__maps(thread), thread__maps(host_thread));
428 thread__put(host_thread);
429 if (err)
430 goto out_err;
431
432 return thread;
433
434out_err:
435 thread__zput(thread);
436 return NULL;
437}
438
439struct thread *machines__findnew_guest_code(struct machines *machines, pid_t pid)
440{
441 struct machine *host_machine = machines__find(machines, HOST_KERNEL_ID);
442 struct machine *machine = machines__findnew(machines, pid);
443
444 return findnew_guest_code(machine, host_machine, pid);
445}
446
447struct thread *machine__findnew_guest_code(struct machine *machine, pid_t pid)
448{
449 struct machines *machines = machine->machines;
450 struct machine *host_machine;
451
452 if (!machines)
453 return NULL;
454
455 host_machine = machines__find(machines, HOST_KERNEL_ID);
456
457 return findnew_guest_code(machine, host_machine, pid);
458}
459
460void machines__process_guests(struct machines *machines,
461 machine__process_t process, void *data)
462{
463 struct rb_node *nd;
464
465 for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
466 struct machine *pos = rb_entry(nd, struct machine, rb_node);
467 process(pos, data);
468 }
469}
470
471void machines__set_id_hdr_size(struct machines *machines, u16 id_hdr_size)
472{
473 struct rb_node *node;
474 struct machine *machine;
475
476 machines->host.id_hdr_size = id_hdr_size;
477
478 for (node = rb_first_cached(&machines->guests); node;
479 node = rb_next(node)) {
480 machine = rb_entry(node, struct machine, rb_node);
481 machine->id_hdr_size = id_hdr_size;
482 }
483
484 return;
485}
486
487static void machine__update_thread_pid(struct machine *machine,
488 struct thread *th, pid_t pid)
489{
490 struct thread *leader;
491
492 if (pid == thread__pid(th) || pid == -1 || thread__pid(th) != -1)
493 return;
494
495 thread__set_pid(th, pid);
496
497 if (thread__pid(th) == thread__tid(th))
498 return;
499
500 leader = machine__findnew_thread(machine, thread__pid(th), thread__pid(th));
501 if (!leader)
502 goto out_err;
503
504 if (!thread__maps(leader))
505 thread__set_maps(leader, maps__new(machine));
506
507 if (!thread__maps(leader))
508 goto out_err;
509
510 if (thread__maps(th) == thread__maps(leader))
511 goto out_put;
512
513 if (thread__maps(th)) {
514 /*
515 * Maps are created from MMAP events which provide the pid and
516 * tid. Consequently there never should be any maps on a thread
517 * with an unknown pid. Just print an error if there are.
518 */
519 if (!maps__empty(thread__maps(th)))
520 pr_err("Discarding thread maps for %d:%d\n",
521 thread__pid(th), thread__tid(th));
522 maps__put(thread__maps(th));
523 }
524
525 thread__set_maps(th, maps__get(thread__maps(leader)));
526out_put:
527 thread__put(leader);
528 return;
529out_err:
530 pr_err("Failed to join map groups for %d:%d\n", thread__pid(th), thread__tid(th));
531 goto out_put;
532}
533
534/*
535 * Caller must eventually drop thread->refcnt returned with a successful
536 * lookup/new thread inserted.
537 */
538static struct thread *__machine__findnew_thread(struct machine *machine,
539 pid_t pid,
540 pid_t tid,
541 bool create)
542{
543 struct thread *th = threads__find(&machine->threads, tid);
544 bool created;
545
546 if (th) {
547 machine__update_thread_pid(machine, th, pid);
548 return th;
549 }
550 if (!create)
551 return NULL;
552
553 th = threads__findnew(&machine->threads, pid, tid, &created);
554 if (created) {
555 /*
556 * We have to initialize maps separately after rb tree is
557 * updated.
558 *
559 * The reason is that we call machine__findnew_thread within
560 * thread__init_maps to find the thread leader and that would
561 * screwed the rb tree.
562 */
563 if (thread__init_maps(th, machine)) {
564 pr_err("Thread init failed thread %d\n", pid);
565 threads__remove(&machine->threads, th);
566 thread__put(th);
567 return NULL;
568 }
569 } else
570 machine__update_thread_pid(machine, th, pid);
571
572 return th;
573}
574
575struct thread *machine__findnew_thread(struct machine *machine, pid_t pid, pid_t tid)
576{
577 return __machine__findnew_thread(machine, pid, tid, /*create=*/true);
578}
579
580struct thread *machine__find_thread(struct machine *machine, pid_t pid,
581 pid_t tid)
582{
583 return __machine__findnew_thread(machine, pid, tid, /*create=*/false);
584}
585
586/*
587 * Threads are identified by pid and tid, and the idle task has pid == tid == 0.
588 * So here a single thread is created for that, but actually there is a separate
589 * idle task per cpu, so there should be one 'struct thread' per cpu, but there
590 * is only 1. That causes problems for some tools, requiring workarounds. For
591 * example get_idle_thread() in builtin-sched.c, or thread_stack__per_cpu().
592 */
593struct thread *machine__idle_thread(struct machine *machine)
594{
595 struct thread *thread = machine__findnew_thread(machine, 0, 0);
596
597 if (!thread || thread__set_comm(thread, "swapper", 0) ||
598 thread__set_namespaces(thread, 0, NULL))
599 pr_err("problem inserting idle task for machine pid %d\n", machine->pid);
600
601 return thread;
602}
603
604struct comm *machine__thread_exec_comm(struct machine *machine,
605 struct thread *thread)
606{
607 if (machine->comm_exec)
608 return thread__exec_comm(thread);
609 else
610 return thread__comm(thread);
611}
612
613int machine__process_comm_event(struct machine *machine, union perf_event *event,
614 struct perf_sample *sample)
615{
616 struct thread *thread = machine__findnew_thread(machine,
617 event->comm.pid,
618 event->comm.tid);
619 bool exec = event->header.misc & PERF_RECORD_MISC_COMM_EXEC;
620 int err = 0;
621
622 if (exec)
623 machine->comm_exec = true;
624
625 if (dump_trace)
626 perf_event__fprintf_comm(event, stdout);
627
628 if (thread == NULL ||
629 __thread__set_comm(thread, event->comm.comm, sample->time, exec)) {
630 dump_printf("problem processing PERF_RECORD_COMM, skipping event.\n");
631 err = -1;
632 }
633
634 thread__put(thread);
635
636 return err;
637}
638
639int machine__process_namespaces_event(struct machine *machine __maybe_unused,
640 union perf_event *event,
641 struct perf_sample *sample __maybe_unused)
642{
643 struct thread *thread = machine__findnew_thread(machine,
644 event->namespaces.pid,
645 event->namespaces.tid);
646 int err = 0;
647
648 WARN_ONCE(event->namespaces.nr_namespaces > NR_NAMESPACES,
649 "\nWARNING: kernel seems to support more namespaces than perf"
650 " tool.\nTry updating the perf tool..\n\n");
651
652 WARN_ONCE(event->namespaces.nr_namespaces < NR_NAMESPACES,
653 "\nWARNING: perf tool seems to support more namespaces than"
654 " the kernel.\nTry updating the kernel..\n\n");
655
656 if (dump_trace)
657 perf_event__fprintf_namespaces(event, stdout);
658
659 if (thread == NULL ||
660 thread__set_namespaces(thread, sample->time, &event->namespaces)) {
661 dump_printf("problem processing PERF_RECORD_NAMESPACES, skipping event.\n");
662 err = -1;
663 }
664
665 thread__put(thread);
666
667 return err;
668}
669
670int machine__process_cgroup_event(struct machine *machine,
671 union perf_event *event,
672 struct perf_sample *sample __maybe_unused)
673{
674 struct cgroup *cgrp;
675
676 if (dump_trace)
677 perf_event__fprintf_cgroup(event, stdout);
678
679 cgrp = cgroup__findnew(machine->env, event->cgroup.id, event->cgroup.path);
680 if (cgrp == NULL)
681 return -ENOMEM;
682
683 return 0;
684}
685
686int machine__process_lost_event(struct machine *machine __maybe_unused,
687 union perf_event *event, struct perf_sample *sample __maybe_unused)
688{
689 dump_printf(": id:%" PRI_lu64 ": lost:%" PRI_lu64 "\n",
690 event->lost.id, event->lost.lost);
691 return 0;
692}
693
694int machine__process_lost_samples_event(struct machine *machine __maybe_unused,
695 union perf_event *event, struct perf_sample *sample)
696{
697 dump_printf(": id:%" PRIu64 ": lost samples :%" PRI_lu64 "%s\n",
698 sample->id, event->lost_samples.lost,
699 event->header.misc & PERF_RECORD_MISC_LOST_SAMPLES_BPF ? " (BPF)" : "");
700 return 0;
701}
702
703int machine__process_aux_event(struct machine *machine __maybe_unused,
704 union perf_event *event)
705{
706 if (dump_trace)
707 perf_event__fprintf_aux(event, stdout);
708 return 0;
709}
710
711int machine__process_itrace_start_event(struct machine *machine __maybe_unused,
712 union perf_event *event)
713{
714 if (dump_trace)
715 perf_event__fprintf_itrace_start(event, stdout);
716 return 0;
717}
718
719int machine__process_aux_output_hw_id_event(struct machine *machine __maybe_unused,
720 union perf_event *event)
721{
722 if (dump_trace)
723 perf_event__fprintf_aux_output_hw_id(event, stdout);
724 return 0;
725}
726
727int machine__process_switch_event(struct machine *machine __maybe_unused,
728 union perf_event *event)
729{
730 bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
731
732 if (dump_trace)
733 perf_event__fprintf_switch(event, stdout);
734 machine->parallelism += out ? -1 : 1;
735 return 0;
736}
737
738static int machine__process_ksymbol_register(struct machine *machine,
739 union perf_event *event,
740 struct perf_sample *sample __maybe_unused)
741{
742 struct symbol *sym;
743 struct dso *dso = NULL;
744 struct map *map;
745 int err = 0;
746
747 /* Ignore mapping symbols in ksymbol events - check early before any state mutation */
748 if (is_ignored_kernel_symbol(event->ksymbol.name))
749 return 0;
750
751 map = maps__find(machine__kernel_maps(machine), event->ksymbol.addr);
752
753 if (!map) {
754 dso = dso__new(event->ksymbol.name);
755
756 if (!dso) {
757 err = -ENOMEM;
758 goto out;
759 }
760 dso__set_kernel(dso, DSO_SPACE__KERNEL);
761 map = map__new2(0, dso);
762 if (!map) {
763 err = -ENOMEM;
764 goto out;
765 }
766 if (event->ksymbol.ksym_type == PERF_RECORD_KSYMBOL_TYPE_OOL) {
767 dso__set_binary_type(dso, DSO_BINARY_TYPE__OOL);
768 dso__data(dso)->file_size = event->ksymbol.len;
769 dso__set_loaded(dso);
770 }
771
772 map__set_start(map, event->ksymbol.addr);
773 map__set_end(map, map__start(map) + event->ksymbol.len);
774 err = maps__fixup_overlap_and_insert(machine__kernel_maps(machine), map);
775 if (err) {
776 err = -ENOMEM;
777 goto out;
778 }
779
780 dso__set_loaded(dso);
781
782 if (is_bpf_image(event->ksymbol.name)) {
783 dso__set_binary_type(dso, DSO_BINARY_TYPE__BPF_IMAGE);
784 dso__set_long_name(dso, "", false);
785 }
786 } else {
787 dso = dso__get(map__dso(map));
788 }
789
790 sym = symbol__new(map__map_ip(map, map__start(map)),
791 event->ksymbol.len,
792 0, 0, event->ksymbol.name);
793 if (!sym) {
794 err = -ENOMEM;
795 goto out;
796 }
797 dso__insert_symbol(dso, sym);
798out:
799 map__put(map);
800 dso__put(dso);
801 return err;
802}
803
804static int machine__process_ksymbol_unregister(struct machine *machine,
805 union perf_event *event,
806 struct perf_sample *sample __maybe_unused)
807{
808 struct symbol *sym;
809 struct map *map;
810
811 /* Ignore mapping symbols in ksymbol events */
812 if (is_ignored_kernel_symbol(event->ksymbol.name))
813 return 0;
814
815 map = maps__find(machine__kernel_maps(machine), event->ksymbol.addr);
816 if (!map)
817 return 0;
818
819 if (!RC_CHK_EQUAL(map, machine->vmlinux_map))
820 maps__remove(machine__kernel_maps(machine), map);
821 else {
822 struct dso *dso = map__dso(map);
823
824 sym = dso__find_symbol(dso, map__map_ip(map, map__start(map)));
825 if (sym)
826 dso__delete_symbol(dso, sym);
827 }
828 map__put(map);
829 return 0;
830}
831
832int machine__process_ksymbol(struct machine *machine __maybe_unused,
833 union perf_event *event,
834 struct perf_sample *sample)
835{
836 if (dump_trace)
837 perf_event__fprintf_ksymbol(event, stdout);
838
839 if (event->header.size < offsetof(struct perf_record_ksymbol, name) + 2 ||
840 !memchr(event->ksymbol.name, '\0',
841 event->header.size - offsetof(struct perf_record_ksymbol, name)))
842 return -EINVAL;
843
844 /* no need to process non-JIT BPF as it cannot get samples */
845 if (event->ksymbol.len == 0)
846 return 0;
847
848 if (event->ksymbol.flags & PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER)
849 return machine__process_ksymbol_unregister(machine, event,
850 sample);
851 return machine__process_ksymbol_register(machine, event, sample);
852}
853
854int machine__process_text_poke(struct machine *machine, union perf_event *event,
855 struct perf_sample *sample __maybe_unused)
856{
857 struct map *map = maps__find(machine__kernel_maps(machine), event->text_poke.addr);
858 u8 cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
859 struct dso *dso = map ? map__dso(map) : NULL;
860
861 if (dump_trace)
862 perf_event__fprintf_text_poke(event, machine, stdout);
863
864 if (!event->text_poke.new_len)
865 goto out;
866
867 if (cpumode != PERF_RECORD_MISC_KERNEL) {
868 pr_debug("%s: unsupported cpumode - ignoring\n", __func__);
869 goto out;
870 }
871
872 if (dso) {
873 u8 *new_bytes = event->text_poke.bytes + event->text_poke.old_len;
874 int ret;
875
876 /*
877 * Kernel maps might be changed when loading symbols so loading
878 * must be done prior to using kernel maps.
879 */
880 map__load(map);
881 ret = dso__data_write_cache_addr(dso, map, machine,
882 event->text_poke.addr,
883 new_bytes,
884 event->text_poke.new_len);
885 if (ret != event->text_poke.new_len)
886 pr_debug("Failed to write kernel text poke at %#" PRI_lx64 "\n",
887 event->text_poke.addr);
888 } else {
889 pr_debug("Failed to find kernel text poke address map for %#" PRI_lx64 "\n",
890 event->text_poke.addr);
891 }
892out:
893 map__put(map);
894 return 0;
895}
896
897static struct map *machine__addnew_module_map(struct machine *machine, u64 start,
898 const char *filename)
899{
900 struct map *map = NULL;
901 struct kmod_path m;
902 struct dso *dso;
903 int err;
904
905 if (kmod_path__parse_name(&m, filename))
906 return NULL;
907
908 dso = dsos__findnew_module_dso(&machine->dsos, machine, &m, filename);
909 if (dso == NULL)
910 goto out;
911
912 map = map__new2(start, dso);
913 if (map == NULL)
914 goto out;
915
916 err = maps__insert(machine__kernel_maps(machine), map);
917 /* If maps__insert failed, return NULL. */
918 if (err) {
919 map__put(map);
920 map = NULL;
921 }
922out:
923 /* put the dso here, corresponding to machine__findnew_module_dso */
924 dso__put(dso);
925 zfree(&m.name);
926 return map;
927}
928
929size_t machines__fprintf_dsos(struct machines *machines, FILE *fp)
930{
931 struct rb_node *nd;
932 size_t ret = dsos__fprintf(&machines->host.dsos, fp);
933
934 for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
935 struct machine *pos = rb_entry(nd, struct machine, rb_node);
936 ret += dsos__fprintf(&pos->dsos, fp);
937 }
938
939 return ret;
940}
941
942size_t machine__fprintf_dsos_buildid(struct machine *m, FILE *fp,
943 bool (skip)(struct dso *dso, int parm), int parm)
944{
945 return dsos__fprintf_buildid(&m->dsos, fp, skip, parm);
946}
947
948size_t machines__fprintf_dsos_buildid(struct machines *machines, FILE *fp,
949 bool (skip)(struct dso *dso, int parm), int parm)
950{
951 struct rb_node *nd;
952 size_t ret = machine__fprintf_dsos_buildid(&machines->host, fp, skip, parm);
953
954 for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
955 struct machine *pos = rb_entry(nd, struct machine, rb_node);
956 ret += machine__fprintf_dsos_buildid(pos, fp, skip, parm);
957 }
958 return ret;
959}
960
961struct machine_fprintf_cb_args {
962 FILE *fp;
963 size_t printed;
964};
965
966static int machine_fprintf_cb(struct thread *thread, void *data)
967{
968 struct machine_fprintf_cb_args *args = data;
969
970 /* TODO: handle fprintf errors. */
971 args->printed += thread__fprintf(thread, args->fp);
972 return 0;
973}
974
975size_t machine__fprintf(struct machine *machine, FILE *fp)
976{
977 struct machine_fprintf_cb_args args = {
978 .fp = fp,
979 .printed = 0,
980 };
981 size_t ret = fprintf(fp, "Threads: %zu\n", threads__nr(&machine->threads));
982
983 machine__for_each_thread(machine, machine_fprintf_cb, &args);
984 return ret + args.printed;
985}
986
987static struct dso *machine__get_kernel(struct machine *machine)
988{
989 const char *vmlinux_name = machine->mmap_name;
990 struct dso *kernel;
991
992 if (machine__is_host(machine)) {
993 if (symbol_conf.vmlinux_name)
994 vmlinux_name = symbol_conf.vmlinux_name;
995
996 kernel = machine__findnew_kernel(machine, vmlinux_name,
997 "[kernel]", DSO_SPACE__KERNEL);
998 } else {
999 if (symbol_conf.default_guest_vmlinux_name)
1000 vmlinux_name = symbol_conf.default_guest_vmlinux_name;
1001
1002 kernel = machine__findnew_kernel(machine, vmlinux_name,
1003 "[guest.kernel]",
1004 DSO_SPACE__KERNEL_GUEST);
1005 }
1006
1007 if (kernel != NULL && (!dso__has_build_id(kernel)))
1008 dso__read_running_kernel_build_id(kernel, machine);
1009
1010 return kernel;
1011}
1012
1013void machine__get_kallsyms_filename(struct machine *machine, char *buf,
1014 size_t bufsz)
1015{
1016 if (machine__is_default_guest(machine))
1017 scnprintf(buf, bufsz, "%s", symbol_conf.default_guest_kallsyms);
1018 else
1019 scnprintf(buf, bufsz, "%s/proc/kallsyms", machine->root_dir);
1020}
1021
1022const char *ref_reloc_sym_names[] = {"_text", "_stext", NULL};
1023
1024/* Figure out the start address of kernel map from /proc/kallsyms.
1025 * Returns the name of the start symbol in *symbol_name. Pass in NULL as
1026 * symbol_name if it's not that important.
1027 */
1028static int machine__get_running_kernel_start(struct machine *machine,
1029 const char **symbol_name,
1030 u64 *start, u64 *end)
1031{
1032 char filename[PATH_MAX];
1033 int i, err = -1;
1034 const char *name;
1035 u64 addr = 0;
1036
1037 machine__get_kallsyms_filename(machine, filename, PATH_MAX);
1038
1039 if (symbol__restricted_filename(filename, "/proc/kallsyms"))
1040 return 0;
1041
1042 for (i = 0; (name = ref_reloc_sym_names[i]) != NULL; i++) {
1043 err = kallsyms__get_function_start(filename, name, &addr);
1044 if (!err)
1045 break;
1046 }
1047
1048 if (err)
1049 return -1;
1050
1051 if (symbol_name)
1052 *symbol_name = name;
1053
1054 *start = addr;
1055
1056 err = kallsyms__get_symbol_start(filename, "_edata", &addr);
1057 if (err)
1058 err = kallsyms__get_symbol_start(filename, "_etext", &addr);
1059 if (!err)
1060 *end = addr;
1061
1062 return 0;
1063}
1064
1065int machine__create_extra_kernel_map(struct machine *machine,
1066 struct dso *kernel,
1067 struct extra_kernel_map *xm)
1068{
1069 struct kmap *kmap;
1070 struct map *map;
1071 int err;
1072
1073 map = map__new2(xm->start, kernel);
1074 if (!map)
1075 return -ENOMEM;
1076
1077 map__set_end(map, xm->end);
1078 map__set_pgoff(map, xm->pgoff);
1079
1080 kmap = map__kmap(map);
1081
1082 strlcpy(kmap->name, xm->name, KMAP_NAME_LEN);
1083
1084 err = maps__insert(machine__kernel_maps(machine), map);
1085
1086 if (!err) {
1087 pr_debug2("Added extra kernel map %s %" PRIx64 "-%" PRIx64 "\n",
1088 kmap->name, map__start(map), map__end(map));
1089 }
1090
1091 map__put(map);
1092
1093 return err;
1094}
1095
1096static u64 find_entry_trampoline(struct dso *dso)
1097{
1098 /* Duplicates are removed so lookup all aliases */
1099 const char *syms[] = {
1100 "_entry_trampoline",
1101 "__entry_trampoline_start",
1102 "entry_SYSCALL_64_trampoline",
1103 };
1104 struct symbol *sym = dso__first_symbol(dso);
1105 unsigned int i;
1106
1107 for (; sym; sym = dso__next_symbol(sym)) {
1108 if (symbol__binding(sym) != STB_GLOBAL)
1109 continue;
1110 for (i = 0; i < ARRAY_SIZE(syms); i++) {
1111 if (!strcmp(sym->name, syms[i]))
1112 return sym->start;
1113 }
1114 }
1115
1116 return 0;
1117}
1118
1119/*
1120 * These values can be used for kernels that do not have symbols for the entry
1121 * trampolines in kallsyms.
1122 */
1123#define X86_64_CPU_ENTRY_AREA_PER_CPU 0xfffffe0000000000ULL
1124#define X86_64_CPU_ENTRY_AREA_SIZE 0x2c000
1125#define X86_64_ENTRY_TRAMPOLINE 0x6000
1126
1127struct machine__map_x86_64_entry_trampolines_args {
1128 struct maps *kmaps;
1129 bool found;
1130};
1131
1132static int machine__map_x86_64_entry_trampolines_cb(struct map *map, void *data)
1133{
1134 struct machine__map_x86_64_entry_trampolines_args *args = data;
1135 struct map *dest_map;
1136 struct kmap *kmap = __map__kmap(map);
1137
1138 if (!kmap || !is_entry_trampoline(kmap->name))
1139 return 0;
1140
1141 dest_map = maps__find(args->kmaps, map__pgoff(map));
1142 if (RC_CHK_ACCESS(dest_map) != RC_CHK_ACCESS(map))
1143 map__set_pgoff(map, map__map_ip(dest_map, map__pgoff(map)));
1144
1145 map__put(dest_map);
1146 args->found = true;
1147 return 0;
1148}
1149
1150/* Map x86_64 PTI entry trampolines */
1151int machine__map_x86_64_entry_trampolines(struct machine *machine,
1152 struct dso *kernel)
1153{
1154 struct machine__map_x86_64_entry_trampolines_args args = {
1155 .kmaps = machine__kernel_maps(machine),
1156 .found = false,
1157 };
1158 int nr_cpus_avail, cpu;
1159 u64 pgoff;
1160
1161 /*
1162 * In the vmlinux case, pgoff is a virtual address which must now be
1163 * mapped to a vmlinux offset.
1164 */
1165 maps__for_each_map(args.kmaps, machine__map_x86_64_entry_trampolines_cb, &args);
1166
1167 if (args.found || machine->trampolines_mapped)
1168 return 0;
1169
1170 pgoff = find_entry_trampoline(kernel);
1171 if (!pgoff)
1172 return 0;
1173
1174 nr_cpus_avail = machine__nr_cpus_avail(machine);
1175
1176 /* Add a 1 page map for each CPU's entry trampoline */
1177 for (cpu = 0; cpu < nr_cpus_avail; cpu++) {
1178 u64 va = X86_64_CPU_ENTRY_AREA_PER_CPU +
1179 cpu * X86_64_CPU_ENTRY_AREA_SIZE +
1180 X86_64_ENTRY_TRAMPOLINE;
1181 struct extra_kernel_map xm = {
1182 .start = va,
1183 .end = va + page_size,
1184 .pgoff = pgoff,
1185 };
1186
1187 strlcpy(xm.name, ENTRY_TRAMPOLINE_NAME, KMAP_NAME_LEN);
1188
1189 if (machine__create_extra_kernel_map(machine, kernel, &xm) < 0)
1190 return -1;
1191 }
1192
1193 machine->trampolines_mapped = nr_cpus_avail;
1194
1195 return 0;
1196}
1197
1198int __weak machine__create_extra_kernel_maps(struct machine *machine __maybe_unused,
1199 struct dso *kernel __maybe_unused)
1200{
1201 return 0;
1202}
1203
1204static int
1205__machine__create_kernel_maps(struct machine *machine, struct dso *kernel)
1206{
1207 /* In case of renewal the kernel map, destroy previous one */
1208 machine__destroy_kernel_maps(machine);
1209
1210 map__put(machine->vmlinux_map);
1211 machine->vmlinux_map = map__new2(0, kernel);
1212 if (machine->vmlinux_map == NULL)
1213 return -ENOMEM;
1214
1215 map__set_mapping_type(machine->vmlinux_map, MAPPING_TYPE__IDENTITY);
1216 return maps__insert(machine__kernel_maps(machine), machine->vmlinux_map);
1217}
1218
1219void machine__destroy_kernel_maps(struct machine *machine)
1220{
1221 struct kmap *kmap;
1222 struct map *map = machine__kernel_map(machine);
1223
1224 if (map == NULL)
1225 return;
1226
1227 kmap = map__kmap(map);
1228 maps__remove(machine__kernel_maps(machine), map);
1229 if (kmap && kmap->ref_reloc_sym) {
1230 zfree((char **)&kmap->ref_reloc_sym->name);
1231 zfree(&kmap->ref_reloc_sym);
1232 }
1233
1234 map__zput(machine->vmlinux_map);
1235}
1236
1237int machines__create_guest_kernel_maps(struct machines *machines)
1238{
1239 int ret = 0;
1240 struct dirent **namelist = NULL;
1241 int i, items = 0;
1242 char path[PATH_MAX];
1243 pid_t pid;
1244 char *endp;
1245
1246 if (symbol_conf.default_guest_vmlinux_name ||
1247 symbol_conf.default_guest_modules ||
1248 symbol_conf.default_guest_kallsyms) {
1249 machines__create_kernel_maps(machines, DEFAULT_GUEST_KERNEL_ID);
1250 }
1251
1252 if (symbol_conf.guestmount) {
1253 items = scandir(symbol_conf.guestmount, &namelist, NULL, NULL);
1254 if (items <= 0)
1255 return -ENOENT;
1256 for (i = 0; i < items; i++) {
1257 if (!isdigit(namelist[i]->d_name[0])) {
1258 /* Filter out . and .. */
1259 free(namelist[i]);
1260 continue;
1261 }
1262 errno = 0;
1263 pid = (pid_t)strtol(namelist[i]->d_name, &endp, 10);
1264 if ((*endp != '\0') ||
1265 (endp == namelist[i]->d_name) ||
1266 (errno == ERANGE)) {
1267 pr_debug("invalid directory (%s). Skipping.\n",
1268 namelist[i]->d_name);
1269 free(namelist[i]);
1270 continue;
1271 }
1272 if (snprintf(path, sizeof(path), "%s/%s/proc/kallsyms",
1273 symbol_conf.guestmount,
1274 namelist[i]->d_name) >= (int)sizeof(path)) {
1275 pr_debug("Guest kallsyms path too long for %s. Skipping.\n",
1276 namelist[i]->d_name);
1277 free(namelist[i]);
1278 continue;
1279 }
1280 if (access(path, R_OK)) {
1281 pr_debug("Can't access file %s\n", path);
1282 free(namelist[i]);
1283 continue;
1284 }
1285 machines__create_kernel_maps(machines, pid);
1286 free(namelist[i]);
1287 }
1288 free(namelist);
1289 }
1290
1291 return ret;
1292}
1293
1294void machines__destroy_kernel_maps(struct machines *machines)
1295{
1296 struct rb_node *next = rb_first_cached(&machines->guests);
1297
1298 machine__destroy_kernel_maps(&machines->host);
1299
1300 while (next) {
1301 struct machine *pos = rb_entry(next, struct machine, rb_node);
1302
1303 next = rb_next(&pos->rb_node);
1304 rb_erase_cached(&pos->rb_node, &machines->guests);
1305 machine__delete(pos);
1306 }
1307}
1308
1309int machines__create_kernel_maps(struct machines *machines, pid_t pid)
1310{
1311 struct machine *machine = machines__findnew(machines, pid);
1312
1313 if (machine == NULL)
1314 return -1;
1315
1316 return machine__create_kernel_maps(machine);
1317}
1318
1319int machine__load_kallsyms(struct machine *machine, const char *filename)
1320{
1321 struct map *map = machine__kernel_map(machine);
1322 struct dso *dso = map__dso(map);
1323 int ret = __dso__load_kallsyms(dso, filename, map, true);
1324
1325 if (ret > 0) {
1326 dso__set_loaded(dso);
1327 /*
1328 * Since /proc/kallsyms will have multiple sessions for the
1329 * kernel, with modules between them, fixup the end of all
1330 * sections.
1331 */
1332 maps__fixup_end(machine__kernel_maps(machine));
1333 }
1334
1335 return ret;
1336}
1337
1338int machine__load_vmlinux_path(struct machine *machine)
1339{
1340 struct map *map = machine__kernel_map(machine);
1341 struct dso *dso = map__dso(map);
1342 int ret = dso__load_vmlinux_path(dso, map);
1343
1344 if (ret > 0)
1345 dso__set_loaded(dso);
1346
1347 return ret;
1348}
1349
1350static char *get_kernel_version(const char *root_dir)
1351{
1352 char version[PATH_MAX];
1353 FILE *file;
1354 char *name, *tmp;
1355 const char *prefix = "Linux version ";
1356
1357 snprintf(version, sizeof(version), "%s/proc/version", root_dir);
1358 file = fopen(version, "r");
1359 if (!file)
1360 return NULL;
1361
1362 tmp = fgets(version, sizeof(version), file);
1363 fclose(file);
1364 if (!tmp)
1365 return NULL;
1366
1367 name = strstr(version, prefix);
1368 if (!name)
1369 return NULL;
1370 name += strlen(prefix);
1371 tmp = strchr(name, ' ');
1372 if (tmp)
1373 *tmp = '\0';
1374
1375 return strdup(name);
1376}
1377
1378static bool is_kmod_dso(struct dso *dso)
1379{
1380 return dso__symtab_type(dso) == DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE ||
1381 dso__symtab_type(dso) == DSO_BINARY_TYPE__GUEST_KMODULE;
1382}
1383
1384static int maps__set_module_path(struct maps *maps, const char *path, struct kmod_path *m)
1385{
1386 char *long_name;
1387 struct dso *dso;
1388 struct map *map = maps__find_by_name(maps, m->name);
1389
1390 if (map == NULL)
1391 return 0;
1392
1393 long_name = strdup(path);
1394 if (long_name == NULL) {
1395 map__put(map);
1396 return -ENOMEM;
1397 }
1398
1399 dso = map__dso(map);
1400 dso__set_long_name(dso, long_name, true);
1401 dso__kernel_module_get_build_id(dso, "");
1402
1403 /*
1404 * Full name could reveal us kmod compression, so
1405 * we need to update the symtab_type if needed.
1406 */
1407 if (m->comp && is_kmod_dso(dso)) {
1408 dso__set_symtab_type(dso, dso__symtab_type(dso)+1);
1409 dso__set_comp(dso, m->comp);
1410 }
1411 map__put(map);
1412 return 0;
1413}
1414
1415static int maps__set_modules_path_dir(struct maps *maps, char *path, size_t path_size, int depth)
1416{
1417 struct io_dirent64 *dent;
1418 struct io_dir iod;
1419 size_t root_len = strlen(path);
1420 int ret = 0;
1421
1422 io_dir__init(&iod, open(path, O_CLOEXEC | O_DIRECTORY | O_RDONLY));
1423 if (iod.dirfd < 0) {
1424 pr_debug("%s: cannot open %s dir\n", __func__, path);
1425 return -1;
1426 }
1427 /* Bounds check, should never happen. */
1428 if (root_len >= path_size) {
1429 ret = -1;
1430 goto out;
1431 }
1432 path[root_len++] = '/';
1433 while ((dent = io_dir__readdir(&iod)) != NULL) {
1434 if (io_dir__is_dir(&iod, dent)) {
1435 if (!strcmp(dent->d_name, ".") ||
1436 !strcmp(dent->d_name, ".."))
1437 continue;
1438
1439 /* Do not follow top-level source and build symlinks */
1440 if (depth == 0) {
1441 if (!strcmp(dent->d_name, "source") ||
1442 !strcmp(dent->d_name, "build"))
1443 continue;
1444 }
1445
1446 /* Bounds check, should never happen. */
1447 if (root_len + strlen(dent->d_name) >= path_size)
1448 continue;
1449
1450 strcpy(path + root_len, dent->d_name);
1451 ret = maps__set_modules_path_dir(maps, path, path_size, depth + 1);
1452 if (ret < 0)
1453 goto out;
1454 } else {
1455 struct kmod_path m;
1456
1457 ret = kmod_path__parse_name(&m, dent->d_name);
1458 if (ret)
1459 goto out;
1460
1461 if (m.kmod) {
1462 /* Bounds check, should never happen. */
1463 if (root_len + strlen(dent->d_name) < path_size) {
1464 strcpy(path + root_len, dent->d_name);
1465 ret = maps__set_module_path(maps, path, &m);
1466
1467 }
1468 }
1469 zfree(&m.name);
1470
1471 if (ret)
1472 goto out;
1473 }
1474 }
1475
1476out:
1477 close(iod.dirfd);
1478 return ret;
1479}
1480
1481static int machine__set_modules_path(struct machine *machine)
1482{
1483 char *version;
1484 char modules_path[PATH_MAX];
1485
1486 version = get_kernel_version(machine->root_dir);
1487 if (!version)
1488 return -1;
1489
1490 snprintf(modules_path, sizeof(modules_path), "%s/lib/modules/%s",
1491 machine->root_dir, version);
1492 free(version);
1493
1494 return maps__set_modules_path_dir(machine__kernel_maps(machine),
1495 modules_path, sizeof(modules_path), 0);
1496}
1497int __weak arch__fix_module_text_start(u64 *start __maybe_unused,
1498 u64 *size __maybe_unused,
1499 const char *name __maybe_unused)
1500{
1501 return 0;
1502}
1503
1504static int machine__create_module(void *arg, const char *name, u64 start,
1505 u64 size)
1506{
1507 struct machine *machine = arg;
1508 struct map *map;
1509
1510 if (arch__fix_module_text_start(&start, &size, name) < 0)
1511 return -1;
1512
1513 map = machine__addnew_module_map(machine, start, name);
1514 if (map == NULL)
1515 return -1;
1516 map__set_end(map, start + size);
1517
1518 dso__kernel_module_get_build_id(map__dso(map), machine->root_dir);
1519 map__put(map);
1520 return 0;
1521}
1522
1523static int machine__create_modules(struct machine *machine)
1524{
1525 const char *modules;
1526 char path[PATH_MAX];
1527
1528 if (machine__is_default_guest(machine)) {
1529 modules = symbol_conf.default_guest_modules;
1530 } else {
1531 snprintf(path, PATH_MAX, "%s/proc/modules", machine->root_dir);
1532 modules = path;
1533 }
1534
1535 if (symbol__restricted_filename(modules, "/proc/modules"))
1536 return -1;
1537
1538 if (modules__parse(modules, machine, machine__create_module))
1539 return -1;
1540
1541 if (!machine__set_modules_path(machine))
1542 return 0;
1543
1544 pr_debug("Problems setting modules path maps, continuing anyway...\n");
1545
1546 return 0;
1547}
1548
1549static void machine__set_kernel_mmap(struct machine *machine,
1550 u64 start, u64 end)
1551{
1552 map__set_start(machine->vmlinux_map, start);
1553 map__set_end(machine->vmlinux_map, end);
1554 /*
1555 * Be a bit paranoid here, some perf.data file came with
1556 * a zero sized synthesized MMAP event for the kernel.
1557 */
1558 if (start == 0 && end == 0)
1559 map__set_end(machine->vmlinux_map, ~0ULL);
1560}
1561
1562struct kernel_mmap_mutation_ctx {
1563 u64 start;
1564 u64 end;
1565};
1566
1567static int kernel_mmap_mutate_cb(struct map *map, void *data)
1568{
1569 struct kernel_mmap_mutation_ctx *ctx = data;
1570
1571 map__set_start(map, ctx->start);
1572 map__set_end(map, ctx->end);
1573 if (ctx->start == 0 && ctx->end == 0)
1574 map__set_end(map, ~0ULL);
1575 return 0;
1576}
1577
1578static int machine__update_kernel_mmap(struct machine *machine,
1579 u64 start, u64 end)
1580{
1581 struct kernel_mmap_mutation_ctx ctx = { .start = start, .end = end };
1582
1583 return maps__mutate_mapping(machine__kernel_maps(machine),
1584 machine->vmlinux_map,
1585 kernel_mmap_mutate_cb, &ctx);
1586}
1587
1588int machine__create_kernel_maps(struct machine *machine)
1589{
1590 struct dso *kernel = machine__get_kernel(machine);
1591 const char *name = NULL;
1592 u64 start = 0, end = ~0ULL;
1593 int ret;
1594
1595 if (kernel == NULL)
1596 return -1;
1597
1598 ret = __machine__create_kernel_maps(machine, kernel);
1599 if (ret < 0)
1600 goto out_put;
1601
1602 if (symbol_conf.use_modules && machine__create_modules(machine) < 0) {
1603 if (machine__is_host(machine))
1604 pr_debug("Problems creating module maps, "
1605 "continuing anyway...\n");
1606 else
1607 pr_debug("Problems creating module maps for guest %d, "
1608 "continuing anyway...\n", machine->pid);
1609 }
1610
1611 if (!machine__get_running_kernel_start(machine, &name, &start, &end)) {
1612 if (name &&
1613 map__set_kallsyms_ref_reloc_sym(machine->vmlinux_map, name, start)) {
1614 machine__destroy_kernel_maps(machine);
1615 ret = -1;
1616 goto out_put;
1617 }
1618
1619 /*
1620 * we have a real start address now, so re-order the kmaps
1621 * assume it's the last in the kmaps
1622 */
1623 ret = machine__update_kernel_mmap(machine, start, end);
1624 if (ret < 0)
1625 goto out_put;
1626 }
1627
1628 if (machine__create_extra_kernel_maps(machine, kernel))
1629 pr_debug("Problems creating extra kernel maps, continuing anyway...\n");
1630
1631 if (end == ~0ULL) {
1632 /* update end address of the kernel map using adjacent module address */
1633 struct map *next = maps__find_next_entry(machine__kernel_maps(machine),
1634 machine__kernel_map(machine));
1635
1636 if (next) {
1637 machine__set_kernel_mmap(machine, start, map__start(next));
1638 map__put(next);
1639 }
1640 }
1641
1642 maps__fixup_end(machine__kernel_maps(machine));
1643
1644out_put:
1645 dso__put(kernel);
1646 return ret;
1647}
1648
1649static int machine__uses_kcore_cb(struct dso *dso, void *data __maybe_unused)
1650{
1651 return dso__is_kcore(dso) ? 1 : 0;
1652}
1653
1654static bool machine__uses_kcore(struct machine *machine)
1655{
1656 return dsos__for_each_dso(&machine->dsos, machine__uses_kcore_cb, NULL) != 0 ? true : false;
1657}
1658
1659static bool machine__is(struct machine *machine, uint16_t e_machine)
1660{
1661 if (!machine)
1662 return false;
1663
1664 if (!machine->env) {
1665 if (machine__is_host(machine))
1666 return e_machine == EM_HOST;
1667 return false;
1668 }
1669
1670 return perf_env__e_machine(machine->env, NULL) == e_machine;
1671}
1672
1673static bool perf_event__is_extra_kernel_mmap(struct machine *machine,
1674 struct extra_kernel_map *xm)
1675{
1676 return machine__is(machine, EM_X86_64) &&
1677 is_entry_trampoline(xm->name);
1678}
1679
1680static int machine__process_extra_kernel_map(struct machine *machine,
1681 struct extra_kernel_map *xm)
1682{
1683 struct dso *kernel = machine__kernel_dso(machine);
1684
1685 if (kernel == NULL)
1686 return -1;
1687
1688 return machine__create_extra_kernel_map(machine, kernel, xm);
1689}
1690
1691static int machine__process_kernel_mmap_event(struct machine *machine,
1692 struct extra_kernel_map *xm,
1693 struct build_id *bid)
1694{
1695 enum dso_space_type dso_space;
1696 bool is_kernel_mmap;
1697 const char *mmap_name = machine->mmap_name;
1698
1699 /* If we have maps from kcore then we do not need or want any others */
1700 if (machine__uses_kcore(machine))
1701 return 0;
1702
1703 if (machine__is_host(machine))
1704 dso_space = DSO_SPACE__KERNEL;
1705 else
1706 dso_space = DSO_SPACE__KERNEL_GUEST;
1707
1708 is_kernel_mmap = memcmp(xm->name, mmap_name, strlen(mmap_name) - 1) == 0;
1709 if (!is_kernel_mmap && !machine__is_host(machine)) {
1710 /*
1711 * If the event was recorded inside the guest and injected into
1712 * the host perf.data file, then it will match a host mmap_name,
1713 * so try that - see machine__set_mmap_name().
1714 */
1715 mmap_name = "[kernel.kallsyms]";
1716 is_kernel_mmap = memcmp(xm->name, mmap_name, strlen(mmap_name) - 1) == 0;
1717 }
1718 if (xm->name[0] == '/' ||
1719 (!is_kernel_mmap && xm->name[0] == '[')) {
1720 struct map *map = machine__addnew_module_map(machine, xm->start, xm->name);
1721
1722 if (map == NULL)
1723 goto out_problem;
1724
1725 map__set_end(map, map__start(map) + xm->end - xm->start);
1726
1727 if (build_id__is_defined(bid))
1728 dso__set_build_id(map__dso(map), bid);
1729
1730 map__put(map);
1731 } else if (is_kernel_mmap) {
1732 const char *symbol_name = xm->name + strlen(mmap_name);
1733 /*
1734 * Should be there already, from the build-id table in
1735 * the header.
1736 */
1737 struct dso *kernel = dsos__find_kernel_dso(&machine->dsos);
1738
1739 if (kernel == NULL)
1740 kernel = machine__findnew_dso(machine, machine->mmap_name);
1741 if (kernel == NULL)
1742 goto out_problem;
1743
1744 dso__set_kernel(kernel, dso_space);
1745 if (__machine__create_kernel_maps(machine, kernel) < 0) {
1746 dso__put(kernel);
1747 goto out_problem;
1748 }
1749
1750 if (strstr(dso__long_name(kernel), "vmlinux"))
1751 dso__set_short_name(kernel, "[kernel.vmlinux]", false);
1752
1753 if (machine__update_kernel_mmap(machine, xm->start, xm->end) < 0) {
1754 dso__put(kernel);
1755 goto out_problem;
1756 }
1757
1758 if (build_id__is_defined(bid))
1759 dso__set_build_id(kernel, bid);
1760
1761 /*
1762 * Avoid using a zero address (kptr_restrict) for the ref reloc
1763 * symbol. Effectively having zero here means that at record
1764 * time /proc/sys/kernel/kptr_restrict was non zero.
1765 */
1766 if (xm->pgoff != 0) {
1767 map__set_kallsyms_ref_reloc_sym(machine->vmlinux_map,
1768 symbol_name,
1769 xm->pgoff);
1770 }
1771
1772 if (machine__is_default_guest(machine)) {
1773 /*
1774 * preload dso of guest kernel and modules
1775 */
1776 dso__load(kernel, machine__kernel_map(machine));
1777 }
1778 dso__put(kernel);
1779 } else if (perf_event__is_extra_kernel_mmap(machine, xm)) {
1780 return machine__process_extra_kernel_map(machine, xm);
1781 }
1782 return 0;
1783out_problem:
1784 return -1;
1785}
1786
1787int machine__process_mmap2_event(struct machine *machine,
1788 union perf_event *event,
1789 struct perf_sample *sample)
1790{
1791 struct thread *thread;
1792 struct map *map;
1793 struct dso_id dso_id = dso_id_empty;
1794 int ret = 0;
1795
1796 if (dump_trace)
1797 perf_event__fprintf_mmap2(event, stdout);
1798
1799 if (event->header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) {
1800 build_id__init(&dso_id.build_id, event->mmap2.build_id, event->mmap2.build_id_size);
1801 } else {
1802 dso_id.maj = event->mmap2.maj;
1803 dso_id.min = event->mmap2.min;
1804 dso_id.ino = event->mmap2.ino;
1805 dso_id.ino_generation = event->mmap2.ino_generation;
1806 dso_id.mmap2_valid = true;
1807 dso_id.mmap2_ino_generation_valid = true;
1808 }
1809
1810 if (sample->cpumode == PERF_RECORD_MISC_GUEST_KERNEL ||
1811 sample->cpumode == PERF_RECORD_MISC_KERNEL) {
1812 struct extra_kernel_map xm = {
1813 .start = event->mmap2.start,
1814 .end = event->mmap2.start + event->mmap2.len,
1815 .pgoff = event->mmap2.pgoff,
1816 };
1817
1818 strlcpy(xm.name, event->mmap2.filename, KMAP_NAME_LEN);
1819 ret = machine__process_kernel_mmap_event(machine, &xm, &dso_id.build_id);
1820 if (ret < 0)
1821 goto out_problem;
1822 return 0;
1823 }
1824
1825 thread = machine__findnew_thread(machine, event->mmap2.pid,
1826 event->mmap2.tid);
1827 if (thread == NULL)
1828 goto out_problem;
1829
1830 map = map__new(machine, event->mmap2.start,
1831 event->mmap2.len, event->mmap2.pgoff,
1832 &dso_id, event->mmap2.prot,
1833 event->mmap2.flags,
1834 event->mmap2.filename, thread);
1835
1836 if (map == NULL)
1837 goto out_problem_map;
1838
1839 ret = thread__insert_map(thread, map);
1840 if (ret)
1841 goto out_problem_insert;
1842
1843 thread__put(thread);
1844 map__put(map);
1845 return 0;
1846
1847out_problem_insert:
1848 map__put(map);
1849out_problem_map:
1850 thread__put(thread);
1851out_problem:
1852 dump_printf("problem processing PERF_RECORD_MMAP2, skipping event.\n");
1853 return 0;
1854}
1855
1856int machine__process_mmap_event(struct machine *machine, union perf_event *event,
1857 struct perf_sample *sample)
1858{
1859 struct thread *thread;
1860 struct map *map;
1861 u32 prot = 0;
1862 int ret = 0;
1863
1864 if (dump_trace)
1865 perf_event__fprintf_mmap(event, stdout);
1866
1867 if (sample->cpumode == PERF_RECORD_MISC_GUEST_KERNEL ||
1868 sample->cpumode == PERF_RECORD_MISC_KERNEL) {
1869 struct extra_kernel_map xm = {
1870 .start = event->mmap.start,
1871 .end = event->mmap.start + event->mmap.len,
1872 .pgoff = event->mmap.pgoff,
1873 };
1874
1875 strlcpy(xm.name, event->mmap.filename, KMAP_NAME_LEN);
1876 ret = machine__process_kernel_mmap_event(machine, &xm, NULL);
1877 if (ret < 0)
1878 goto out_problem;
1879 return 0;
1880 }
1881
1882 thread = machine__findnew_thread(machine, event->mmap.pid,
1883 event->mmap.tid);
1884 if (thread == NULL)
1885 goto out_problem;
1886
1887 if (!(event->header.misc & PERF_RECORD_MISC_MMAP_DATA))
1888 prot = PROT_EXEC;
1889
1890 map = map__new(machine, event->mmap.start,
1891 event->mmap.len, event->mmap.pgoff,
1892 &dso_id_empty, prot, /*flags=*/0, event->mmap.filename, thread);
1893
1894 if (map == NULL)
1895 goto out_problem_map;
1896
1897 ret = thread__insert_map(thread, map);
1898 if (ret)
1899 goto out_problem_insert;
1900
1901 thread__put(thread);
1902 map__put(map);
1903 return 0;
1904
1905out_problem_insert:
1906 map__put(map);
1907out_problem_map:
1908 thread__put(thread);
1909out_problem:
1910 dump_printf("problem processing PERF_RECORD_MMAP, skipping event.\n");
1911 return 0;
1912}
1913
1914void machine__remove_thread(struct machine *machine, struct thread *th)
1915{
1916 return threads__remove(&machine->threads, th);
1917}
1918
1919int machine__process_fork_event(struct machine *machine, union perf_event *event,
1920 struct perf_sample *sample)
1921{
1922 struct thread *thread = machine__find_thread(machine,
1923 event->fork.pid,
1924 event->fork.tid);
1925 struct thread *parent = machine__findnew_thread(machine,
1926 event->fork.ppid,
1927 event->fork.ptid);
1928 bool do_maps_clone = true;
1929 int err = 0;
1930
1931 if (dump_trace)
1932 perf_event__fprintf_task(event, stdout);
1933
1934 /*
1935 * There may be an existing thread that is not actually the parent,
1936 * either because we are processing events out of order, or because the
1937 * (fork) event that would have removed the thread was lost. Assume the
1938 * latter case and continue on as best we can.
1939 */
1940 if (parent != NULL &&
1941 thread__pid(parent) != (pid_t)event->fork.ppid) {
1942 dump_printf("removing erroneous parent thread %d/%d\n",
1943 thread__pid(parent), thread__tid(parent));
1944 machine__remove_thread(machine, parent);
1945 thread__put(parent);
1946 parent = machine__findnew_thread(machine, event->fork.ppid,
1947 event->fork.ptid);
1948 }
1949
1950 /* if a thread currently exists for the thread id remove it */
1951 if (thread != NULL) {
1952 machine__remove_thread(machine, thread);
1953 thread__put(thread);
1954 }
1955
1956 thread = machine__findnew_thread(machine, event->fork.pid,
1957 event->fork.tid);
1958 /*
1959 * When synthesizing FORK events, we are trying to create thread
1960 * objects for the already running tasks on the machine.
1961 *
1962 * Normally, for a kernel FORK event, we want to clone the parent's
1963 * maps because that is what the kernel just did.
1964 *
1965 * But when synthesizing, this should not be done. If we do, we end up
1966 * with overlapping maps as we process the synthesized MMAP2 events that
1967 * get delivered shortly thereafter.
1968 *
1969 * Use the FORK event misc flags in an internal way to signal this
1970 * situation, so we can elide the map clone when appropriate.
1971 */
1972 if (event->fork.header.misc & PERF_RECORD_MISC_FORK_EXEC)
1973 do_maps_clone = false;
1974
1975 if (thread == NULL || parent == NULL ||
1976 thread__fork(thread, parent, sample->time, do_maps_clone) < 0) {
1977 dump_printf("problem processing PERF_RECORD_FORK, skipping event.\n");
1978 err = -1;
1979 }
1980 thread__put(thread);
1981 thread__put(parent);
1982
1983 return err;
1984}
1985
1986int machine__process_exit_event(struct machine *machine, union perf_event *event,
1987 struct perf_sample *sample __maybe_unused)
1988{
1989 struct thread *thread = machine__find_thread(machine,
1990 event->fork.pid,
1991 event->fork.tid);
1992
1993 if (dump_trace)
1994 perf_event__fprintf_task(event, stdout);
1995
1996 /* There is no context switch out before exit, so we decrement here. */
1997 machine->parallelism--;
1998 if (thread != NULL) {
1999 if (symbol_conf.keep_exited_threads)
2000 thread__set_exited(thread, /*exited=*/true);
2001 else
2002 machine__remove_thread(machine, thread);
2003 }
2004 thread__put(thread);
2005 return 0;
2006}
2007
2008int machine__process_event(struct machine *machine, union perf_event *event,
2009 struct perf_sample *sample)
2010{
2011 int ret;
2012
2013 switch (event->header.type) {
2014 case PERF_RECORD_COMM:
2015 ret = machine__process_comm_event(machine, event, sample); break;
2016 case PERF_RECORD_MMAP:
2017 ret = machine__process_mmap_event(machine, event, sample); break;
2018 case PERF_RECORD_NAMESPACES:
2019 ret = machine__process_namespaces_event(machine, event, sample); break;
2020 case PERF_RECORD_CGROUP:
2021 ret = machine__process_cgroup_event(machine, event, sample); break;
2022 case PERF_RECORD_MMAP2:
2023 ret = machine__process_mmap2_event(machine, event, sample); break;
2024 case PERF_RECORD_FORK:
2025 ret = machine__process_fork_event(machine, event, sample); break;
2026 case PERF_RECORD_EXIT:
2027 ret = machine__process_exit_event(machine, event, sample); break;
2028 case PERF_RECORD_LOST:
2029 ret = machine__process_lost_event(machine, event, sample); break;
2030 case PERF_RECORD_AUX:
2031 ret = machine__process_aux_event(machine, event); break;
2032 case PERF_RECORD_ITRACE_START:
2033 ret = machine__process_itrace_start_event(machine, event); break;
2034 case PERF_RECORD_LOST_SAMPLES:
2035 ret = machine__process_lost_samples_event(machine, event, sample); break;
2036 case PERF_RECORD_SWITCH:
2037 case PERF_RECORD_SWITCH_CPU_WIDE:
2038 ret = machine__process_switch_event(machine, event); break;
2039 case PERF_RECORD_KSYMBOL:
2040 ret = machine__process_ksymbol(machine, event, sample); break;
2041 case PERF_RECORD_BPF_EVENT:
2042 ret = machine__process_bpf(machine, event, sample); break;
2043 case PERF_RECORD_TEXT_POKE:
2044 ret = machine__process_text_poke(machine, event, sample); break;
2045 case PERF_RECORD_AUX_OUTPUT_HW_ID:
2046 ret = machine__process_aux_output_hw_id_event(machine, event); break;
2047 default:
2048 ret = -1;
2049 break;
2050 }
2051
2052 return ret;
2053}
2054
2055static bool symbol__match_regex(struct symbol *sym, regex_t *regex)
2056{
2057 return regexec(regex, sym->name, 0, NULL, 0) == 0;
2058}
2059
2060static void ip__resolve_ams(struct thread *thread,
2061 struct addr_map_symbol *ams,
2062 u64 ip)
2063{
2064 struct addr_location al;
2065
2066 addr_location__init(&al);
2067 /*
2068 * We cannot use the header.misc hint to determine whether a
2069 * branch stack address is user, kernel, guest, hypervisor.
2070 * Branches may straddle the kernel/user/hypervisor boundaries.
2071 * Thus, we have to try consecutively until we find a match
2072 * or else, the symbol is unknown
2073 */
2074 thread__find_cpumode_addr_location(thread, ip, /*symbols=*/true, &al);
2075
2076 ams->addr = ip;
2077 ams->al_addr = al.addr;
2078 ams->al_level = al.level;
2079 ams->ms.thread = thread__get(al.thread);
2080 ams->ms.sym = al.sym;
2081 ams->ms.map = map__get(al.map);
2082 ams->phys_addr = 0;
2083 ams->data_page_size = 0;
2084 addr_location__exit(&al);
2085}
2086
2087static void ip__resolve_data(struct thread *thread,
2088 u8 m, struct addr_map_symbol *ams,
2089 u64 addr, u64 phys_addr, u64 daddr_page_size)
2090{
2091 struct addr_location al;
2092
2093 addr_location__init(&al);
2094
2095 thread__find_symbol(thread, m, addr, &al);
2096
2097 ams->addr = addr;
2098 ams->al_addr = al.addr;
2099 ams->al_level = al.level;
2100 ams->ms.thread = thread__get(al.thread);
2101 ams->ms.sym = al.sym;
2102 ams->ms.map = map__get(al.map);
2103 ams->phys_addr = phys_addr;
2104 ams->data_page_size = daddr_page_size;
2105 addr_location__exit(&al);
2106}
2107
2108struct mem_info *sample__resolve_mem(struct perf_sample *sample,
2109 struct addr_location *al)
2110{
2111 struct mem_info *mi = mem_info__new();
2112
2113 if (!mi)
2114 return NULL;
2115
2116 ip__resolve_ams(al->thread, mem_info__iaddr(mi), sample->ip);
2117 ip__resolve_data(al->thread, al->cpumode, mem_info__daddr(mi),
2118 sample->addr, sample->phys_addr,
2119 sample->data_page_size);
2120 mem_info__data_src(mi)->val = sample->data_src;
2121
2122 return mi;
2123}
2124
2125static char *callchain_srcline(struct map_symbol *ms, u64 ip)
2126{
2127 struct map *map = ms->map;
2128 char *srcline = NULL;
2129 struct dso *dso;
2130
2131 if (!map || callchain_param.key == CCKEY_FUNCTION)
2132 return srcline;
2133
2134 dso = map__dso(map);
2135 srcline = srcline__tree_find(dso__srclines(dso), ip);
2136 if (!srcline) {
2137 bool show_sym = false;
2138 bool show_addr = callchain_param.key == CCKEY_ADDRESS;
2139
2140 srcline = get_srcline(dso, map__rip_2objdump(map, ip),
2141 ms->sym, show_sym, show_addr, ip);
2142 srcline__tree_insert(dso__srclines(dso), ip, srcline);
2143 }
2144
2145 return srcline;
2146}
2147
2148struct iterations {
2149 int nr_loop_iter;
2150 u64 cycles;
2151};
2152
2153static int append_inlines(struct callchain_cursor *cursor, struct map_symbol *ms, u64 ip,
2154 bool branch, struct branch_flags *flags, int nr_loop_iter,
2155 u64 iter_cycles, u64 branch_from)
2156{
2157 struct symbol *sym = ms->sym;
2158 struct map *map = ms->map;
2159 struct inline_node *inline_node;
2160 struct inline_list *ilist;
2161 struct dso *dso;
2162 u64 addr;
2163 int ret = 1;
2164 struct map_symbol ilist_ms;
2165 bool first = true;
2166
2167 if (!symbol_conf.inline_name || !map || !sym)
2168 return ret;
2169
2170 addr = map__dso_map_ip(map, ip);
2171 addr = map__rip_2objdump(map, addr);
2172 dso = map__dso(map);
2173
2174 inline_node = inlines__tree_find(dso__inlined_nodes(dso), addr);
2175 if (!inline_node) {
2176 inline_node = dso__parse_addr_inlines(dso, addr, sym);
2177 if (!inline_node)
2178 return ret;
2179 inlines__tree_insert(dso__inlined_nodes(dso), inline_node);
2180 }
2181
2182 ilist_ms = (struct map_symbol) {
2183 .thread = thread__get(ms->thread),
2184 .map = map__get(map),
2185 };
2186 list_for_each_entry(ilist, &inline_node->val, list) {
2187 ilist_ms.sym = ilist->symbol;
2188 if (first) {
2189 ret = callchain_cursor_append(cursor, ip, &ilist_ms,
2190 branch, flags, nr_loop_iter,
2191 iter_cycles, branch_from, ilist->srcline);
2192 } else {
2193 ret = callchain_cursor_append(cursor, ip, &ilist_ms, false,
2194 NULL, 0, 0, 0, ilist->srcline);
2195 }
2196 first = false;
2197
2198 if (ret != 0)
2199 return ret;
2200 }
2201 map_symbol__exit(&ilist_ms);
2202
2203 return ret;
2204}
2205
2206static int add_callchain_ip(struct thread *thread,
2207 struct callchain_cursor *cursor,
2208 struct symbol **parent,
2209 struct addr_location *root_al,
2210 u8 *cpumode,
2211 u64 ip,
2212 bool branch,
2213 struct branch_flags *flags,
2214 struct iterations *iter,
2215 u64 branch_from,
2216 bool symbols)
2217{
2218 struct map_symbol ms = {};
2219 struct addr_location al;
2220 int nr_loop_iter = 0, err = 0;
2221 u64 iter_cycles = 0;
2222 const char *srcline = NULL;
2223
2224 addr_location__init(&al);
2225 al.filtered = 0;
2226 al.sym = NULL;
2227 al.srcline = NULL;
2228 if (!cpumode) {
2229 thread__find_cpumode_addr_location(thread, ip, symbols, &al);
2230 } else {
2231 if (ip >= PERF_CONTEXT_MAX) {
2232 switch (ip) {
2233 case PERF_CONTEXT_HV:
2234 *cpumode = PERF_RECORD_MISC_HYPERVISOR;
2235 break;
2236 case PERF_CONTEXT_KERNEL:
2237 *cpumode = PERF_RECORD_MISC_KERNEL;
2238 break;
2239 case PERF_CONTEXT_USER:
2240 case PERF_CONTEXT_USER_DEFERRED:
2241 *cpumode = PERF_RECORD_MISC_USER;
2242 break;
2243 default:
2244 pr_debug("invalid callchain context: "
2245 "%"PRId64"\n", (s64) ip);
2246 /*
2247 * It seems the callchain is corrupted.
2248 * Discard all.
2249 */
2250 callchain_cursor_reset(cursor);
2251 err = 1;
2252 goto out;
2253 }
2254 goto out;
2255 }
2256 if (symbols)
2257 thread__find_symbol(thread, *cpumode, ip, &al);
2258 else
2259 thread__find_map(thread, *cpumode, ip, &al);
2260 }
2261
2262 if (al.sym != NULL) {
2263 if (perf_hpp_list.parent && !*parent &&
2264 symbol__match_regex(al.sym, &parent_regex))
2265 *parent = al.sym;
2266 else if (have_ignore_callees && root_al &&
2267 symbol__match_regex(al.sym, &ignore_callees_regex)) {
2268 /* Treat this symbol as the root,
2269 forgetting its callees. */
2270 addr_location__copy(root_al, &al);
2271 callchain_cursor_reset(cursor);
2272 }
2273 }
2274
2275 if (symbol_conf.hide_unresolved && al.sym == NULL)
2276 goto out;
2277
2278 if (iter) {
2279 nr_loop_iter = iter->nr_loop_iter;
2280 iter_cycles = iter->cycles;
2281 }
2282
2283 ms.thread = thread__get(al.thread);
2284 ms.map = map__get(al.map);
2285 ms.sym = al.sym;
2286
2287 if (append_inlines(cursor, &ms, ip, branch, flags, nr_loop_iter,
2288 iter_cycles, branch_from) == 0)
2289 goto out;
2290
2291 srcline = callchain_srcline(&ms, al.addr);
2292 err = callchain_cursor_append(cursor, ip, &ms,
2293 branch, flags, nr_loop_iter,
2294 iter_cycles, branch_from, srcline);
2295out:
2296 addr_location__exit(&al);
2297 map_symbol__exit(&ms);
2298 return err;
2299}
2300
2301struct branch_info *sample__resolve_bstack(struct perf_sample *sample,
2302 struct addr_location *al)
2303{
2304 unsigned int i;
2305 const struct branch_stack *bs = sample->branch_stack;
2306 struct branch_entry *entries = perf_sample__branch_entries(sample);
2307 u64 *branch_stack_cntr = sample->branch_stack_cntr;
2308 struct branch_info *bi = calloc(bs->nr, sizeof(struct branch_info));
2309
2310 if (!bi)
2311 return NULL;
2312
2313 for (i = 0; i < bs->nr; i++) {
2314 ip__resolve_ams(al->thread, &bi[i].to, entries[i].to);
2315 ip__resolve_ams(al->thread, &bi[i].from, entries[i].from);
2316 bi[i].flags = entries[i].flags;
2317 if (branch_stack_cntr)
2318 bi[i].branch_stack_cntr = branch_stack_cntr[i];
2319 }
2320 return bi;
2321}
2322
2323static void save_iterations(struct iterations *iter,
2324 struct branch_entry *be, int nr)
2325{
2326 int i;
2327
2328 iter->nr_loop_iter++;
2329 iter->cycles = 0;
2330
2331 for (i = 0; i < nr; i++)
2332 iter->cycles += be[i].flags.cycles;
2333}
2334
2335#define CHASHSZ 127
2336#define CHASHBITS 7
2337#define NO_ENTRY 0xff
2338
2339#define PERF_MAX_BRANCH_DEPTH 127
2340
2341/* Remove loops. */
2342static int remove_loops(struct branch_entry *l, int nr,
2343 struct iterations *iter)
2344{
2345 int i, j, off;
2346 unsigned char chash[CHASHSZ];
2347
2348 memset(chash, NO_ENTRY, sizeof(chash));
2349
2350 BUG_ON(PERF_MAX_BRANCH_DEPTH > 255);
2351
2352 for (i = 0; i < nr; i++) {
2353 int h = hash_64(l[i].from, CHASHBITS) % CHASHSZ;
2354
2355 /* no collision handling for now */
2356 if (chash[h] == NO_ENTRY) {
2357 chash[h] = i;
2358 } else if (l[chash[h]].from == l[i].from) {
2359 bool is_loop = true;
2360 /* check if it is a real loop */
2361 off = 0;
2362 for (j = chash[h]; j < i && i + off < nr; j++, off++)
2363 if (l[j].from != l[i + off].from) {
2364 is_loop = false;
2365 break;
2366 }
2367 if (is_loop) {
2368 j = nr - (i + off);
2369 if (j > 0) {
2370 save_iterations(iter + i + off,
2371 l + i, off);
2372
2373 memmove(iter + i, iter + i + off,
2374 j * sizeof(*iter));
2375
2376 memmove(l + i, l + i + off,
2377 j * sizeof(*l));
2378 }
2379
2380 nr -= off;
2381 }
2382 }
2383 }
2384 return nr;
2385}
2386
2387static int lbr_callchain_add_kernel_ip(struct thread *thread,
2388 struct callchain_cursor *cursor,
2389 struct perf_sample *sample,
2390 struct symbol **parent,
2391 struct addr_location *root_al,
2392 u64 branch_from,
2393 bool callee, int end,
2394 bool symbols)
2395{
2396 struct ip_callchain *chain = sample->callchain;
2397 u8 cpumode = PERF_RECORD_MISC_USER;
2398 int err, i;
2399
2400 if (callee) {
2401 for (i = 0; i < end + 1; i++) {
2402 err = add_callchain_ip(thread, cursor, parent,
2403 root_al, &cpumode, chain->ips[i],
2404 false, NULL, NULL, branch_from,
2405 symbols);
2406 if (err)
2407 return err;
2408 }
2409 return 0;
2410 }
2411
2412 for (i = end; i >= 0; i--) {
2413 err = add_callchain_ip(thread, cursor, parent,
2414 root_al, &cpumode, chain->ips[i],
2415 false, NULL, NULL, branch_from,
2416 symbols);
2417 if (err)
2418 return err;
2419 }
2420
2421 return 0;
2422}
2423
2424static void save_lbr_cursor_node(struct thread *thread,
2425 struct callchain_cursor *cursor,
2426 int idx)
2427{
2428 struct lbr_stitch *lbr_stitch = thread__lbr_stitch(thread);
2429
2430 if (!lbr_stitch)
2431 return;
2432
2433 if (cursor->pos == cursor->nr) {
2434 lbr_stitch->prev_lbr_cursor[idx].valid = false;
2435 return;
2436 }
2437
2438 if (!cursor->curr)
2439 cursor->curr = cursor->first;
2440 else
2441 cursor->curr = cursor->curr->next;
2442
2443 map_symbol__exit(&lbr_stitch->prev_lbr_cursor[idx].ms);
2444 memcpy(&lbr_stitch->prev_lbr_cursor[idx], cursor->curr,
2445 sizeof(struct callchain_cursor_node));
2446 lbr_stitch->prev_lbr_cursor[idx].ms.thread = thread__get(cursor->curr->ms.thread);
2447 lbr_stitch->prev_lbr_cursor[idx].ms.map = map__get(cursor->curr->ms.map);
2448
2449 lbr_stitch->prev_lbr_cursor[idx].valid = true;
2450 cursor->pos++;
2451}
2452
2453static int lbr_callchain_add_lbr_ip(struct thread *thread,
2454 struct callchain_cursor *cursor,
2455 struct perf_sample *sample,
2456 struct symbol **parent,
2457 struct addr_location *root_al,
2458 u64 *branch_from,
2459 bool callee,
2460 bool symbols)
2461{
2462 struct branch_stack *lbr_stack = sample->branch_stack;
2463 struct branch_entry *entries = perf_sample__branch_entries(sample);
2464 u8 cpumode = PERF_RECORD_MISC_USER;
2465 int lbr_nr = lbr_stack->nr;
2466 struct branch_flags *flags;
2467 int err, i;
2468 u64 ip;
2469
2470 /*
2471 * The curr and pos are not used in writing session. They are cleared
2472 * in callchain_cursor_commit() when the writing session is closed.
2473 * Using curr and pos to track the current cursor node.
2474 */
2475 if (thread__lbr_stitch(thread)) {
2476 cursor->curr = NULL;
2477 cursor->pos = cursor->nr;
2478 if (cursor->nr) {
2479 cursor->curr = cursor->first;
2480 for (i = 0; i < (int)(cursor->nr - 1); i++)
2481 cursor->curr = cursor->curr->next;
2482 }
2483 }
2484
2485 if (callee) {
2486 /*
2487 * Set the (first) leaf function's IP to sample->ip (the
2488 * location of the sample) but if not recorded use entries.to
2489 */
2490 if (sample->ip)
2491 ip = sample->ip;
2492 else
2493 ip = entries[0].to;
2494 flags = &entries[0].flags;
2495 *branch_from = entries[0].from;
2496 err = add_callchain_ip(thread, cursor, parent,
2497 root_al, &cpumode, ip,
2498 true, flags, NULL,
2499 *branch_from, symbols);
2500 if (err)
2501 return err;
2502
2503 /*
2504 * The number of cursor node increases.
2505 * Move the current cursor node.
2506 * But does not need to save current cursor node for entry 0.
2507 * It's impossible to stitch the whole LBRs of previous sample.
2508 */
2509 if (thread__lbr_stitch(thread) && (cursor->pos != cursor->nr)) {
2510 if (!cursor->curr)
2511 cursor->curr = cursor->first;
2512 else
2513 cursor->curr = cursor->curr->next;
2514 cursor->pos++;
2515 }
2516
2517 /* Add LBR ip from entries.from one by one. */
2518 for (i = 0; i < lbr_nr; i++) {
2519 ip = entries[i].from;
2520 flags = &entries[i].flags;
2521 err = add_callchain_ip(thread, cursor, parent,
2522 root_al, &cpumode, ip,
2523 true, flags, NULL,
2524 *branch_from, symbols);
2525 if (err)
2526 return err;
2527 save_lbr_cursor_node(thread, cursor, i);
2528 }
2529 return 0;
2530 }
2531
2532 /* Add LBR ip from entries.from one by one. */
2533 for (i = lbr_nr - 1; i >= 0; i--) {
2534 ip = entries[i].from;
2535 flags = &entries[i].flags;
2536 err = add_callchain_ip(thread, cursor, parent,
2537 root_al, &cpumode, ip,
2538 true, flags, NULL,
2539 *branch_from, symbols);
2540 if (err)
2541 return err;
2542 save_lbr_cursor_node(thread, cursor, i);
2543 }
2544
2545 if (lbr_nr > 0) {
2546 /*
2547 * Set the (first) leaf function's IP to sample->ip (the
2548 * location of the sample) but if not recorded use entries.to
2549 */
2550 if (sample->ip)
2551 ip = sample->ip;
2552 else
2553 ip = entries[0].to;
2554 flags = &entries[0].flags;
2555 *branch_from = entries[0].from;
2556 err = add_callchain_ip(thread, cursor, parent,
2557 root_al, &cpumode, ip,
2558 true, flags, NULL,
2559 *branch_from, symbols);
2560 if (err)
2561 return err;
2562 }
2563
2564 return 0;
2565}
2566
2567static int lbr_callchain_add_stitched_lbr_ip(struct thread *thread,
2568 struct callchain_cursor *cursor)
2569{
2570 struct lbr_stitch *lbr_stitch = thread__lbr_stitch(thread);
2571 struct callchain_cursor_node *cnode;
2572 struct stitch_list *stitch_node;
2573 int err;
2574
2575 list_for_each_entry(stitch_node, &lbr_stitch->lists, node) {
2576 cnode = &stitch_node->cursor;
2577
2578 err = callchain_cursor_append(cursor, cnode->ip,
2579 &cnode->ms,
2580 cnode->branch,
2581 &cnode->branch_flags,
2582 cnode->nr_loop_iter,
2583 cnode->iter_cycles,
2584 cnode->branch_from,
2585 cnode->srcline);
2586 if (err)
2587 return err;
2588 }
2589 return 0;
2590}
2591
2592static struct stitch_list *get_stitch_node(struct thread *thread)
2593{
2594 struct lbr_stitch *lbr_stitch = thread__lbr_stitch(thread);
2595 struct stitch_list *stitch_node;
2596
2597 if (!list_empty(&lbr_stitch->free_lists)) {
2598 stitch_node = list_first_entry(&lbr_stitch->free_lists,
2599 struct stitch_list, node);
2600 list_del(&stitch_node->node);
2601
2602 return stitch_node;
2603 }
2604
2605 return malloc(sizeof(struct stitch_list));
2606}
2607
2608static bool has_stitched_lbr(struct thread *thread,
2609 struct perf_sample *cur,
2610 struct perf_sample *prev,
2611 unsigned int max_lbr,
2612 bool callee)
2613{
2614 struct branch_stack *cur_stack = cur->branch_stack;
2615 struct branch_entry *cur_entries = perf_sample__branch_entries(cur);
2616 struct branch_stack *prev_stack = prev->branch_stack;
2617 struct branch_entry *prev_entries = perf_sample__branch_entries(prev);
2618 struct lbr_stitch *lbr_stitch = thread__lbr_stitch(thread);
2619 int i, j, nr_identical_branches = 0;
2620 struct stitch_list *stitch_node;
2621 u64 cur_base, distance;
2622
2623 if (!cur_stack || !prev_stack)
2624 return false;
2625
2626 /* Find the physical index of the base-of-stack for current sample. */
2627 cur_base = max_lbr - cur_stack->nr + cur_stack->hw_idx + 1;
2628
2629 distance = (prev_stack->hw_idx > cur_base) ? (prev_stack->hw_idx - cur_base) :
2630 (max_lbr + prev_stack->hw_idx - cur_base);
2631 /* Previous sample has shorter stack. Nothing can be stitched. */
2632 if (distance + 1 > prev_stack->nr)
2633 return false;
2634
2635 /*
2636 * Check if there are identical LBRs between two samples.
2637 * Identical LBRs must have same from, to and flags values. Also,
2638 * they have to be saved in the same LBR registers (same physical
2639 * index).
2640 *
2641 * Starts from the base-of-stack of current sample.
2642 */
2643 for (i = distance, j = cur_stack->nr - 1; (i >= 0) && (j >= 0); i--, j--) {
2644 if ((prev_entries[i].from != cur_entries[j].from) ||
2645 (prev_entries[i].to != cur_entries[j].to) ||
2646 (prev_entries[i].flags.value != cur_entries[j].flags.value))
2647 break;
2648 nr_identical_branches++;
2649 }
2650
2651 if (!nr_identical_branches)
2652 return false;
2653
2654 /*
2655 * Save the LBRs between the base-of-stack of previous sample
2656 * and the base-of-stack of current sample into lbr_stitch->lists.
2657 * These LBRs will be stitched later.
2658 */
2659 for (i = prev_stack->nr - 1; i > (int)distance; i--) {
2660
2661 if (!lbr_stitch->prev_lbr_cursor[i].valid)
2662 continue;
2663
2664 stitch_node = get_stitch_node(thread);
2665 if (!stitch_node)
2666 return false;
2667
2668 memcpy(&stitch_node->cursor, &lbr_stitch->prev_lbr_cursor[i],
2669 sizeof(struct callchain_cursor_node));
2670
2671 stitch_node->cursor.ms.thread =
2672 thread__get(lbr_stitch->prev_lbr_cursor[i].ms.thread);
2673 stitch_node->cursor.ms.map = map__get(lbr_stitch->prev_lbr_cursor[i].ms.map);
2674
2675 if (callee)
2676 list_add(&stitch_node->node, &lbr_stitch->lists);
2677 else
2678 list_add_tail(&stitch_node->node, &lbr_stitch->lists);
2679 }
2680
2681 return true;
2682}
2683
2684static bool alloc_lbr_stitch(struct thread *thread, unsigned int max_lbr)
2685{
2686 if (thread__lbr_stitch(thread))
2687 return true;
2688
2689 thread__set_lbr_stitch(thread, zalloc(sizeof(struct lbr_stitch)));
2690 if (!thread__lbr_stitch(thread))
2691 goto err;
2692
2693 thread__lbr_stitch(thread)->prev_lbr_cursor =
2694 calloc(max_lbr + 1, sizeof(struct callchain_cursor_node));
2695 if (!thread__lbr_stitch(thread)->prev_lbr_cursor)
2696 goto free_lbr_stitch;
2697
2698 thread__lbr_stitch(thread)->prev_lbr_cursor_size = max_lbr + 1;
2699
2700 INIT_LIST_HEAD(&thread__lbr_stitch(thread)->lists);
2701 INIT_LIST_HEAD(&thread__lbr_stitch(thread)->free_lists);
2702
2703 return true;
2704
2705free_lbr_stitch:
2706 free(thread__lbr_stitch(thread));
2707 thread__set_lbr_stitch(thread, NULL);
2708err:
2709 pr_warning("Failed to allocate space for stitched LBRs. Disable LBR stitch\n");
2710 thread__set_lbr_stitch_enable(thread, false);
2711 return false;
2712}
2713
2714/*
2715 * Resolve LBR callstack chain sample
2716 * Return:
2717 * 1 on success get LBR callchain information
2718 * 0 no available LBR callchain information, should try fp
2719 * negative error code on other errors.
2720 */
2721static int resolve_lbr_callchain_sample(struct thread *thread,
2722 struct callchain_cursor *cursor,
2723 struct perf_sample *sample,
2724 struct symbol **parent,
2725 struct addr_location *root_al,
2726 int max_stack,
2727 unsigned int max_lbr,
2728 bool symbols)
2729{
2730 bool callee = (callchain_param.order == ORDER_CALLEE);
2731 struct ip_callchain *chain = sample->callchain;
2732 int chain_nr = min(max_stack, (int)chain->nr), i;
2733 struct lbr_stitch *lbr_stitch;
2734 bool stitched_lbr = false;
2735 u64 branch_from = 0;
2736 int err;
2737
2738 for (i = 0; i < chain_nr; i++) {
2739 if (chain->ips[i] == PERF_CONTEXT_USER)
2740 break;
2741 }
2742
2743 /* LBR only affects the user callchain */
2744 if (i == chain_nr)
2745 return 0;
2746
2747 if (thread__lbr_stitch_enable(thread) && !sample->no_hw_idx &&
2748 (max_lbr > 0) && alloc_lbr_stitch(thread, max_lbr)) {
2749 lbr_stitch = thread__lbr_stitch(thread);
2750
2751 stitched_lbr = has_stitched_lbr(thread, sample,
2752 &lbr_stitch->prev_sample,
2753 max_lbr, callee);
2754
2755 if (!stitched_lbr && !list_empty(&lbr_stitch->lists)) {
2756 struct stitch_list *stitch_node;
2757
2758 list_for_each_entry(stitch_node, &lbr_stitch->lists, node)
2759 map_symbol__exit(&stitch_node->cursor.ms);
2760
2761 list_splice_init(&lbr_stitch->lists, &lbr_stitch->free_lists);
2762 }
2763 memcpy(&lbr_stitch->prev_sample, sample, sizeof(*sample));
2764 }
2765
2766 if (callee) {
2767 /* Add kernel ip */
2768 err = lbr_callchain_add_kernel_ip(thread, cursor, sample,
2769 parent, root_al, branch_from,
2770 true, i, symbols);
2771 if (err)
2772 goto error;
2773
2774 err = lbr_callchain_add_lbr_ip(thread, cursor, sample, parent,
2775 root_al, &branch_from, true, symbols);
2776 if (err)
2777 goto error;
2778
2779 if (stitched_lbr) {
2780 err = lbr_callchain_add_stitched_lbr_ip(thread, cursor);
2781 if (err)
2782 goto error;
2783 }
2784
2785 } else {
2786 if (stitched_lbr) {
2787 err = lbr_callchain_add_stitched_lbr_ip(thread, cursor);
2788 if (err)
2789 goto error;
2790 }
2791 err = lbr_callchain_add_lbr_ip(thread, cursor, sample, parent,
2792 root_al, &branch_from, false, symbols);
2793 if (err)
2794 goto error;
2795
2796 /* Add kernel ip */
2797 err = lbr_callchain_add_kernel_ip(thread, cursor, sample,
2798 parent, root_al, branch_from,
2799 false, i, symbols);
2800 if (err)
2801 goto error;
2802 }
2803 return 1;
2804
2805error:
2806 return (err < 0) ? err : 0;
2807}
2808
2809static int find_prev_cpumode(struct ip_callchain *chain, struct thread *thread,
2810 struct callchain_cursor *cursor,
2811 struct symbol **parent,
2812 struct addr_location *root_al,
2813 u8 *cpumode, int ent, bool symbols)
2814{
2815 int err = 0;
2816
2817 while (--ent >= 0) {
2818 u64 ip = chain->ips[ent];
2819
2820 if (ip >= PERF_CONTEXT_MAX) {
2821 err = add_callchain_ip(thread, cursor, parent,
2822 root_al, cpumode, ip,
2823 false, NULL, NULL, 0, symbols);
2824 break;
2825 }
2826 }
2827 return err;
2828}
2829
2830static u64 get_leaf_frame_caller(struct perf_sample *sample,
2831 struct thread *thread, int usr_idx)
2832{
2833 if (thread__e_machine(thread, /*machine=*/NULL, /*e_flags=*/NULL) == EM_AARCH64)
2834 return get_leaf_frame_caller_aarch64(sample, thread, usr_idx);
2835 else
2836 return 0;
2837}
2838
2839static int thread__resolve_callchain_sample(struct thread *thread,
2840 struct callchain_cursor *cursor,
2841 struct perf_sample *sample,
2842 struct symbol **parent,
2843 struct addr_location *root_al,
2844 int max_stack,
2845 bool symbols)
2846{
2847 struct evsel *evsel = sample->evsel;
2848 struct branch_stack *branch = sample->branch_stack;
2849 struct branch_entry *entries = perf_sample__branch_entries(sample);
2850 struct ip_callchain *chain = sample->callchain;
2851 int chain_nr = 0;
2852 u8 cpumode = PERF_RECORD_MISC_USER;
2853 int i, j, err, nr_entries, usr_idx;
2854 int skip_idx = -1;
2855 int first_call = 0;
2856 u64 leaf_frame_caller;
2857
2858 if (chain)
2859 chain_nr = chain->nr;
2860
2861 if (evsel__has_branch_callstack(evsel)) {
2862 struct perf_env *env = evsel__env(evsel);
2863
2864 err = resolve_lbr_callchain_sample(thread, cursor, sample, parent,
2865 root_al, max_stack,
2866 !env ? 0 : env->max_branches,
2867 symbols);
2868 if (err)
2869 return (err < 0) ? err : 0;
2870 }
2871
2872 /*
2873 * Based on DWARF debug information, some architectures skip
2874 * a callchain entry saved by the kernel.
2875 */
2876 skip_idx = arch_skip_callchain_idx(thread, chain);
2877
2878 /*
2879 * Add branches to call stack for easier browsing. This gives
2880 * more context for a sample than just the callers.
2881 *
2882 * This uses individual histograms of paths compared to the
2883 * aggregated histograms the normal LBR mode uses.
2884 *
2885 * Limitations for now:
2886 * - No extra filters
2887 * - No annotations (should annotate somehow)
2888 */
2889
2890 if (branch && callchain_param.branch_callstack) {
2891 int nr = min(max_stack, (int)branch->nr);
2892 struct branch_entry be[nr];
2893 struct iterations iter[nr];
2894
2895 if (branch->nr > PERF_MAX_BRANCH_DEPTH) {
2896 pr_warning("corrupted branch chain. skipping...\n");
2897 goto check_calls;
2898 }
2899
2900 for (i = 0; i < nr; i++) {
2901 if (callchain_param.order == ORDER_CALLEE) {
2902 be[i] = entries[i];
2903
2904 if (chain == NULL)
2905 continue;
2906
2907 /*
2908 * Check for overlap into the callchain.
2909 * The return address is one off compared to
2910 * the branch entry. To adjust for this
2911 * assume the calling instruction is not longer
2912 * than 8 bytes.
2913 */
2914 if (i == skip_idx ||
2915 chain->ips[first_call] >= PERF_CONTEXT_MAX)
2916 first_call++;
2917 else if (be[i].from < chain->ips[first_call] &&
2918 be[i].from >= chain->ips[first_call] - 8)
2919 first_call++;
2920 } else
2921 be[i] = entries[branch->nr - i - 1];
2922 }
2923
2924 memset(iter, 0, sizeof(struct iterations) * nr);
2925 nr = remove_loops(be, nr, iter);
2926
2927 for (i = 0; i < nr; i++) {
2928 err = add_callchain_ip(thread, cursor, parent,
2929 root_al,
2930 NULL, be[i].to,
2931 true, &be[i].flags,
2932 NULL, be[i].from, symbols);
2933
2934 if (!err) {
2935 err = add_callchain_ip(thread, cursor, parent, root_al,
2936 NULL, be[i].from,
2937 true, &be[i].flags,
2938 &iter[i], 0, symbols);
2939 }
2940 if (err == -EINVAL)
2941 break;
2942 if (err)
2943 return err;
2944 }
2945
2946 if (chain_nr == 0)
2947 return 0;
2948
2949 chain_nr -= nr;
2950 }
2951
2952check_calls:
2953 if (chain && callchain_param.order != ORDER_CALLEE) {
2954 err = find_prev_cpumode(chain, thread, cursor, parent, root_al,
2955 &cpumode, chain->nr - first_call, symbols);
2956 if (err)
2957 return (err < 0) ? err : 0;
2958 }
2959 for (i = first_call, nr_entries = 0;
2960 i < chain_nr && nr_entries < max_stack; i++) {
2961 u64 ip;
2962
2963 if (callchain_param.order == ORDER_CALLEE)
2964 j = i;
2965 else
2966 j = chain->nr - i - 1;
2967
2968#ifdef HAVE_SKIP_CALLCHAIN_IDX
2969 if (j == skip_idx)
2970 continue;
2971#endif
2972 ip = chain->ips[j];
2973 if (ip < PERF_CONTEXT_MAX)
2974 ++nr_entries;
2975 else if (callchain_param.order != ORDER_CALLEE) {
2976 err = find_prev_cpumode(chain, thread, cursor, parent,
2977 root_al, &cpumode, j, symbols);
2978 if (err)
2979 return (err < 0) ? err : 0;
2980 continue;
2981 }
2982
2983 /*
2984 * PERF_CONTEXT_USER allows us to locate where the user stack ends.
2985 * Depending on callchain_param.order and the position of PERF_CONTEXT_USER,
2986 * the index will be different in order to add the missing frame
2987 * at the right place.
2988 */
2989
2990 usr_idx = callchain_param.order == ORDER_CALLEE ? j-2 : j-1;
2991
2992 if (usr_idx >= 0 && chain->ips[usr_idx] == PERF_CONTEXT_USER) {
2993
2994 leaf_frame_caller = get_leaf_frame_caller(sample, thread, usr_idx);
2995
2996 /*
2997 * check if leaf_frame_Caller != ip to not add the same
2998 * value twice.
2999 */
3000
3001 if (leaf_frame_caller && leaf_frame_caller != ip) {
3002
3003 err = add_callchain_ip(thread, cursor, parent,
3004 root_al, &cpumode, leaf_frame_caller,
3005 false, NULL, NULL, 0, symbols);
3006 if (err)
3007 return (err < 0) ? err : 0;
3008 }
3009 }
3010
3011 err = add_callchain_ip(thread, cursor, parent,
3012 root_al, &cpumode, ip,
3013 false, NULL, NULL, 0, symbols);
3014
3015 if (err)
3016 return (err < 0) ? err : 0;
3017 }
3018
3019 return 0;
3020}
3021
3022static int unwind_entry(struct unwind_entry *entry, void *arg)
3023{
3024 struct callchain_cursor *cursor = arg;
3025 const char *srcline = NULL;
3026 u64 addr = entry->ip;
3027
3028 if (symbol_conf.hide_unresolved && entry->ms.sym == NULL)
3029 return 0;
3030
3031 if (append_inlines(cursor, &entry->ms, entry->ip, /*branch=*/false, /*branch_flags=*/NULL,
3032 /*nr_loop_iter=*/0, /*iter_cycles=*/0, /*branch_from=*/0) == 0)
3033 return 0;
3034
3035 /*
3036 * Convert entry->ip from a virtual address to an offset in
3037 * its corresponding binary.
3038 */
3039 if (entry->ms.map)
3040 addr = map__dso_map_ip(entry->ms.map, entry->ip);
3041
3042 srcline = callchain_srcline(&entry->ms, addr);
3043 return callchain_cursor_append(cursor, entry->ip, &entry->ms,
3044 false, NULL, 0, 0, 0, srcline);
3045}
3046
3047static int thread__resolve_callchain_unwind(struct thread *thread,
3048 struct callchain_cursor *cursor,
3049 struct perf_sample *sample,
3050 int max_stack, bool symbols)
3051{
3052 struct evsel *evsel = sample->evsel;
3053
3054 /* Can we do dwarf post unwind? */
3055 if (!((evsel->core.attr.sample_type & PERF_SAMPLE_REGS_USER) &&
3056 (evsel->core.attr.sample_type & PERF_SAMPLE_STACK_USER)))
3057 return 0;
3058
3059 /* Bail out if nothing was captured. */
3060 if (!sample->user_regs || !sample->user_regs->regs ||
3061 !sample->user_stack.size)
3062 return 0;
3063
3064 if (!symbols)
3065 pr_debug("Not resolving symbols with an unwinder isn't currently supported\n");
3066
3067 return unwind__get_entries(unwind_entry, cursor,
3068 thread, sample, max_stack, false);
3069}
3070
3071int __thread__resolve_callchain(struct thread *thread,
3072 struct callchain_cursor *cursor,
3073 struct perf_sample *sample,
3074 struct symbol **parent,
3075 struct addr_location *root_al,
3076 int max_stack,
3077 bool symbols)
3078{
3079 int ret = 0;
3080
3081 if (cursor == NULL)
3082 return -ENOMEM;
3083
3084 callchain_cursor_reset(cursor);
3085
3086 if (callchain_param.order == ORDER_CALLEE) {
3087 ret = thread__resolve_callchain_sample(thread, cursor,
3088 sample,
3089 parent, root_al,
3090 max_stack, symbols);
3091 if (ret)
3092 return ret;
3093 ret = thread__resolve_callchain_unwind(thread, cursor,
3094 sample,
3095 max_stack, symbols);
3096 } else {
3097 ret = thread__resolve_callchain_unwind(thread, cursor,
3098 sample,
3099 max_stack, symbols);
3100 if (ret)
3101 return ret;
3102 ret = thread__resolve_callchain_sample(thread, cursor,
3103 sample,
3104 parent, root_al,
3105 max_stack, symbols);
3106 }
3107
3108 return ret;
3109}
3110
3111int machine__for_each_thread(struct machine *machine,
3112 int (*fn)(struct thread *thread, void *p),
3113 void *priv)
3114{
3115 return threads__for_each_thread(&machine->threads, fn, priv);
3116}
3117
3118int machines__for_each_thread(struct machines *machines,
3119 int (*fn)(struct thread *thread, void *p),
3120 void *priv)
3121{
3122 struct rb_node *nd;
3123 int rc = 0;
3124
3125 rc = machine__for_each_thread(&machines->host, fn, priv);
3126 if (rc != 0)
3127 return rc;
3128
3129 for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
3130 struct machine *machine = rb_entry(nd, struct machine, rb_node);
3131
3132 rc = machine__for_each_thread(machine, fn, priv);
3133 if (rc != 0)
3134 return rc;
3135 }
3136 return rc;
3137}
3138
3139
3140static int thread_list_cb(struct thread *thread, void *data)
3141{
3142 struct list_head *list = data;
3143 struct thread_list *entry = malloc(sizeof(*entry));
3144
3145 if (!entry)
3146 return -ENOMEM;
3147
3148 entry->thread = thread__get(thread);
3149 list_add_tail(&entry->list, list);
3150 return 0;
3151}
3152
3153int machine__thread_list(struct machine *machine, struct list_head *list)
3154{
3155 return machine__for_each_thread(machine, thread_list_cb, list);
3156}
3157
3158void thread_list__delete(struct list_head *list)
3159{
3160 struct thread_list *pos, *next;
3161
3162 list_for_each_entry_safe(pos, next, list, list) {
3163 thread__zput(pos->thread);
3164 list_del(&pos->list);
3165 free(pos);
3166 }
3167}
3168
3169pid_t machine__get_current_tid(struct machine *machine, int cpu)
3170{
3171 if (cpu < 0 || (size_t)cpu >= machine->current_tid_sz)
3172 return -1;
3173
3174 return machine->current_tid[cpu];
3175}
3176
3177int machine__set_current_tid(struct machine *machine, int cpu, pid_t pid,
3178 pid_t tid)
3179{
3180 struct thread *thread;
3181 const pid_t init_val = -1;
3182
3183 if (cpu < 0)
3184 return -EINVAL;
3185
3186 if (realloc_array_as_needed(machine->current_tid,
3187 machine->current_tid_sz,
3188 (unsigned int)cpu,
3189 &init_val))
3190 return -ENOMEM;
3191
3192 machine->current_tid[cpu] = tid;
3193
3194 thread = machine__findnew_thread(machine, pid, tid);
3195 if (!thread)
3196 return -ENOMEM;
3197
3198 thread__set_cpu(thread, cpu);
3199 thread__put(thread);
3200
3201 return 0;
3202}
3203
3204int machine__nr_cpus_avail(struct machine *machine)
3205{
3206 return machine ? perf_env__nr_cpus_avail(machine->env) : 0;
3207}
3208
3209int machine__get_kernel_start(struct machine *machine)
3210{
3211 struct map *map = machine__kernel_map(machine);
3212 int err = 0;
3213
3214 /*
3215 * The only addresses above 2^63 are kernel addresses of a 64-bit
3216 * kernel. Note that addresses are unsigned so that on a 32-bit system
3217 * all addresses including kernel addresses are less than 2^32. In
3218 * that case (32-bit system), if the kernel mapping is unknown, all
3219 * addresses will be assumed to be in user space - see
3220 * machine__kernel_ip().
3221 */
3222 machine->kernel_start = 1ULL << 63;
3223 if (map) {
3224 err = map__load(map);
3225 /*
3226 * On x86_64, PTI entry trampolines are less than the
3227 * start of kernel text, but still above 2^63. So leave
3228 * kernel_start = 1ULL << 63 for x86_64.
3229 */
3230 if (!err && !machine__is(machine, EM_X86_64))
3231 machine->kernel_start = map__start(map);
3232 }
3233 return err;
3234}
3235
3236u8 machine__addr_cpumode(struct machine *machine, u8 cpumode, u64 addr)
3237{
3238 u8 addr_cpumode = cpumode;
3239 bool kernel_ip;
3240
3241 if (!machine->single_address_space)
3242 goto out;
3243
3244 kernel_ip = machine__kernel_ip(machine, addr);
3245 switch (cpumode) {
3246 case PERF_RECORD_MISC_KERNEL:
3247 case PERF_RECORD_MISC_USER:
3248 addr_cpumode = kernel_ip ? PERF_RECORD_MISC_KERNEL :
3249 PERF_RECORD_MISC_USER;
3250 break;
3251 case PERF_RECORD_MISC_GUEST_KERNEL:
3252 case PERF_RECORD_MISC_GUEST_USER:
3253 addr_cpumode = kernel_ip ? PERF_RECORD_MISC_GUEST_KERNEL :
3254 PERF_RECORD_MISC_GUEST_USER;
3255 break;
3256 default:
3257 break;
3258 }
3259out:
3260 return addr_cpumode;
3261}
3262
3263struct dso *machine__findnew_dso_id(struct machine *machine, const char *filename,
3264 const struct dso_id *id)
3265{
3266 return dsos__findnew_id(&machine->dsos, filename, id);
3267}
3268
3269struct dso *machine__findnew_dso(struct machine *machine, const char *filename)
3270{
3271 return machine__findnew_dso_id(machine, filename, &dso_id_empty);
3272}
3273
3274char *machine__resolve_kernel_addr(void *vmachine, unsigned long long *addrp, char **modp)
3275{
3276 struct machine *machine = vmachine;
3277 struct map *map;
3278 struct symbol *sym = machine__find_kernel_symbol(machine, *addrp, &map);
3279
3280 if (sym == NULL)
3281 return NULL;
3282
3283 *modp = __map__is_kmodule(map) ? (char *)dso__short_name(map__dso(map)) : NULL;
3284 *addrp = map__unmap_ip(map, sym->start);
3285 return sym->name;
3286}
3287
3288struct machine__for_each_dso_cb_args {
3289 struct machine *machine;
3290 machine__dso_t fn;
3291 void *priv;
3292};
3293
3294static int machine__for_each_dso_cb(struct dso *dso, void *data)
3295{
3296 struct machine__for_each_dso_cb_args *args = data;
3297
3298 return args->fn(dso, args->machine, args->priv);
3299}
3300
3301int machine__for_each_dso(struct machine *machine, machine__dso_t fn, void *priv)
3302{
3303 struct machine__for_each_dso_cb_args args = {
3304 .machine = machine,
3305 .fn = fn,
3306 .priv = priv,
3307 };
3308
3309 return dsos__for_each_dso(&machine->dsos, machine__for_each_dso_cb, &args);
3310}
3311
3312int machine__for_each_kernel_map(struct machine *machine, machine__map_t fn, void *priv)
3313{
3314 struct maps *maps = machine__kernel_maps(machine);
3315
3316 return maps__for_each_map(maps, fn, priv);
3317}
3318
3319bool machine__is_lock_function(struct machine *machine, u64 addr)
3320{
3321 if (!machine->sched.text_start) {
3322 struct map *kmap;
3323 struct symbol *sym = machine__find_kernel_symbol_by_name(machine, "__sched_text_start", &kmap);
3324
3325 if (!sym) {
3326 /* to avoid retry */
3327 machine->sched.text_start = 1;
3328 return false;
3329 }
3330
3331 machine->sched.text_start = map__unmap_ip(kmap, sym->start);
3332
3333 /* should not fail from here */
3334 sym = machine__find_kernel_symbol_by_name(machine, "__sched_text_end", &kmap);
3335 machine->sched.text_end = map__unmap_ip(kmap, sym->start);
3336
3337 sym = machine__find_kernel_symbol_by_name(machine, "__lock_text_start", &kmap);
3338 machine->lock.text_start = map__unmap_ip(kmap, sym->start);
3339
3340 sym = machine__find_kernel_symbol_by_name(machine, "__lock_text_end", &kmap);
3341 machine->lock.text_end = map__unmap_ip(kmap, sym->start);
3342
3343 sym = machine__find_kernel_symbol_by_name(machine, "__traceiter_contention_begin", &kmap);
3344 if (sym) {
3345 machine->traceiter.text_start = map__unmap_ip(kmap, sym->start);
3346 machine->traceiter.text_end = map__unmap_ip(kmap, sym->end);
3347 }
3348 sym = machine__find_kernel_symbol_by_name(machine, "trace_contention_begin", &kmap);
3349 if (sym) {
3350 machine->trace.text_start = map__unmap_ip(kmap, sym->start);
3351 machine->trace.text_end = map__unmap_ip(kmap, sym->end);
3352 }
3353 }
3354
3355 /* failed to get kernel symbols */
3356 if (machine->sched.text_start == 1)
3357 return false;
3358
3359 /* mutex and rwsem functions are in sched text section */
3360 if (machine->sched.text_start <= addr && addr < machine->sched.text_end)
3361 return true;
3362
3363 /* spinlock functions are in lock text section */
3364 if (machine->lock.text_start <= addr && addr < machine->lock.text_end)
3365 return true;
3366
3367 /* traceiter functions currently don't have their own section
3368 * but we consider them lock functions
3369 */
3370 if (machine->traceiter.text_start != 0) {
3371 if (machine->traceiter.text_start <= addr && addr < machine->traceiter.text_end)
3372 return true;
3373 }
3374
3375 if (machine->trace.text_start != 0) {
3376 if (machine->trace.text_start <= addr && addr < machine->trace.text_end)
3377 return true;
3378 }
3379
3380 return false;
3381}
3382
3383int machine__hit_all_dsos(struct machine *machine)
3384{
3385 return dsos__hit_all(&machine->dsos);
3386}