projects/linux·blob master · refs

4440 lines · 118.8 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * builtin-record.c
4 *
5 * Builtin record command: Record the profile of a workload
6 * (or a CPU, or a PID) into the perf.data output file - for
7 * later analysis via perf report.
8 */
9#include "builtin.h"
10
11#include "util/build-id.h"
12#include <subcmd/parse-options.h>
13#include <internal/xyarray.h>
14#include "util/parse-events.h"
15#include "util/config.h"
16
17#include "util/arm64-frame-pointer-unwind-support.h"
18#include "util/callchain.h"
19#include "util/cgroup.h"
20#include "util/header.h"
21#include "util/event.h"
22#include "util/evlist.h"
23#include "util/evsel.h"
24#include "util/debug.h"
25#include "util/mmap.h"
26#include "util/mutex.h"
27#include "util/target.h"
28#include "util/session.h"
29#include "util/tool.h"
30#include "util/stat.h"
31#include "util/symbol.h"
32#include "util/record.h"
33#include "util/cpumap.h"
34#include "util/thread_map.h"
35#include "util/data.h"
36#include "util/perf_regs.h"
37#include "util/auxtrace.h"
38#include "util/tsc.h"
39#include "util/parse-branch-options.h"
40#include "util/parse-regs-options.h"
41#include "util/perf_api_probe.h"
42#include "util/trigger.h"
43#include "util/perf-hooks.h"
44#include "util/synthetic-events.h"
45#include "util/time-utils.h"
46#include "util/units.h"
47#include "util/bpf-event.h"
48#include "util/util.h"
49#include "util/pfm.h"
50#include "util/pmu.h"
51#include "util/pmus.h"
52#include "util/clockid.h"
53#include "util/off_cpu.h"
54#include "util/bpf-filter.h"
55#include "util/strbuf.h"
56#include "asm/bug.h"
57#include "perf.h"
58#include "cputopo.h"
59#include "dwarf-regs.h"
60
61#include <errno.h>
62#include <inttypes.h>
63#include <locale.h>
64#include <poll.h>
65#include <pthread.h>
66#include <unistd.h>
67#include <string.h>
68#ifndef HAVE_GETTID
69#include <syscall.h>
70#endif
71#include <sched.h>
72#include <signal.h>
73#ifdef HAVE_EVENTFD_SUPPORT
74#include <sys/eventfd.h>
75#endif
76#include <sys/mman.h>
77#include <sys/wait.h>
78#include <sys/types.h>
79#include <sys/stat.h>
80#include <fcntl.h>
81#include <linux/err.h>
82#include <linux/string.h>
83#include <linux/time64.h>
84#include <linux/zalloc.h>
85#include <linux/bitmap.h>
86#include <sys/time.h>
87
88struct switch_output {
89 bool enabled;
90 bool signal;
91 unsigned long size;
92 unsigned long time;
93 const char *str;
94 bool set;
95 char **filenames;
96 int num_files;
97 int cur_file;
98};
99
100struct thread_mask {
101 struct mmap_cpu_mask maps;
102 struct mmap_cpu_mask affinity;
103};
104
105struct record_thread {
106 pid_t tid;
107 struct thread_mask *mask;
108 struct {
109 int msg[2];
110 int ack[2];
111 } pipes;
112 struct fdarray pollfd;
113 int ctlfd_pos;
114 int nr_mmaps;
115 struct mmap **maps;
116 struct mmap **overwrite_maps;
117 struct record *rec;
118 unsigned long long samples;
119 unsigned long waking;
120 u64 bytes_written;
121 u64 bytes_transferred;
122 u64 bytes_compressed;
123};
124
125static __thread struct record_thread *thread;
126
127enum thread_msg {
128 THREAD_MSG__UNDEFINED = 0,
129 THREAD_MSG__READY,
130 THREAD_MSG__MAX,
131};
132
133static const char *thread_msg_tags[THREAD_MSG__MAX] = {
134 "UNDEFINED", "READY"
135};
136
137enum thread_spec {
138 THREAD_SPEC__UNDEFINED = 0,
139 THREAD_SPEC__CPU,
140 THREAD_SPEC__CORE,
141 THREAD_SPEC__PACKAGE,
142 THREAD_SPEC__NUMA,
143 THREAD_SPEC__USER,
144 THREAD_SPEC__MAX,
145};
146
147static const char *thread_spec_tags[THREAD_SPEC__MAX] = {
148 "undefined", "cpu", "core", "package", "numa", "user"
149};
150
151struct pollfd_index_map {
152 int evlist_pollfd_index;
153 int thread_pollfd_index;
154};
155
156struct record {
157 struct perf_tool tool;
158 struct record_opts opts;
159 u64 bytes_written;
160 u64 thread_bytes_written;
161 struct perf_data data;
162 struct auxtrace_record *itr;
163 struct evlist *evlist;
164 struct perf_session *session;
165 struct evlist *sb_evlist;
166 pthread_t thread_id;
167 int realtime_prio;
168 bool latency;
169 bool switch_output_event_set;
170 bool no_buildid;
171 bool no_buildid_set;
172 bool no_buildid_cache;
173 bool no_buildid_cache_set;
174 bool buildid_all;
175 bool buildid_mmap;
176 bool buildid_mmap_set;
177 bool timestamp_filename;
178 bool timestamp_boundary;
179 bool off_cpu;
180 const char *filter_action;
181 const char *uid_str;
182 struct switch_output switch_output;
183 unsigned long long samples;
184 unsigned long output_max_size; /* = 0: unlimited */
185 struct perf_debuginfod debuginfod;
186 int nr_threads;
187 struct thread_mask *thread_masks;
188 struct record_thread *thread_data;
189 struct pollfd_index_map *index_map;
190 size_t index_map_sz;
191 size_t index_map_cnt;
192};
193
194static volatile int done;
195
196static volatile int auxtrace_record__snapshot_started;
197static DEFINE_TRIGGER(auxtrace_snapshot_trigger);
198static DEFINE_TRIGGER(switch_output_trigger);
199
200static const char *affinity_tags[PERF_AFFINITY_MAX] = {
201 "SYS", "NODE", "CPU"
202};
203
204static int build_id__process_mmap(const struct perf_tool *tool, union perf_event *event,
205 struct perf_sample *sample, struct machine *machine);
206static int build_id__process_mmap2(const struct perf_tool *tool, union perf_event *event,
207 struct perf_sample *sample, struct machine *machine);
208static int process_timestamp_boundary(const struct perf_tool *tool,
209 union perf_event *event,
210 struct perf_sample *sample,
211 struct machine *machine);
212
213#ifndef HAVE_GETTID
214static inline pid_t gettid(void)
215{
216 return (pid_t)syscall(__NR_gettid);
217}
218#endif
219
220static int record__threads_enabled(struct record *rec)
221{
222 return rec->opts.threads_spec;
223}
224
225static bool switch_output_signal(struct record *rec)
226{
227 return rec->switch_output.signal &&
228 trigger_is_ready(&switch_output_trigger);
229}
230
231static bool switch_output_size(struct record *rec)
232{
233 return rec->switch_output.size &&
234 trigger_is_ready(&switch_output_trigger) &&
235 (rec->bytes_written >= rec->switch_output.size);
236}
237
238static bool switch_output_time(struct record *rec)
239{
240 return rec->switch_output.time &&
241 trigger_is_ready(&switch_output_trigger);
242}
243
244static u64 record__bytes_written(struct record *rec)
245{
246 return rec->bytes_written + rec->thread_bytes_written;
247}
248
249static bool record__output_max_size_exceeded(struct record *rec)
250{
251 return rec->output_max_size &&
252 (record__bytes_written(rec) >= rec->output_max_size);
253}
254
255static int record__write(struct record *rec, struct mmap *map __maybe_unused,
256 void *bf, size_t size)
257{
258 struct perf_data_file *file = &rec->session->data->file;
259
260 if (map && map->file)
261 file = map->file;
262
263 if (perf_data_file__write(file, bf, size) < 0) {
264 pr_err("failed to write perf data, error: %m\n");
265 return -1;
266 }
267
268 if (map && map->file) {
269 thread->bytes_written += size;
270 rec->thread_bytes_written += size;
271 } else {
272 rec->bytes_written += size;
273 }
274
275 if (record__output_max_size_exceeded(rec) && !done) {
276 fprintf(stderr, "[ perf record: perf size limit reached (%" PRIu64 " KB),"
277 " stopping session ]\n",
278 record__bytes_written(rec) >> 10);
279 done = 1;
280 }
281
282 if (switch_output_size(rec))
283 trigger_hit(&switch_output_trigger);
284
285 return 0;
286}
287
288static int record__aio_enabled(struct record *rec);
289static int record__comp_enabled(struct record *rec);
290static ssize_t zstd_compress(struct perf_session *session, struct mmap *map,
291 void *dst, size_t dst_size, void *src, size_t src_size);
292
293#ifdef HAVE_AIO_SUPPORT
294static int record__aio_write(struct aiocb *cblock, int trace_fd,
295 void *buf, size_t size, off_t off)
296{
297 int rc;
298
299 cblock->aio_fildes = trace_fd;
300 cblock->aio_buf = buf;
301 cblock->aio_nbytes = size;
302 cblock->aio_offset = off;
303 cblock->aio_sigevent.sigev_notify = SIGEV_NONE;
304
305 do {
306 rc = aio_write(cblock);
307 if (rc == 0) {
308 break;
309 } else if (errno != EAGAIN) {
310 cblock->aio_fildes = -1;
311 pr_err("failed to queue perf data, error: %m\n");
312 break;
313 }
314 } while (1);
315
316 return rc;
317}
318
319static int record__aio_complete(struct mmap *md, struct aiocb *cblock)
320{
321 void *rem_buf;
322 off_t rem_off;
323 size_t rem_size;
324 int rc, aio_errno;
325 ssize_t aio_ret, written;
326
327 aio_errno = aio_error(cblock);
328 if (aio_errno == EINPROGRESS)
329 return 0;
330
331 written = aio_ret = aio_return(cblock);
332 if (aio_ret < 0) {
333 if (aio_errno != EINTR)
334 pr_err("failed to write perf data, error: %m\n");
335 written = 0;
336 }
337
338 rem_size = cblock->aio_nbytes - written;
339
340 if (rem_size == 0) {
341 cblock->aio_fildes = -1;
342 /*
343 * md->refcount is incremented in record__aio_pushfn() for
344 * every aio write request started in record__aio_push() so
345 * decrement it because the request is now complete.
346 */
347 perf_mmap__put(&md->core);
348 rc = 1;
349 } else {
350 /*
351 * aio write request may require restart with the
352 * remainder if the kernel didn't write whole
353 * chunk at once.
354 */
355 rem_off = cblock->aio_offset + written;
356 rem_buf = (void *)(cblock->aio_buf + written);
357 record__aio_write(cblock, cblock->aio_fildes,
358 rem_buf, rem_size, rem_off);
359 rc = 0;
360 }
361
362 return rc;
363}
364
365static int record__aio_sync(struct mmap *md, bool sync_all)
366{
367 struct aiocb **aiocb = md->aio.aiocb;
368 struct aiocb *cblocks = md->aio.cblocks;
369 struct timespec timeout = { 0, 1000 * 1000 * 1 }; /* 1ms */
370 int i, do_suspend;
371
372 do {
373 do_suspend = 0;
374 for (i = 0; i < md->aio.nr_cblocks; ++i) {
375 if (cblocks[i].aio_fildes == -1 || record__aio_complete(md, &cblocks[i])) {
376 if (sync_all)
377 aiocb[i] = NULL;
378 else
379 return i;
380 } else {
381 /*
382 * Started aio write is not complete yet
383 * so it has to be waited before the
384 * next allocation.
385 */
386 aiocb[i] = &cblocks[i];
387 do_suspend = 1;
388 }
389 }
390 if (!do_suspend)
391 return -1;
392
393 while (aio_suspend((const struct aiocb **)aiocb, md->aio.nr_cblocks, &timeout)) {
394 if (!(errno == EAGAIN || errno == EINTR))
395 pr_err("failed to sync perf data, error: %m\n");
396 }
397 } while (1);
398}
399
400struct record_aio {
401 struct record *rec;
402 void *data;
403 size_t size;
404};
405
406static int record__aio_pushfn(struct mmap *map, void *to, void *buf, size_t size)
407{
408 struct record_aio *aio = to;
409
410 /*
411 * map->core.base data pointed by buf is copied into free map->aio.data[] buffer
412 * to release space in the kernel buffer as fast as possible, calling
413 * perf_mmap__consume() from perf_mmap__push() function.
414 *
415 * That lets the kernel to proceed with storing more profiling data into
416 * the kernel buffer earlier than other per-cpu kernel buffers are handled.
417 *
418 * Coping can be done in two steps in case the chunk of profiling data
419 * crosses the upper bound of the kernel buffer. In this case we first move
420 * part of data from map->start till the upper bound and then the remainder
421 * from the beginning of the kernel buffer till the end of the data chunk.
422 */
423
424 if (record__comp_enabled(aio->rec)) {
425 ssize_t compressed = zstd_compress(aio->rec->session, NULL, aio->data + aio->size,
426 mmap__mmap_len(map) - aio->size,
427 buf, size);
428 if (compressed < 0)
429 return (int)compressed;
430
431 size = compressed;
432 } else {
433 memcpy(aio->data + aio->size, buf, size);
434 }
435
436 if (!aio->size) {
437 /*
438 * Increment map->refcount to guard map->aio.data[] buffer
439 * from premature deallocation because map object can be
440 * released earlier than aio write request started on
441 * map->aio.data[] buffer is complete.
442 *
443 * perf_mmap__put() is done at record__aio_complete()
444 * after started aio request completion or at record__aio_push()
445 * if the request failed to start.
446 */
447 perf_mmap__get(&map->core);
448 }
449
450 aio->size += size;
451
452 return size;
453}
454
455static int record__aio_push(struct record *rec, struct mmap *map, off_t *off)
456{
457 int ret, idx;
458 int trace_fd = perf_data__fd(rec->session->data);
459 struct record_aio aio = { .rec = rec, .size = 0 };
460
461 /*
462 * Call record__aio_sync() to wait till map->aio.data[] buffer
463 * becomes available after previous aio write operation.
464 */
465
466 idx = record__aio_sync(map, false);
467 aio.data = map->aio.data[idx];
468 ret = perf_mmap__push(map, &aio, record__aio_pushfn);
469 if (ret != 0) /* ret > 0 - no data, ret < 0 - error */
470 return ret;
471
472 rec->samples++;
473 ret = record__aio_write(&(map->aio.cblocks[idx]), trace_fd, aio.data, aio.size, *off);
474 if (!ret) {
475 *off += aio.size;
476 rec->bytes_written += aio.size;
477 if (switch_output_size(rec))
478 trigger_hit(&switch_output_trigger);
479 } else {
480 /*
481 * Decrement map->refcount incremented in record__aio_pushfn()
482 * back if record__aio_write() operation failed to start, otherwise
483 * map->refcount is decremented in record__aio_complete() after
484 * aio write operation finishes successfully.
485 */
486 perf_mmap__put(&map->core);
487 }
488
489 return ret;
490}
491
492static off_t record__aio_get_pos(int trace_fd)
493{
494 return lseek(trace_fd, 0, SEEK_CUR);
495}
496
497static void record__aio_set_pos(int trace_fd, off_t pos)
498{
499 lseek(trace_fd, pos, SEEK_SET);
500}
501
502static void record__aio_mmap_read_sync(struct record *rec)
503{
504 int i;
505 struct evlist *evlist = rec->evlist;
506 struct mmap *maps = evlist__mmap(evlist);
507
508 if (!record__aio_enabled(rec))
509 return;
510
511 for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) {
512 struct mmap *map = &maps[i];
513
514 if (map->core.base)
515 record__aio_sync(map, true);
516 }
517}
518
519static int nr_cblocks_default = 1;
520static int nr_cblocks_max = 4;
521
522static int record__aio_parse(const struct option *opt,
523 const char *str,
524 int unset)
525{
526 struct record_opts *opts = (struct record_opts *)opt->value;
527
528 if (unset) {
529 opts->nr_cblocks = 0;
530 } else {
531 if (str)
532 opts->nr_cblocks = strtol(str, NULL, 0);
533 if (!opts->nr_cblocks)
534 opts->nr_cblocks = nr_cblocks_default;
535 }
536
537 return 0;
538}
539#else /* HAVE_AIO_SUPPORT */
540static int nr_cblocks_max = 0;
541
542static int record__aio_push(struct record *rec __maybe_unused, struct mmap *map __maybe_unused,
543 off_t *off __maybe_unused)
544{
545 return -1;
546}
547
548static off_t record__aio_get_pos(int trace_fd __maybe_unused)
549{
550 return -1;
551}
552
553static void record__aio_set_pos(int trace_fd __maybe_unused, off_t pos __maybe_unused)
554{
555}
556
557static void record__aio_mmap_read_sync(struct record *rec __maybe_unused)
558{
559}
560#endif
561
562static int record__aio_enabled(struct record *rec)
563{
564 return rec->opts.nr_cblocks > 0;
565}
566
567#define MMAP_FLUSH_DEFAULT 1
568static int record__mmap_flush_parse(const struct option *opt,
569 const char *str,
570 int unset)
571{
572 int flush_max;
573 struct record_opts *opts = (struct record_opts *)opt->value;
574 static struct parse_tag tags[] = {
575 { .tag = 'B', .mult = 1 },
576 { .tag = 'K', .mult = 1 << 10 },
577 { .tag = 'M', .mult = 1 << 20 },
578 { .tag = 'G', .mult = 1 << 30 },
579 { .tag = 0 },
580 };
581
582 if (unset)
583 return 0;
584
585 if (str) {
586 opts->mmap_flush = parse_tag_value(str, tags);
587 if (opts->mmap_flush == (int)-1)
588 opts->mmap_flush = strtol(str, NULL, 0);
589 }
590
591 if (!opts->mmap_flush)
592 opts->mmap_flush = MMAP_FLUSH_DEFAULT;
593
594 flush_max = evlist__mmap_size(opts->mmap_pages);
595 flush_max /= 4;
596 if (opts->mmap_flush > flush_max)
597 opts->mmap_flush = flush_max;
598
599 return 0;
600}
601
602#ifdef HAVE_ZSTD_SUPPORT
603static unsigned int comp_level_default = 1;
604
605static int record__parse_comp_level(const struct option *opt, const char *str, int unset)
606{
607 struct record_opts *opts = opt->value;
608
609 if (unset) {
610 opts->comp_level = 0;
611 } else {
612 if (str)
613 opts->comp_level = strtol(str, NULL, 0);
614 if (!opts->comp_level)
615 opts->comp_level = comp_level_default;
616 }
617
618 return 0;
619}
620#endif
621static unsigned int comp_level_max = 22;
622
623static int record__comp_enabled(struct record *rec)
624{
625 return rec->opts.comp_level > 0;
626}
627
628static int process_synthesized_event(const struct perf_tool *tool,
629 union perf_event *event,
630 struct perf_sample *sample __maybe_unused,
631 struct machine *machine __maybe_unused)
632{
633 struct record *rec = container_of(tool, struct record, tool);
634 return record__write(rec, NULL, event, event->header.size);
635}
636
637static struct mutex synth_lock;
638
639static int process_locked_synthesized_event(const struct perf_tool *tool,
640 union perf_event *event,
641 struct perf_sample *sample __maybe_unused,
642 struct machine *machine __maybe_unused)
643{
644 int ret;
645
646 mutex_lock(&synth_lock);
647 ret = process_synthesized_event(tool, event, sample, machine);
648 mutex_unlock(&synth_lock);
649 return ret;
650}
651
652static int record__pushfn(struct mmap *map, void *to, void *bf, size_t size)
653{
654 struct record *rec = to;
655
656 if (record__comp_enabled(rec)) {
657 ssize_t compressed = zstd_compress(rec->session, map, map->data,
658 mmap__mmap_len(map), bf, size);
659
660 if (compressed < 0)
661 return (int)compressed;
662
663 thread->samples++;
664 return record__write(rec, map, map->data, compressed);
665 }
666
667 thread->samples++;
668 return record__write(rec, map, bf, size);
669}
670
671static volatile sig_atomic_t signr = -1;
672static volatile sig_atomic_t child_finished;
673#ifdef HAVE_EVENTFD_SUPPORT
674static volatile sig_atomic_t done_fd = -1;
675#endif
676
677static void sig_handler(int sig)
678{
679 if (sig == SIGCHLD)
680 child_finished = 1;
681 else
682 signr = sig;
683
684 done = 1;
685#ifdef HAVE_EVENTFD_SUPPORT
686 if (done_fd >= 0) {
687 u64 tmp = 1;
688 int orig_errno = errno;
689
690 /*
691 * It is possible for this signal handler to run after done is
692 * checked in the main loop, but before the perf counter fds are
693 * polled. If this happens, the poll() will continue to wait
694 * even though done is set, and will only break out if either
695 * another signal is received, or the counters are ready for
696 * read. To ensure the poll() doesn't sleep when done is set,
697 * use an eventfd (done_fd) to wake up the poll().
698 */
699 if (write(done_fd, &tmp, sizeof(tmp)) < 0)
700 pr_err("failed to signal wakeup fd, error: %m\n");
701
702 errno = orig_errno;
703 }
704#endif // HAVE_EVENTFD_SUPPORT
705}
706
707static void sigsegv_handler(int sig)
708{
709 perf_hooks__recover();
710 sighandler_dump_stack(sig);
711}
712
713static void record__sig_exit(void)
714{
715 if (signr == -1)
716 return;
717
718 signal(signr, SIG_DFL);
719 raise(signr);
720}
721
722static int record__process_auxtrace(const struct perf_tool *tool,
723 struct mmap *map,
724 union perf_event *event, void *data1,
725 size_t len1, void *data2, size_t len2)
726{
727 struct record *rec = container_of(tool, struct record, tool);
728 struct perf_data *data = &rec->data;
729 size_t padding;
730 u8 pad[8] = {0};
731
732 if (!perf_data__is_pipe(data) && perf_data__is_single_file(data)) {
733 off_t file_offset;
734 int fd = perf_data__fd(data);
735 int err;
736
737 file_offset = lseek(fd, 0, SEEK_CUR);
738 if (file_offset == -1)
739 return -1;
740 err = auxtrace_index__auxtrace_event(&rec->session->auxtrace_index,
741 event, file_offset);
742 if (err)
743 return err;
744 }
745
746 /* event.auxtrace.size includes padding, see __auxtrace_mmap__read() */
747 padding = (len1 + len2) & 7;
748 if (padding)
749 padding = 8 - padding;
750
751 record__write(rec, map, event, event->header.size);
752 record__write(rec, map, data1, len1);
753 if (len2)
754 record__write(rec, map, data2, len2);
755 record__write(rec, map, &pad, padding);
756
757 return 0;
758}
759
760static int record__auxtrace_mmap_read(struct record *rec,
761 struct mmap *map)
762{
763 int ret;
764
765 ret = auxtrace_mmap__read(map, rec->itr,
766 perf_session__env(rec->session),
767 &rec->tool,
768 record__process_auxtrace);
769 if (ret < 0)
770 return ret;
771
772 if (ret)
773 rec->samples++;
774
775 return 0;
776}
777
778static int record__auxtrace_mmap_read_snapshot(struct record *rec,
779 struct mmap *map)
780{
781 int ret;
782
783 ret = auxtrace_mmap__read_snapshot(map, rec->itr,
784 perf_session__env(rec->session),
785 &rec->tool,
786 record__process_auxtrace,
787 rec->opts.auxtrace_snapshot_size);
788 if (ret < 0)
789 return ret;
790
791 if (ret)
792 rec->samples++;
793
794 return 0;
795}
796
797static int record__auxtrace_read_snapshot_all(struct record *rec)
798{
799 int i;
800 int rc = 0;
801
802 for (i = 0; i < evlist__core(rec->evlist)->nr_mmaps; i++) {
803 struct mmap *map = &evlist__mmap(rec->evlist)[i];
804
805 if (!map->auxtrace_mmap.base)
806 continue;
807
808 if (record__auxtrace_mmap_read_snapshot(rec, map) != 0) {
809 rc = -1;
810 goto out;
811 }
812 }
813out:
814 return rc;
815}
816
817static void record__read_auxtrace_snapshot(struct record *rec, bool on_exit)
818{
819 pr_debug("Recording AUX area tracing snapshot\n");
820 if (record__auxtrace_read_snapshot_all(rec) < 0) {
821 trigger_error(&auxtrace_snapshot_trigger);
822 } else {
823 if (auxtrace_record__snapshot_finish(rec->itr, on_exit))
824 trigger_error(&auxtrace_snapshot_trigger);
825 else
826 trigger_ready(&auxtrace_snapshot_trigger);
827 }
828}
829
830static int record__auxtrace_snapshot_exit(struct record *rec)
831{
832 if (trigger_is_error(&auxtrace_snapshot_trigger))
833 return 0;
834
835 if (!auxtrace_record__snapshot_started &&
836 auxtrace_record__snapshot_start(rec->itr))
837 return -1;
838
839 record__read_auxtrace_snapshot(rec, true);
840 if (trigger_is_error(&auxtrace_snapshot_trigger))
841 return -1;
842
843 return 0;
844}
845
846static int record__auxtrace_init(struct record *rec)
847{
848 int err;
849
850 if ((rec->opts.auxtrace_snapshot_opts || rec->opts.auxtrace_sample_opts)
851 && record__threads_enabled(rec)) {
852 pr_err("AUX area tracing options are not available in parallel streaming mode.\n");
853 return -EINVAL;
854 }
855
856 if (!rec->itr) {
857 err = -EINVAL;
858 rec->itr = auxtrace_record__init(rec->evlist, &err);
859 if (err)
860 return err;
861 }
862
863 err = auxtrace_parse_snapshot_options(rec->itr, &rec->opts,
864 rec->opts.auxtrace_snapshot_opts);
865 if (err)
866 return err;
867
868 err = auxtrace_parse_sample_options(rec->itr, rec->evlist, &rec->opts,
869 rec->opts.auxtrace_sample_opts);
870 if (err)
871 return err;
872
873 err = auxtrace_parse_aux_action(rec->evlist);
874 if (err)
875 return err;
876
877 return auxtrace_parse_filters(rec->evlist);
878}
879
880static int record__config_text_poke(struct evlist *evlist)
881{
882 struct evsel *evsel;
883
884 /* Nothing to do if text poke is already configured */
885 evlist__for_each_entry(evlist, evsel) {
886 if (evsel->core.attr.text_poke)
887 return 0;
888 }
889
890 evsel = evlist__add_dummy_on_all_cpus(evlist);
891 if (!evsel)
892 return -ENOMEM;
893
894 evsel->core.attr.text_poke = 1;
895 evsel->core.attr.ksymbol = 1;
896 evsel->immediate = true;
897 evsel__set_sample_bit(evsel, TIME);
898
899 return 0;
900}
901
902static int record__config_off_cpu(struct record *rec)
903{
904 return off_cpu_prepare(rec->evlist, &rec->opts.target, &rec->opts);
905}
906
907static bool record__tracking_system_wide(struct record *rec)
908{
909 struct evlist *evlist = rec->evlist;
910 struct evsel *evsel;
911
912 /*
913 * If non-dummy evsel exists, system_wide sideband is need to
914 * help parse sample information.
915 * For example, PERF_EVENT_MMAP event to help parse symbol,
916 * and PERF_EVENT_COMM event to help parse task executable name.
917 */
918 evlist__for_each_entry(evlist, evsel) {
919 if (!evsel__is_dummy_event(evsel))
920 return true;
921 }
922
923 return false;
924}
925
926static int record__config_tracking_events(struct record *rec)
927{
928 struct record_opts *opts = &rec->opts;
929 struct evlist *evlist = rec->evlist;
930 bool system_wide = false;
931 struct evsel *evsel;
932
933 /*
934 * For initial_delay, system wide or a hybrid system, we need to add
935 * tracking event so that we can track PERF_RECORD_MMAP to cover the
936 * delay of waiting or event synthesis.
937 */
938 if (opts->target.initial_delay || target__has_cpu(&opts->target) ||
939 perf_pmus__num_core_pmus() > 1) {
940 /*
941 * User space tasks can migrate between CPUs, so when tracing
942 * selected CPUs, sideband for all CPUs is still needed.
943 */
944 if (!!opts->target.cpu_list && record__tracking_system_wide(rec))
945 system_wide = true;
946
947 evsel = evlist__findnew_tracking_event(evlist, system_wide);
948 if (!evsel)
949 return -ENOMEM;
950
951 /*
952 * Enable the tracking event when the process is forked for
953 * initial_delay, immediately for system wide.
954 */
955 if (opts->target.initial_delay && !evsel->immediate &&
956 !target__has_cpu(&opts->target))
957 evsel->core.attr.enable_on_exec = 1;
958 else
959 evsel->immediate = 1;
960 }
961
962 return 0;
963}
964
965static bool record__kcore_readable(struct machine *machine)
966{
967 char kcore[PATH_MAX];
968 int fd;
969
970 scnprintf(kcore, sizeof(kcore), "%s/proc/kcore", machine->root_dir);
971
972 fd = open(kcore, O_RDONLY);
973 if (fd < 0)
974 return false;
975
976 close(fd);
977
978 return true;
979}
980
981static int record__kcore_copy(struct machine *machine, struct perf_data *data)
982{
983 char from_dir[PATH_MAX];
984 char kcore_dir[PATH_MAX];
985 int ret;
986
987 snprintf(from_dir, sizeof(from_dir), "%s/proc", machine->root_dir);
988
989 ret = perf_data__make_kcore_dir(data, kcore_dir, sizeof(kcore_dir));
990 if (ret)
991 return ret;
992
993 return kcore_copy(from_dir, kcore_dir);
994}
995
996static void record__thread_data_init_pipes(struct record_thread *thread_data)
997{
998 thread_data->pipes.msg[0] = -1;
999 thread_data->pipes.msg[1] = -1;
1000 thread_data->pipes.ack[0] = -1;
1001 thread_data->pipes.ack[1] = -1;
1002}
1003
1004static int record__thread_data_open_pipes(struct record_thread *thread_data)
1005{
1006 if (pipe(thread_data->pipes.msg))
1007 return -EINVAL;
1008
1009 if (pipe(thread_data->pipes.ack)) {
1010 close(thread_data->pipes.msg[0]);
1011 thread_data->pipes.msg[0] = -1;
1012 close(thread_data->pipes.msg[1]);
1013 thread_data->pipes.msg[1] = -1;
1014 return -EINVAL;
1015 }
1016
1017 pr_debug2("thread_data[%p]: msg=[%d,%d], ack=[%d,%d]\n", thread_data,
1018 thread_data->pipes.msg[0], thread_data->pipes.msg[1],
1019 thread_data->pipes.ack[0], thread_data->pipes.ack[1]);
1020
1021 return 0;
1022}
1023
1024static void record__thread_data_close_pipes(struct record_thread *thread_data)
1025{
1026 if (thread_data->pipes.msg[0] != -1) {
1027 close(thread_data->pipes.msg[0]);
1028 thread_data->pipes.msg[0] = -1;
1029 }
1030 if (thread_data->pipes.msg[1] != -1) {
1031 close(thread_data->pipes.msg[1]);
1032 thread_data->pipes.msg[1] = -1;
1033 }
1034 if (thread_data->pipes.ack[0] != -1) {
1035 close(thread_data->pipes.ack[0]);
1036 thread_data->pipes.ack[0] = -1;
1037 }
1038 if (thread_data->pipes.ack[1] != -1) {
1039 close(thread_data->pipes.ack[1]);
1040 thread_data->pipes.ack[1] = -1;
1041 }
1042}
1043
1044static bool evlist__per_thread(struct evlist *evlist)
1045{
1046 return cpu_map__is_dummy(evlist__core(evlist)->user_requested_cpus);
1047}
1048
1049static int record__thread_data_init_maps(struct record_thread *thread_data, struct evlist *evlist)
1050{
1051 int m, tm, nr_mmaps = evlist__core(evlist)->nr_mmaps;
1052 struct mmap *mmap = evlist__mmap(evlist);
1053 struct mmap *overwrite_mmap = evlist__overwrite_mmap(evlist);
1054 struct perf_cpu_map *cpus = evlist__core(evlist)->all_cpus;
1055 bool per_thread = evlist__per_thread(evlist);
1056
1057 if (per_thread)
1058 thread_data->nr_mmaps = nr_mmaps;
1059 else
1060 thread_data->nr_mmaps = bitmap_weight(thread_data->mask->maps.bits,
1061 thread_data->mask->maps.nbits);
1062 if (mmap) {
1063 thread_data->maps = calloc(thread_data->nr_mmaps, sizeof(struct mmap *));
1064 if (!thread_data->maps)
1065 return -ENOMEM;
1066 }
1067 if (overwrite_mmap) {
1068 thread_data->overwrite_maps = calloc(thread_data->nr_mmaps, sizeof(struct mmap *));
1069 if (!thread_data->overwrite_maps) {
1070 zfree(&thread_data->maps);
1071 return -ENOMEM;
1072 }
1073 }
1074 pr_debug2("thread_data[%p]: nr_mmaps=%d, maps=%p, ow_maps=%p\n", thread_data,
1075 thread_data->nr_mmaps, thread_data->maps, thread_data->overwrite_maps);
1076
1077 for (m = 0, tm = 0; m < nr_mmaps && tm < thread_data->nr_mmaps; m++) {
1078 if (per_thread ||
1079 test_bit(perf_cpu_map__cpu(cpus, m).cpu, thread_data->mask->maps.bits)) {
1080 if (thread_data->maps) {
1081 thread_data->maps[tm] = &mmap[m];
1082 pr_debug2("thread_data[%p]: cpu%d: maps[%d] -> mmap[%d]\n",
1083 thread_data, perf_cpu_map__cpu(cpus, m).cpu, tm, m);
1084 }
1085 if (thread_data->overwrite_maps) {
1086 thread_data->overwrite_maps[tm] = &overwrite_mmap[m];
1087 pr_debug2("thread_data[%p]: cpu%d: ow_maps[%d] -> ow_mmap[%d]\n",
1088 thread_data, perf_cpu_map__cpu(cpus, m).cpu, tm, m);
1089 }
1090 tm++;
1091 }
1092 }
1093
1094 return 0;
1095}
1096
1097static int record__thread_data_init_pollfd(struct record_thread *thread_data, struct evlist *evlist)
1098{
1099 int f, tm, pos;
1100 struct mmap *map, *overwrite_map;
1101
1102 fdarray__init(&thread_data->pollfd, 64);
1103
1104 for (tm = 0; tm < thread_data->nr_mmaps; tm++) {
1105 map = thread_data->maps ? thread_data->maps[tm] : NULL;
1106 overwrite_map = thread_data->overwrite_maps ?
1107 thread_data->overwrite_maps[tm] : NULL;
1108
1109 for (f = 0; f < evlist__core(evlist)->pollfd.nr; f++) {
1110 void *ptr = evlist__core(evlist)->pollfd.priv[f].ptr;
1111
1112 if ((map && ptr == map) || (overwrite_map && ptr == overwrite_map)) {
1113 pos = fdarray__dup_entry_from(&thread_data->pollfd, f,
1114 &evlist__core(evlist)->pollfd);
1115 if (pos < 0)
1116 return pos;
1117 pr_debug2("thread_data[%p]: pollfd[%d] <- event_fd=%d\n",
1118 thread_data, pos,
1119 evlist__core(evlist)->pollfd.entries[f].fd);
1120 }
1121 }
1122 }
1123
1124 return 0;
1125}
1126
1127static void record__free_thread_data(struct record *rec)
1128{
1129 int t;
1130 struct record_thread *thread_data = rec->thread_data;
1131
1132 if (thread_data == NULL)
1133 return;
1134
1135 for (t = 0; t < rec->nr_threads; t++) {
1136 record__thread_data_close_pipes(&thread_data[t]);
1137 zfree(&thread_data[t].maps);
1138 zfree(&thread_data[t].overwrite_maps);
1139 fdarray__exit(&thread_data[t].pollfd);
1140 }
1141
1142 zfree(&rec->thread_data);
1143}
1144
1145static int record__map_thread_evlist_pollfd_indexes(struct record *rec,
1146 int evlist_pollfd_index,
1147 int thread_pollfd_index)
1148{
1149 size_t x = rec->index_map_cnt;
1150
1151 if (realloc_array_as_needed(rec->index_map, rec->index_map_sz, x, NULL))
1152 return -ENOMEM;
1153 rec->index_map[x].evlist_pollfd_index = evlist_pollfd_index;
1154 rec->index_map[x].thread_pollfd_index = thread_pollfd_index;
1155 rec->index_map_cnt += 1;
1156 return 0;
1157}
1158
1159static int record__update_evlist_pollfd_from_thread(struct record *rec,
1160 struct evlist *evlist,
1161 struct record_thread *thread_data)
1162{
1163 struct pollfd *e_entries = evlist__core(evlist)->pollfd.entries;
1164 struct pollfd *t_entries = thread_data->pollfd.entries;
1165 int err = 0;
1166 size_t i;
1167
1168 for (i = 0; i < rec->index_map_cnt; i++) {
1169 int e_pos = rec->index_map[i].evlist_pollfd_index;
1170 int t_pos = rec->index_map[i].thread_pollfd_index;
1171
1172 if (e_entries[e_pos].fd != t_entries[t_pos].fd ||
1173 e_entries[e_pos].events != t_entries[t_pos].events) {
1174 pr_err("Thread and evlist pollfd index mismatch\n");
1175 err = -EINVAL;
1176 continue;
1177 }
1178 e_entries[e_pos].revents = t_entries[t_pos].revents;
1179 }
1180 return err;
1181}
1182
1183static int record__dup_non_perf_events(struct record *rec,
1184 struct evlist *evlist,
1185 struct record_thread *thread_data)
1186{
1187 struct fdarray *fda = &evlist__core(evlist)->pollfd;
1188 int i, ret;
1189
1190 for (i = 0; i < fda->nr; i++) {
1191 if (!(fda->priv[i].flags & fdarray_flag__non_perf_event))
1192 continue;
1193 ret = fdarray__dup_entry_from(&thread_data->pollfd, i, fda);
1194 if (ret < 0) {
1195 pr_err("Failed to duplicate descriptor in main thread pollfd\n");
1196 return ret;
1197 }
1198 pr_debug2("thread_data[%p]: pollfd[%d] <- non_perf_event fd=%d\n",
1199 thread_data, ret, fda->entries[i].fd);
1200 ret = record__map_thread_evlist_pollfd_indexes(rec, i, ret);
1201 if (ret < 0) {
1202 pr_err("Failed to map thread and evlist pollfd indexes\n");
1203 return ret;
1204 }
1205 }
1206 return 0;
1207}
1208
1209static int record__alloc_thread_data(struct record *rec, struct evlist *evlist)
1210{
1211 int t, ret;
1212 struct record_thread *thread_data;
1213
1214 rec->thread_data = calloc(rec->nr_threads, sizeof(*(rec->thread_data)));
1215 if (!rec->thread_data) {
1216 pr_err("Failed to allocate thread data\n");
1217 return -ENOMEM;
1218 }
1219 thread_data = rec->thread_data;
1220
1221 for (t = 0; t < rec->nr_threads; t++)
1222 record__thread_data_init_pipes(&thread_data[t]);
1223
1224 for (t = 0; t < rec->nr_threads; t++) {
1225 thread_data[t].rec = rec;
1226 thread_data[t].mask = &rec->thread_masks[t];
1227 ret = record__thread_data_init_maps(&thread_data[t], evlist);
1228 if (ret) {
1229 pr_err("Failed to initialize thread[%d] maps\n", t);
1230 goto out_free;
1231 }
1232 ret = record__thread_data_init_pollfd(&thread_data[t], evlist);
1233 if (ret) {
1234 pr_err("Failed to initialize thread[%d] pollfd\n", t);
1235 goto out_free;
1236 }
1237 if (t) {
1238 thread_data[t].tid = -1;
1239 ret = record__thread_data_open_pipes(&thread_data[t]);
1240 if (ret) {
1241 pr_err("Failed to open thread[%d] communication pipes\n", t);
1242 goto out_free;
1243 }
1244 ret = fdarray__add(&thread_data[t].pollfd, thread_data[t].pipes.msg[0],
1245 POLLIN | POLLERR | POLLHUP, fdarray_flag__nonfilterable);
1246 if (ret < 0) {
1247 pr_err("Failed to add descriptor to thread[%d] pollfd\n", t);
1248 goto out_free;
1249 }
1250 thread_data[t].ctlfd_pos = ret;
1251 pr_debug2("thread_data[%p]: pollfd[%d] <- ctl_fd=%d\n",
1252 thread_data, thread_data[t].ctlfd_pos,
1253 thread_data[t].pipes.msg[0]);
1254 } else {
1255 thread_data[t].tid = gettid();
1256
1257 ret = record__dup_non_perf_events(rec, evlist, &thread_data[t]);
1258 if (ret < 0)
1259 goto out_free;
1260
1261 thread_data[t].ctlfd_pos = -1; /* Not used */
1262 }
1263 }
1264
1265 return 0;
1266
1267out_free:
1268 record__free_thread_data(rec);
1269
1270 return ret;
1271}
1272
1273static int record__mmap_evlist(struct record *rec,
1274 struct evlist *evlist)
1275{
1276 int i, ret;
1277 struct record_opts *opts = &rec->opts;
1278 bool auxtrace_overwrite = opts->auxtrace_snapshot_mode ||
1279 opts->auxtrace_sample_mode;
1280
1281 if (opts->affinity != PERF_AFFINITY_SYS)
1282 cpu__setup_cpunode_map();
1283
1284 if (evlist__mmap_ex(evlist, opts->mmap_pages,
1285 opts->auxtrace_mmap_pages,
1286 auxtrace_overwrite,
1287 opts->nr_cblocks, opts->affinity,
1288 opts->mmap_flush, opts->comp_level) < 0) {
1289 if (errno == EPERM) {
1290 pr_err("Permission error mapping pages.\n"
1291 "Consider increasing "
1292 "/proc/sys/kernel/perf_event_mlock_kb,\n"
1293 "or try again with a smaller value of -m/--mmap_pages.\n"
1294 "(current value: %u,%u)\n",
1295 opts->mmap_pages, opts->auxtrace_mmap_pages);
1296 return -errno;
1297 } else {
1298 pr_err("failed to mmap: %m\n");
1299 if (errno)
1300 return -errno;
1301 else
1302 return -EINVAL;
1303 }
1304 }
1305
1306 if (evlist__initialize_ctlfd(evlist, opts->ctl_fd, opts->ctl_fd_ack))
1307 return -1;
1308
1309 ret = record__alloc_thread_data(rec, evlist);
1310 if (ret)
1311 return ret;
1312
1313 if (record__threads_enabled(rec)) {
1314 ret = perf_data__create_dir(&rec->data, evlist__core(evlist)->nr_mmaps);
1315 if (ret) {
1316 errno = -ret;
1317 pr_err("Failed to create data directory: %m\n");
1318 return ret;
1319 }
1320 for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) {
1321 if (evlist__mmap(evlist))
1322 evlist__mmap(evlist)[i].file = &rec->data.dir.files[i];
1323 if (evlist__overwrite_mmap(evlist))
1324 evlist__overwrite_mmap(evlist)[i].file = &rec->data.dir.files[i];
1325 }
1326 }
1327
1328 return 0;
1329}
1330
1331static int record__mmap(struct record *rec)
1332{
1333 return record__mmap_evlist(rec, rec->evlist);
1334}
1335
1336static int record__open(struct record *rec)
1337{
1338 char msg[BUFSIZ];
1339 struct evsel *pos;
1340 struct evlist *evlist = rec->evlist;
1341 struct perf_session *session = rec->session;
1342 struct record_opts *opts = &rec->opts;
1343 int rc = 0;
1344 bool skipped = false;
1345 bool removed_tracking = false;
1346
1347 evlist__for_each_entry(evlist, pos) {
1348 if (removed_tracking) {
1349 /*
1350 * Normally the head of the list has tracking enabled
1351 * for sideband data like mmaps. If this event is
1352 * removed, make sure to add tracking to the next
1353 * processed event.
1354 */
1355 if (!pos->tracking) {
1356 pos->tracking = true;
1357 evsel__config(pos, opts, &callchain_param);
1358 }
1359 removed_tracking = false;
1360 }
1361try_again:
1362 if (evsel__open(pos, pos->core.cpus, pos->core.threads) < 0) {
1363 bool report_error = true;
1364
1365 if (evsel__fallback(pos, &opts->target, errno, msg, sizeof(msg))) {
1366 if (verbose > 0)
1367 ui__warning("%s\n", msg);
1368 goto try_again;
1369 }
1370 if ((errno == EINVAL || errno == EBADF) &&
1371 pos->core.leader != &pos->core &&
1372 pos->weak_group) {
1373 pos = evlist__reset_weak_group(evlist, pos, true);
1374 goto try_again;
1375 }
1376#if defined(__aarch64__) || defined(__arm__)
1377 if (strstr(evsel__name(pos), "cycles")) {
1378 struct evsel *pos2;
1379 /*
1380 * Unfortunately ARM has many events named
1381 * "cycles" on PMUs like the system-level (L3)
1382 * cache which don't support sampling. Only
1383 * display such failures to open when there is
1384 * only 1 cycles event or verbose is enabled.
1385 */
1386 evlist__for_each_entry(evlist, pos2) {
1387 if (pos2 == pos)
1388 continue;
1389 if (strstr(evsel__name(pos2), "cycles")) {
1390 report_error = false;
1391 break;
1392 }
1393 }
1394 }
1395#endif
1396 if (report_error || verbose > 0) {
1397 evsel__open_strerror(pos, &opts->target, errno, msg, sizeof(msg));
1398 ui__error("Failure to open event '%s' on PMU '%s' which will be "
1399 "removed.\n%s\n",
1400 evsel__name(pos), evsel__pmu_name(pos), msg);
1401 }
1402 if (pos->tracking)
1403 removed_tracking = true;
1404 pos->skippable = true;
1405 skipped = true;
1406 }
1407 }
1408
1409 if (skipped) {
1410 struct evsel *tmp;
1411 int idx = 0;
1412 bool evlist_empty = true;
1413
1414 /* Remove evsels that failed to open and update indices. */
1415 evlist__for_each_entry_safe(evlist, tmp, pos) {
1416 if (pos->skippable) {
1417 evlist__remove(evlist, pos);
1418 continue;
1419 }
1420
1421 /*
1422 * Note, dummy events may be command line parsed or
1423 * added by the tool. We care about supporting `perf
1424 * record -e dummy` which may be used as a permission
1425 * check. Dummy events that are added to the command
1426 * line and opened along with other events that fail,
1427 * will still fail as if the dummy events were tool
1428 * added events for the sake of code simplicity.
1429 */
1430 if (!evsel__is_dummy_event(pos))
1431 evlist_empty = false;
1432 }
1433 evlist__for_each_entry(evlist, pos) {
1434 pos->core.idx = idx++;
1435 }
1436 /* If list is empty then fail. */
1437 if (evlist_empty) {
1438 ui__error("Failure to open any events for recording.\n");
1439 rc = -1;
1440 goto out;
1441 }
1442 }
1443 if (symbol_conf.kptr_restrict && !evlist__exclude_kernel(evlist)) {
1444 pr_warning(
1445"WARNING: Kernel address maps (/proc/{kallsyms,modules}) are restricted,\n"
1446"check /proc/sys/kernel/kptr_restrict and /proc/sys/kernel/perf_event_paranoid.\n\n"
1447"Samples in kernel functions may not be resolved if a suitable vmlinux\n"
1448"file is not found in the buildid cache or in the vmlinux path.\n\n"
1449"Samples in kernel modules won't be resolved at all.\n\n"
1450"If some relocation was applied (e.g. kexec) symbols may be misresolved\n"
1451"even with a suitable vmlinux or kallsyms file.\n\n");
1452 }
1453
1454 if (evlist__apply_filters(evlist, &pos, &opts->target)) {
1455 pr_err("failed to set filter \"%s\" on event %s: %m\n",
1456 pos->filter ?: "BPF", evsel__name(pos));
1457 rc = -1;
1458 goto out;
1459 }
1460
1461 rc = record__mmap(rec);
1462 if (rc)
1463 goto out;
1464
1465 session->evlist = evlist;
1466 perf_session__set_id_hdr_size(session);
1467out:
1468 return rc;
1469}
1470
1471static void set_timestamp_boundary(struct record *rec, u64 sample_time)
1472{
1473 if (evlist__first_sample_time(rec->evlist) == 0)
1474 evlist__set_first_sample_time(rec->evlist, sample_time);
1475
1476 if (sample_time)
1477 evlist__set_last_sample_time(rec->evlist, sample_time);
1478}
1479
1480static int process_sample_event(const struct perf_tool *tool,
1481 union perf_event *event,
1482 struct perf_sample *sample,
1483 struct machine *machine)
1484{
1485 struct record *rec = container_of(tool, struct record, tool);
1486
1487 set_timestamp_boundary(rec, sample->time);
1488
1489 if (rec->buildid_all)
1490 return 0;
1491
1492 rec->samples++;
1493 return build_id__mark_dso_hit(tool, event, sample, machine);
1494}
1495
1496static int process_buildids(struct record *rec)
1497{
1498 struct perf_session *session = rec->session;
1499
1500 if (perf_data__size(&rec->data) == 0)
1501 return 0;
1502
1503 /* A single DSO is needed and not all inline frames. */
1504 symbol_conf.inline_name = false;
1505 /*
1506 * During this process, it'll load kernel map and replace the
1507 * dso->long_name to a real pathname it found. In this case
1508 * we prefer the vmlinux path like
1509 * /lib/modules/3.16.4/build/vmlinux
1510 *
1511 * rather than build-id path (in debug directory).
1512 * $HOME/.debug/.build-id/f0/6e17aa50adf4d00b88925e03775de107611551
1513 */
1514 symbol_conf.ignore_vmlinux_buildid = true;
1515 /*
1516 * If --buildid-all is given, it marks all DSO regardless of hits,
1517 * so no need to process samples. But if timestamp_boundary is enabled,
1518 * it still needs to walk on all samples to get the timestamps of
1519 * first/last samples.
1520 */
1521 if (rec->buildid_all && !rec->timestamp_boundary)
1522 rec->tool.sample = process_event_sample_stub;
1523
1524 return perf_session__process_events(session);
1525}
1526
1527static void perf_event__synthesize_guest_os(struct machine *machine, void *data)
1528{
1529 int err;
1530 struct perf_tool *tool = data;
1531 /*
1532 *As for guest kernel when processing subcommand record&report,
1533 *we arrange module mmap prior to guest kernel mmap and trigger
1534 *a preload dso because default guest module symbols are loaded
1535 *from guest kallsyms instead of /lib/modules/XXX/XXX. This
1536 *method is used to avoid symbol missing when the first addr is
1537 *in module instead of in guest kernel.
1538 */
1539 err = perf_event__synthesize_modules(tool, process_synthesized_event,
1540 machine);
1541 if (err < 0)
1542 pr_err("Couldn't record guest kernel [%d]'s reference"
1543 " relocation symbol.\n", machine->pid);
1544
1545 /*
1546 * We use _stext for guest kernel because guest kernel's /proc/kallsyms
1547 * have no _text sometimes.
1548 */
1549 err = perf_event__synthesize_kernel_mmap(tool, process_synthesized_event,
1550 machine);
1551 if (err < 0)
1552 pr_err("Couldn't record guest kernel [%d]'s reference"
1553 " relocation symbol.\n", machine->pid);
1554}
1555
1556static struct perf_event_header finished_round_event = {
1557 .size = sizeof(struct perf_event_header),
1558 .type = PERF_RECORD_FINISHED_ROUND,
1559};
1560
1561static struct perf_event_header finished_init_event = {
1562 .size = sizeof(struct perf_event_header),
1563 .type = PERF_RECORD_FINISHED_INIT,
1564};
1565
1566static void record__adjust_affinity(struct record *rec, struct mmap *map)
1567{
1568 if (rec->opts.affinity != PERF_AFFINITY_SYS &&
1569 !bitmap_equal(thread->mask->affinity.bits, map->affinity_mask.bits,
1570 thread->mask->affinity.nbits)) {
1571 bitmap_zero(thread->mask->affinity.bits, thread->mask->affinity.nbits);
1572 bitmap_or(thread->mask->affinity.bits, thread->mask->affinity.bits,
1573 map->affinity_mask.bits, thread->mask->affinity.nbits);
1574 sched_setaffinity(0, MMAP_CPU_MASK_BYTES(&thread->mask->affinity),
1575 (cpu_set_t *)thread->mask->affinity.bits);
1576 if (verbose == 2) {
1577 pr_debug("threads[%d]: running on cpu%d: ", thread->tid, sched_getcpu());
1578 mmap_cpu_mask__scnprintf(&thread->mask->affinity, "affinity");
1579 }
1580 }
1581}
1582
1583/*
1584 * Called once with data_size == 0 to start a record, then once with
1585 * data_size == compressed payload size to finalize and 8-byte-pad it
1586 * (unaligned records trip ASan in the reader).
1587 * Returns the bytes written, or -1 if it won't fit.
1588 */
1589static ssize_t process_comp_header(void *record, size_t dst_size,
1590 size_t data_size)
1591{
1592 struct perf_record_compressed2 *event = record;
1593 size_t size = sizeof(*event);
1594
1595 if (data_size) {
1596 size_t padding;
1597
1598 event->data_size = data_size;
1599 event->header.size = PERF_ALIGN(size + data_size, sizeof(u64));
1600 padding = event->header.size - size - data_size;
1601 if (padding > dst_size)
1602 return -1;
1603 memset(record + size + data_size, 0, padding);
1604 return padding;
1605 }
1606
1607 if (size > dst_size)
1608 return -1;
1609
1610 event->header.type = PERF_RECORD_COMPRESSED2;
1611 event->header.size = size;
1612 event->data_size = 0;
1613
1614 return size;
1615}
1616
1617static ssize_t zstd_compress(struct perf_session *session, struct mmap *map,
1618 void *dst, size_t dst_size, void *src, size_t src_size)
1619{
1620 ssize_t compressed;
1621 /*
1622 * Reserve space so per-record PERF_ALIGN() padding keeps header.size
1623 * within u16.
1624 */
1625 size_t max_record_size = PERF_SAMPLE_MAX_SIZE
1626 - sizeof(struct perf_record_compressed2) - sizeof(u64);
1627 struct zstd_data *zstd_data = &session->zstd_data;
1628
1629 if (map && map->file)
1630 zstd_data = &map->zstd_data;
1631
1632 compressed = zstd_compress_stream_to_records(zstd_data, dst, dst_size, src, src_size,
1633 max_record_size, process_comp_header);
1634 if (compressed < 0)
1635 return compressed;
1636
1637 if (map && map->file) {
1638 thread->bytes_transferred += src_size;
1639 thread->bytes_compressed += compressed;
1640 } else {
1641 session->bytes_transferred += src_size;
1642 session->bytes_compressed += compressed;
1643 }
1644
1645 return compressed;
1646}
1647
1648static int record__mmap_read_evlist(struct record *rec, struct evlist *evlist,
1649 bool overwrite, bool synch)
1650{
1651 u64 bytes_written = rec->bytes_written;
1652 int i;
1653 int rc = 0;
1654 int nr_mmaps;
1655 struct mmap **maps;
1656 int trace_fd = perf_data__fd(&rec->data);
1657 off_t off = 0;
1658
1659 if (!evlist)
1660 return 0;
1661
1662 nr_mmaps = thread->nr_mmaps;
1663 maps = overwrite ? thread->overwrite_maps : thread->maps;
1664
1665 if (!maps)
1666 return 0;
1667
1668 if (overwrite && evlist__bkw_mmap_state(evlist) != BKW_MMAP_DATA_PENDING)
1669 return 0;
1670
1671 if (record__aio_enabled(rec))
1672 off = record__aio_get_pos(trace_fd);
1673
1674 for (i = 0; i < nr_mmaps; i++) {
1675 u64 flush = 0;
1676 struct mmap *map = maps[i];
1677
1678 if (map->core.base) {
1679 record__adjust_affinity(rec, map);
1680 if (synch) {
1681 flush = map->core.flush;
1682 map->core.flush = 1;
1683 }
1684 if (!record__aio_enabled(rec)) {
1685 if (perf_mmap__push(map, rec, record__pushfn) < 0) {
1686 if (synch)
1687 map->core.flush = flush;
1688 rc = -1;
1689 goto out;
1690 }
1691 } else {
1692 if (record__aio_push(rec, map, &off) < 0) {
1693 record__aio_set_pos(trace_fd, off);
1694 if (synch)
1695 map->core.flush = flush;
1696 rc = -1;
1697 goto out;
1698 }
1699 }
1700 if (synch)
1701 map->core.flush = flush;
1702 }
1703
1704 if (map->auxtrace_mmap.base && !rec->opts.auxtrace_snapshot_mode &&
1705 !rec->opts.auxtrace_sample_mode &&
1706 record__auxtrace_mmap_read(rec, map) != 0) {
1707 rc = -1;
1708 goto out;
1709 }
1710 }
1711
1712 if (record__aio_enabled(rec))
1713 record__aio_set_pos(trace_fd, off);
1714
1715 /*
1716 * Mark the round finished in case we wrote
1717 * at least one event.
1718 *
1719 * No need for round events in directory mode,
1720 * because per-cpu maps and files have data
1721 * sorted by kernel.
1722 */
1723 if (!record__threads_enabled(rec) && bytes_written != rec->bytes_written)
1724 rc = record__write(rec, NULL, &finished_round_event, sizeof(finished_round_event));
1725
1726 if (overwrite)
1727 evlist__toggle_bkw_mmap(evlist, BKW_MMAP_EMPTY);
1728out:
1729 return rc;
1730}
1731
1732static int record__mmap_read_all(struct record *rec, bool synch)
1733{
1734 int err;
1735
1736 err = record__mmap_read_evlist(rec, rec->evlist, false, synch);
1737 if (err)
1738 return err;
1739
1740 return record__mmap_read_evlist(rec, rec->evlist, true, synch);
1741}
1742
1743static void record__thread_munmap_filtered(struct fdarray *fda, int fd,
1744 void *arg __maybe_unused)
1745{
1746 struct perf_mmap *map = fda->priv[fd].ptr;
1747
1748 if (map)
1749 perf_mmap__put(map);
1750}
1751
1752static void *record__thread(void *arg)
1753{
1754 enum thread_msg msg = THREAD_MSG__READY;
1755 bool terminate = false;
1756 struct fdarray *pollfd;
1757 int err, ctlfd_pos;
1758
1759 thread = arg;
1760 thread->tid = gettid();
1761
1762 err = write(thread->pipes.ack[1], &msg, sizeof(msg));
1763 if (err == -1)
1764 pr_warning("threads[%d]: failed to notify on start: %m\n", thread->tid);
1765
1766 pr_debug("threads[%d]: started on cpu%d\n", thread->tid, sched_getcpu());
1767
1768 pollfd = &thread->pollfd;
1769 ctlfd_pos = thread->ctlfd_pos;
1770
1771 for (;;) {
1772 unsigned long long hits = thread->samples;
1773
1774 if (record__mmap_read_all(thread->rec, false) < 0 || terminate)
1775 break;
1776
1777 if (hits == thread->samples) {
1778
1779 err = fdarray__poll(pollfd, -1);
1780 /*
1781 * Propagate error, only if there's any. Ignore positive
1782 * number of returned events and interrupt error.
1783 */
1784 if (err > 0 || (err < 0 && errno == EINTR))
1785 err = 0;
1786 thread->waking++;
1787
1788 if (fdarray__filter(pollfd, POLLERR | POLLHUP,
1789 record__thread_munmap_filtered, NULL) == 0)
1790 break;
1791 }
1792
1793 if (pollfd->entries[ctlfd_pos].revents & POLLHUP) {
1794 terminate = true;
1795 close(thread->pipes.msg[0]);
1796 thread->pipes.msg[0] = -1;
1797 pollfd->entries[ctlfd_pos].fd = -1;
1798 pollfd->entries[ctlfd_pos].events = 0;
1799 }
1800
1801 pollfd->entries[ctlfd_pos].revents = 0;
1802 }
1803 record__mmap_read_all(thread->rec, true);
1804
1805 err = write(thread->pipes.ack[1], &msg, sizeof(msg));
1806 if (err == -1)
1807 pr_warning("threads[%d]: failed to notify on termination: %m\n", thread->tid);
1808
1809 return NULL;
1810}
1811
1812static void record__init_features(struct record *rec)
1813{
1814 struct perf_session *session = rec->session;
1815 int feat;
1816
1817 for (feat = HEADER_FIRST_FEATURE; feat < HEADER_LAST_FEATURE; feat++)
1818 perf_header__set_feat(&session->header, feat);
1819
1820 if (rec->no_buildid)
1821 perf_header__clear_feat(&session->header, HEADER_BUILD_ID);
1822
1823 if (!have_tracepoints(&evlist__core(rec->evlist)->entries))
1824 perf_header__clear_feat(&session->header, HEADER_TRACING_DATA);
1825
1826 if (!rec->opts.branch_stack)
1827 perf_header__clear_feat(&session->header, HEADER_BRANCH_STACK);
1828
1829 if (!rec->opts.full_auxtrace)
1830 perf_header__clear_feat(&session->header, HEADER_AUXTRACE);
1831
1832 if (!(rec->opts.use_clockid && rec->opts.clockid_res_ns))
1833 perf_header__clear_feat(&session->header, HEADER_CLOCKID);
1834
1835 if (!rec->opts.use_clockid)
1836 perf_header__clear_feat(&session->header, HEADER_CLOCK_DATA);
1837
1838 if (!record__threads_enabled(rec))
1839 perf_header__clear_feat(&session->header, HEADER_DIR_FORMAT);
1840
1841 if (!record__comp_enabled(rec))
1842 perf_header__clear_feat(&session->header, HEADER_COMPRESSED);
1843
1844 perf_header__clear_feat(&session->header, HEADER_STAT);
1845}
1846
1847static void
1848record__finish_output(struct record *rec)
1849{
1850 int i;
1851 struct perf_data *data = &rec->data;
1852 int fd = perf_data__fd(data);
1853
1854 if (data->is_pipe) {
1855 /* Just to display approx. size */
1856 data->file.size = rec->bytes_written;
1857 return;
1858 }
1859
1860 rec->session->header.data_size += rec->bytes_written;
1861 data->file.size = perf_data__seek(data, 0, SEEK_CUR);
1862 if (record__threads_enabled(rec)) {
1863 for (i = 0; i < data->dir.nr; i++) {
1864 data->dir.files[i].size =
1865 perf_data_file__seek(&data->dir.files[i], 0, SEEK_CUR);
1866 }
1867 }
1868
1869 /* Buildid scanning disabled or build ID in kernel and synthesized map events. */
1870 if (!rec->no_buildid || !rec->no_buildid_cache) {
1871 process_buildids(rec);
1872
1873 if (rec->buildid_all)
1874 perf_session__dsos_hit_all(rec->session);
1875 }
1876 perf_session__write_header(rec->session, rec->evlist, fd, true);
1877 perf_session__cache_build_ids(rec->session);
1878}
1879
1880static int record__synthesize_workload(struct record *rec, bool tail)
1881{
1882 int err;
1883 struct perf_thread_map *thread_map;
1884 bool needs_mmap = rec->opts.synth & PERF_SYNTH_MMAP;
1885
1886 if (rec->opts.tail_synthesize != tail)
1887 return 0;
1888
1889 thread_map = thread_map__new_by_tid(evlist__workload_pid(rec->evlist));
1890 if (thread_map == NULL)
1891 return -1;
1892
1893 err = perf_event__synthesize_thread_map(&rec->tool, thread_map,
1894 process_synthesized_event,
1895 &rec->session->machines.host,
1896 needs_mmap,
1897 rec->opts.record_data_mmap);
1898 perf_thread_map__put(thread_map);
1899 return err;
1900}
1901
1902static int write_finished_init(struct record *rec, bool tail)
1903{
1904 if (rec->opts.tail_synthesize != tail)
1905 return 0;
1906
1907 return record__write(rec, NULL, &finished_init_event, sizeof(finished_init_event));
1908}
1909
1910static int record__synthesize(struct record *rec, bool tail);
1911
1912static int
1913record__switch_output(struct record *rec, bool at_exit)
1914{
1915 struct perf_data *data = &rec->data;
1916 char *new_filename = NULL;
1917 int fd, err;
1918
1919 /* Same Size: "2015122520103046"*/
1920 char timestamp[] = "InvalidTimestamp";
1921
1922 record__aio_mmap_read_sync(rec);
1923
1924 write_finished_init(rec, true);
1925
1926 record__synthesize(rec, true);
1927 if (target__none(&rec->opts.target))
1928 record__synthesize_workload(rec, true);
1929
1930 rec->samples = 0;
1931 record__finish_output(rec);
1932 err = fetch_current_timestamp(timestamp, sizeof(timestamp));
1933 if (err) {
1934 pr_err("Failed to get current timestamp\n");
1935 return -EINVAL;
1936 }
1937
1938 fd = perf_data__switch(data, timestamp,
1939 rec->session->header.data_offset,
1940 at_exit, &new_filename);
1941 if (fd >= 0 && !at_exit) {
1942 rec->bytes_written = 0;
1943 rec->session->header.data_size = 0;
1944 }
1945
1946 if (!quiet) {
1947 fprintf(stderr, "[ perf record: Dump %s.%s ]\n",
1948 data->path, timestamp);
1949 }
1950
1951 if (rec->switch_output.num_files) {
1952 int n = rec->switch_output.cur_file + 1;
1953
1954 if (n >= rec->switch_output.num_files)
1955 n = 0;
1956 rec->switch_output.cur_file = n;
1957 if (rec->switch_output.filenames[n]) {
1958 remove(rec->switch_output.filenames[n]);
1959 zfree(&rec->switch_output.filenames[n]);
1960 }
1961 rec->switch_output.filenames[n] = new_filename;
1962 } else {
1963 free(new_filename);
1964 }
1965
1966 /* Output tracking events */
1967 if (!at_exit) {
1968 record__synthesize(rec, false);
1969
1970 /*
1971 * In 'perf record --switch-output' without -a,
1972 * record__synthesize() in record__switch_output() won't
1973 * generate tracking events because there's no thread_map
1974 * in evlist. Which causes newly created perf.data doesn't
1975 * contain map and comm information.
1976 * Create a fake thread_map and directly call
1977 * perf_event__synthesize_thread_map() for those events.
1978 */
1979 if (target__none(&rec->opts.target))
1980 record__synthesize_workload(rec, false);
1981 write_finished_init(rec, false);
1982 }
1983 return fd;
1984}
1985
1986static void __record__save_lost_samples(struct record *rec, struct evsel *evsel,
1987 struct perf_record_lost_samples *lost,
1988 int cpu_idx, int thread_idx, u64 lost_count,
1989 u16 misc_flag)
1990{
1991 struct perf_sample_id *sid;
1992 struct perf_sample sample;
1993 int id_hdr_size;
1994
1995 perf_sample__init(&sample, /*all=*/true);
1996 lost->lost = lost_count;
1997 if (evsel->core.ids) {
1998 sid = xyarray__entry(evsel->core.sample_id, cpu_idx, thread_idx);
1999 sample.id = sid->id;
2000 }
2001
2002 id_hdr_size = perf_event__synthesize_id_sample((void *)(lost + 1),
2003 evsel->core.attr.sample_type, &sample);
2004 lost->header.size = sizeof(*lost) + id_hdr_size;
2005 lost->header.misc = misc_flag;
2006 record__write(rec, NULL, lost, lost->header.size);
2007 perf_sample__exit(&sample);
2008}
2009
2010static void record__read_lost_samples(struct record *rec)
2011{
2012 struct perf_session *session = rec->session;
2013 struct perf_record_lost_samples_and_ids lost;
2014 struct evsel *evsel;
2015
2016 /* there was an error during record__open */
2017 if (session->evlist == NULL)
2018 return;
2019
2020 evlist__for_each_entry(session->evlist, evsel) {
2021 struct xyarray *xy = evsel->core.sample_id;
2022 u64 lost_count;
2023
2024 if (xy == NULL || evsel->core.fd == NULL)
2025 continue;
2026 if (xyarray__max_x(evsel->core.fd) != xyarray__max_x(xy) ||
2027 xyarray__max_y(evsel->core.fd) != xyarray__max_y(xy)) {
2028 pr_debug("Unmatched FD vs. sample ID: skip reading LOST count\n");
2029 continue;
2030 }
2031
2032 for (int x = 0; x < xyarray__max_x(xy); x++) {
2033 for (int y = 0; y < xyarray__max_y(xy); y++) {
2034 struct perf_counts_values count;
2035
2036 if (perf_evsel__read(&evsel->core, x, y, &count) < 0) {
2037 pr_debug("read LOST count failed\n");
2038 return;
2039 }
2040
2041 if (count.lost) {
2042 memset(&lost, 0, sizeof(lost));
2043 lost.lost.header.type = PERF_RECORD_LOST_SAMPLES;
2044 __record__save_lost_samples(rec, evsel, &lost.lost,
2045 x, y, count.lost, 0);
2046 }
2047 }
2048 }
2049
2050 lost_count = perf_bpf_filter__lost_count(evsel);
2051 if (lost_count) {
2052 memset(&lost, 0, sizeof(lost));
2053 lost.lost.header.type = PERF_RECORD_LOST_SAMPLES;
2054 __record__save_lost_samples(rec, evsel, &lost.lost, 0, 0, lost_count,
2055 PERF_RECORD_MISC_LOST_SAMPLES_BPF);
2056 }
2057 }
2058}
2059
2060static volatile sig_atomic_t workload_exec_errno;
2061
2062/*
2063 * evlist__prepare_workload will send a SIGUSR1
2064 * if the fork fails, since we asked by setting its
2065 * want_signal to true.
2066 */
2067static void workload_exec_failed_signal(int signo __maybe_unused,
2068 siginfo_t *info,
2069 void *ucontext __maybe_unused)
2070{
2071 workload_exec_errno = info->si_value.sival_int;
2072 done = 1;
2073 child_finished = 1;
2074}
2075
2076static void snapshot_sig_handler(int sig);
2077static void alarm_sig_handler(int sig);
2078
2079static const struct perf_event_mmap_page *evlist__pick_pc(struct evlist *evlist)
2080{
2081 if (evlist) {
2082 if (evlist__mmap(evlist) && evlist__mmap(evlist)[0].core.base)
2083 return evlist__mmap(evlist)[0].core.base;
2084 if (evlist__overwrite_mmap(evlist) && evlist__overwrite_mmap(evlist)[0].core.base)
2085 return evlist__overwrite_mmap(evlist)[0].core.base;
2086 }
2087 return NULL;
2088}
2089
2090static const struct perf_event_mmap_page *record__pick_pc(struct record *rec)
2091{
2092 const struct perf_event_mmap_page *pc = evlist__pick_pc(rec->evlist);
2093 if (pc)
2094 return pc;
2095 return NULL;
2096}
2097
2098static int record__synthesize(struct record *rec, bool tail)
2099{
2100 struct perf_session *session = rec->session;
2101 struct machine *machine = &session->machines.host;
2102 struct perf_data *data = &rec->data;
2103 struct record_opts *opts = &rec->opts;
2104 struct perf_tool *tool = &rec->tool;
2105 int err = 0;
2106 event_op f = process_synthesized_event;
2107
2108 if (rec->opts.tail_synthesize != tail)
2109 return 0;
2110
2111 if (data->is_pipe) {
2112 err = perf_event__synthesize_for_pipe(tool, session, data,
2113 process_synthesized_event);
2114 if (err < 0)
2115 goto out;
2116
2117 rec->bytes_written += err;
2118 }
2119
2120 err = perf_event__synth_time_conv(record__pick_pc(rec), tool,
2121 process_synthesized_event, machine);
2122 if (err)
2123 goto out;
2124
2125 /* Synthesize id_index before auxtrace_info */
2126 err = perf_event__synthesize_id_index(tool,
2127 process_synthesized_event,
2128 session->evlist, machine);
2129 if (err)
2130 goto out;
2131
2132 if (rec->opts.full_auxtrace) {
2133 err = perf_event__synthesize_auxtrace_info(rec->itr, tool,
2134 session, process_synthesized_event);
2135 if (err)
2136 goto out;
2137 }
2138
2139 if (!evlist__exclude_kernel(rec->evlist)) {
2140 err = perf_event__synthesize_kernel_mmap(tool, process_synthesized_event,
2141 machine);
2142 WARN_ONCE(err < 0, "Couldn't record kernel reference relocation symbol\n"
2143 "Symbol resolution may be skewed if relocation was used (e.g. kexec).\n"
2144 "Check /proc/kallsyms permission or run as root.\n");
2145
2146 err = perf_event__synthesize_modules(tool, process_synthesized_event,
2147 machine);
2148 WARN_ONCE(err < 0, "Couldn't record kernel module information.\n"
2149 "Symbol resolution may be skewed if relocation was used (e.g. kexec).\n"
2150 "Check /proc/modules permission or run as root.\n");
2151 }
2152
2153 if (perf_guest) {
2154 machines__process_guests(&session->machines,
2155 perf_event__synthesize_guest_os, tool);
2156 }
2157
2158 err = perf_event__synthesize_extra_attr(&rec->tool,
2159 rec->evlist,
2160 process_synthesized_event,
2161 data->is_pipe);
2162 if (err)
2163 goto out;
2164
2165 err = perf_event__synthesize_thread_map2(&rec->tool, evlist__core(rec->evlist)->threads,
2166 process_synthesized_event,
2167 NULL);
2168 if (err < 0) {
2169 pr_err("Couldn't synthesize thread map.\n");
2170 return err;
2171 }
2172
2173 err = perf_event__synthesize_cpu_map(&rec->tool, evlist__core(rec->evlist)->all_cpus,
2174 process_synthesized_event, NULL);
2175 if (err < 0) {
2176 pr_err("Couldn't synthesize cpu map.\n");
2177 return err;
2178 }
2179
2180 err = perf_event__synthesize_bpf_events(session, process_synthesized_event,
2181 machine, opts);
2182 if (err < 0) {
2183 pr_warning("Couldn't synthesize bpf events.\n");
2184 err = 0;
2185 }
2186
2187 if (rec->opts.synth & PERF_SYNTH_CGROUP) {
2188 err = perf_event__synthesize_cgroups(tool, process_synthesized_event,
2189 machine);
2190 if (err < 0) {
2191 pr_warning("Couldn't synthesize cgroup events.\n");
2192 err = 0;
2193 }
2194 }
2195
2196 if (rec->opts.nr_threads_synthesize > 1) {
2197 mutex_init(&synth_lock);
2198 perf_set_multithreaded();
2199 f = process_locked_synthesized_event;
2200 }
2201
2202 if (rec->opts.synth & PERF_SYNTH_TASK) {
2203 bool needs_mmap = rec->opts.synth & PERF_SYNTH_MMAP;
2204
2205 err = __machine__synthesize_threads(machine, tool, &opts->target,
2206 evlist__core(rec->evlist)->threads,
2207 f, needs_mmap, opts->record_data_mmap,
2208 rec->opts.nr_threads_synthesize);
2209 }
2210
2211 if (rec->opts.nr_threads_synthesize > 1) {
2212 perf_set_singlethreaded();
2213 mutex_destroy(&synth_lock);
2214 }
2215
2216out:
2217 return err;
2218}
2219
2220static void record__synthesize_final_bpf_metadata(struct record *rec __maybe_unused)
2221{
2222#ifdef HAVE_LIBBPF_SUPPORT
2223 perf_event__synthesize_final_bpf_metadata(rec->session,
2224 process_synthesized_event);
2225#endif
2226}
2227
2228static int record__process_signal_event(union perf_event *event __maybe_unused, void *data)
2229{
2230 struct record *rec = data;
2231 pthread_kill(rec->thread_id, SIGUSR2);
2232 return 0;
2233}
2234
2235static int record__setup_sb_evlist(struct record *rec)
2236{
2237 struct record_opts *opts = &rec->opts;
2238
2239 if (rec->sb_evlist != NULL) {
2240 /*
2241 * We get here if --switch-output-event populated the
2242 * sb_evlist, so associate a callback that will send a SIGUSR2
2243 * to the main thread.
2244 */
2245 evlist__set_cb(rec->sb_evlist, record__process_signal_event, rec);
2246 rec->thread_id = pthread_self();
2247 }
2248#ifdef HAVE_LIBBPF_SUPPORT
2249 if (!opts->no_bpf_event) {
2250 if (rec->sb_evlist == NULL) {
2251 rec->sb_evlist = evlist__new();
2252
2253 if (rec->sb_evlist == NULL) {
2254 pr_err("Couldn't create side band evlist.\n.");
2255 return -1;
2256 }
2257 }
2258
2259 if (evlist__add_bpf_sb_event(rec->sb_evlist, perf_session__env(rec->session))) {
2260 pr_err("Couldn't ask for PERF_RECORD_BPF_EVENT side band events.\n.");
2261 evlist__put(rec->sb_evlist);
2262 rec->sb_evlist = NULL;
2263 return -1;
2264 }
2265 }
2266#endif
2267 if (evlist__start_sb_thread(rec->sb_evlist, &rec->opts.target)) {
2268 pr_debug("Couldn't start the BPF side band thread:\nBPF programs starting from now on won't be annotatable\n");
2269 opts->no_bpf_event = true;
2270 }
2271
2272 return 0;
2273}
2274
2275static int record__init_clock(struct record *rec)
2276{
2277 struct perf_session *session = rec->session;
2278 struct timespec ref_clockid;
2279 struct timeval ref_tod;
2280 struct perf_env *env = perf_session__env(session);
2281 u64 ref;
2282
2283 if (!rec->opts.use_clockid)
2284 return 0;
2285
2286 if (rec->opts.use_clockid && rec->opts.clockid_res_ns)
2287 env->clock.clockid_res_ns = rec->opts.clockid_res_ns;
2288
2289 env->clock.clockid = rec->opts.clockid;
2290
2291 if (gettimeofday(&ref_tod, NULL) != 0) {
2292 pr_err("gettimeofday failed, cannot set reference time.\n");
2293 return -1;
2294 }
2295
2296 if (clock_gettime(rec->opts.clockid, &ref_clockid)) {
2297 pr_err("clock_gettime failed, cannot set reference time.\n");
2298 return -1;
2299 }
2300
2301 ref = (u64) ref_tod.tv_sec * NSEC_PER_SEC +
2302 (u64) ref_tod.tv_usec * NSEC_PER_USEC;
2303
2304 env->clock.tod_ns = ref;
2305
2306 ref = (u64) ref_clockid.tv_sec * NSEC_PER_SEC +
2307 (u64) ref_clockid.tv_nsec;
2308
2309 env->clock.clockid_ns = ref;
2310 return 0;
2311}
2312
2313static void hit_auxtrace_snapshot_trigger(struct record *rec)
2314{
2315 if (trigger_is_ready(&auxtrace_snapshot_trigger)) {
2316 trigger_hit(&auxtrace_snapshot_trigger);
2317 auxtrace_record__snapshot_started = 1;
2318 if (auxtrace_record__snapshot_start(rec->itr))
2319 trigger_error(&auxtrace_snapshot_trigger);
2320 }
2321}
2322
2323static int record__terminate_thread(struct record_thread *thread_data)
2324{
2325 int err;
2326 enum thread_msg ack = THREAD_MSG__UNDEFINED;
2327 pid_t tid = thread_data->tid;
2328
2329 close(thread_data->pipes.msg[1]);
2330 thread_data->pipes.msg[1] = -1;
2331 err = read(thread_data->pipes.ack[0], &ack, sizeof(ack));
2332 if (err > 0)
2333 pr_debug2("threads[%d]: sent %s\n", tid, thread_msg_tags[ack]);
2334 else
2335 pr_warning("threads[%d]: failed to receive termination notification from %d\n",
2336 thread->tid, tid);
2337
2338 return 0;
2339}
2340
2341static int record__start_threads(struct record *rec)
2342{
2343 int t, tt, err, ret = 0, nr_threads = rec->nr_threads;
2344 struct record_thread *thread_data = rec->thread_data;
2345 sigset_t full, mask;
2346 pthread_t handle;
2347 pthread_attr_t attrs;
2348
2349 thread = &thread_data[0];
2350
2351 if (!record__threads_enabled(rec))
2352 return 0;
2353
2354 sigfillset(&full);
2355 if (sigprocmask(SIG_SETMASK, &full, &mask)) {
2356 pr_err("Failed to block signals on threads start: %m\n");
2357 return -1;
2358 }
2359
2360 pthread_attr_init(&attrs);
2361 pthread_attr_setdetachstate(&attrs, PTHREAD_CREATE_DETACHED);
2362
2363 for (t = 1; t < nr_threads; t++) {
2364 enum thread_msg msg = THREAD_MSG__UNDEFINED;
2365
2366#ifdef HAVE_PTHREAD_ATTR_SETAFFINITY_NP
2367 pthread_attr_setaffinity_np(&attrs,
2368 MMAP_CPU_MASK_BYTES(&(thread_data[t].mask->affinity)),
2369 (cpu_set_t *)(thread_data[t].mask->affinity.bits));
2370#endif
2371 if (pthread_create(&handle, &attrs, record__thread, &thread_data[t])) {
2372 for (tt = 1; tt < t; tt++)
2373 record__terminate_thread(&thread_data[t]);
2374 pr_err("Failed to start threads: %m\n");
2375 ret = -1;
2376 goto out_err;
2377 }
2378
2379 err = read(thread_data[t].pipes.ack[0], &msg, sizeof(msg));
2380 if (err > 0)
2381 pr_debug2("threads[%d]: sent %s\n", rec->thread_data[t].tid,
2382 thread_msg_tags[msg]);
2383 else
2384 pr_warning("threads[%d]: failed to receive start notification from %d\n",
2385 thread->tid, rec->thread_data[t].tid);
2386 }
2387
2388 sched_setaffinity(0, MMAP_CPU_MASK_BYTES(&thread->mask->affinity),
2389 (cpu_set_t *)thread->mask->affinity.bits);
2390
2391 pr_debug("threads[%d]: started on cpu%d\n", thread->tid, sched_getcpu());
2392
2393out_err:
2394 pthread_attr_destroy(&attrs);
2395
2396 if (sigprocmask(SIG_SETMASK, &mask, NULL)) {
2397 pr_err("Failed to unblock signals on threads start: %m\n");
2398 ret = -1;
2399 }
2400
2401 return ret;
2402}
2403
2404static int record__stop_threads(struct record *rec)
2405{
2406 int t;
2407 struct record_thread *thread_data = rec->thread_data;
2408
2409 for (t = 1; t < rec->nr_threads; t++)
2410 record__terminate_thread(&thread_data[t]);
2411
2412 for (t = 0; t < rec->nr_threads; t++) {
2413 rec->samples += thread_data[t].samples;
2414 if (!record__threads_enabled(rec))
2415 continue;
2416 rec->session->bytes_transferred += thread_data[t].bytes_transferred;
2417 rec->session->bytes_compressed += thread_data[t].bytes_compressed;
2418 pr_debug("threads[%d]: samples=%lld, wakes=%ld, ", thread_data[t].tid,
2419 thread_data[t].samples, thread_data[t].waking);
2420 if (thread_data[t].bytes_transferred && thread_data[t].bytes_compressed)
2421 pr_debug("transferred=%" PRIu64 ", compressed=%" PRIu64 "\n",
2422 thread_data[t].bytes_transferred, thread_data[t].bytes_compressed);
2423 else
2424 pr_debug("written=%" PRIu64 "\n", thread_data[t].bytes_written);
2425 }
2426
2427 return 0;
2428}
2429
2430static unsigned long record__waking(struct record *rec)
2431{
2432 int t;
2433 unsigned long waking = 0;
2434 struct record_thread *thread_data = rec->thread_data;
2435
2436 for (t = 0; t < rec->nr_threads; t++)
2437 waking += thread_data[t].waking;
2438
2439 return waking;
2440}
2441
2442static int __cmd_record(struct record *rec, int argc, const char **argv)
2443{
2444 int err;
2445 int status = 0;
2446 const bool forks = argc > 0;
2447 struct perf_tool *tool = &rec->tool;
2448 struct record_opts *opts = &rec->opts;
2449 struct perf_data *data = &rec->data;
2450 struct perf_session *session;
2451 bool disabled = false, draining = false;
2452 int fd;
2453 float ratio = 0;
2454 enum evlist_ctl_cmd cmd = EVLIST_CTL_CMD_UNSUPPORTED;
2455 struct perf_env *env;
2456
2457 atexit(record__sig_exit);
2458 signal(SIGCHLD, sig_handler);
2459 signal(SIGINT, sig_handler);
2460 signal(SIGTERM, sig_handler);
2461 signal(SIGSEGV, sigsegv_handler);
2462
2463 if (rec->opts.record_cgroup) {
2464#ifndef HAVE_FILE_HANDLE
2465 pr_err("cgroup tracking is not supported\n");
2466 return -1;
2467#endif
2468 }
2469
2470 if (rec->opts.auxtrace_snapshot_mode || rec->switch_output.enabled) {
2471 signal(SIGUSR2, snapshot_sig_handler);
2472 if (rec->opts.auxtrace_snapshot_mode)
2473 trigger_on(&auxtrace_snapshot_trigger);
2474 if (rec->switch_output.enabled)
2475 trigger_on(&switch_output_trigger);
2476 } else {
2477 signal(SIGUSR2, SIG_IGN);
2478 }
2479
2480 perf_tool__init(tool, /*ordered_events=*/true);
2481 tool->sample = process_sample_event;
2482 tool->fork = perf_event__process_fork;
2483 tool->exit = perf_event__process_exit;
2484 tool->comm = perf_event__process_comm;
2485 tool->namespaces = perf_event__process_namespaces;
2486 tool->mmap = build_id__process_mmap;
2487 tool->mmap2 = build_id__process_mmap2;
2488 tool->itrace_start = process_timestamp_boundary;
2489 tool->aux = process_timestamp_boundary;
2490 tool->namespace_events = rec->opts.record_namespaces;
2491 tool->cgroup_events = rec->opts.record_cgroup;
2492 session = perf_session__new(data, tool);
2493 if (IS_ERR(session)) {
2494 pr_err("Perf session creation failed.\n");
2495 return PTR_ERR(session);
2496 }
2497 env = perf_session__env(session);
2498 if (record__threads_enabled(rec)) {
2499 if (perf_data__is_pipe(&rec->data)) {
2500 pr_err("Parallel trace streaming is not available in pipe mode.\n");
2501 return -1;
2502 }
2503 if (rec->opts.full_auxtrace) {
2504 pr_err("Parallel trace streaming is not available in AUX area tracing mode.\n");
2505 return -1;
2506 }
2507 }
2508
2509 fd = perf_data__fd(data);
2510 rec->session = session;
2511
2512 if (zstd_init(&session->zstd_data, rec->opts.comp_level) < 0) {
2513 pr_err("Compression initialization failed.\n");
2514 return -1;
2515 }
2516#ifdef HAVE_EVENTFD_SUPPORT
2517 done_fd = eventfd(0, EFD_NONBLOCK);
2518 if (done_fd < 0) {
2519 pr_err("Failed to create wakeup eventfd, error: %m\n");
2520 status = -1;
2521 goto out_delete_session;
2522 }
2523 err = evlist__add_wakeup_eventfd(rec->evlist, done_fd);
2524 if (err < 0) {
2525 pr_err("Failed to add wakeup eventfd to poll list\n");
2526 status = err;
2527 goto out_delete_session;
2528 }
2529#endif // HAVE_EVENTFD_SUPPORT
2530
2531 env->comp_type = PERF_COMP_ZSTD;
2532 env->comp_level = rec->opts.comp_level;
2533
2534 if (rec->opts.kcore &&
2535 !record__kcore_readable(&session->machines.host)) {
2536 pr_err("ERROR: kcore is not readable.\n");
2537 return -1;
2538 }
2539
2540 if (record__init_clock(rec))
2541 return -1;
2542
2543 record__init_features(rec);
2544
2545 if (forks) {
2546 err = evlist__prepare_workload(rec->evlist, &opts->target, argv, data->is_pipe,
2547 workload_exec_failed_signal);
2548 if (err < 0) {
2549 pr_err("Couldn't run the workload!\n");
2550 status = err;
2551 goto out_delete_session;
2552 }
2553 }
2554
2555 /*
2556 * If we have just single event and are sending data
2557 * through pipe, we need to force the ids allocation,
2558 * because we synthesize event name through the pipe
2559 * and need the id for that.
2560 */
2561 if (data->is_pipe && evlist__nr_entries(rec->evlist) == 1)
2562 rec->opts.sample_id = true;
2563
2564 if (rec->timestamp_filename && perf_data__is_pipe(data)) {
2565 rec->timestamp_filename = false;
2566 pr_warning("WARNING: --timestamp-filename option is not available in pipe mode.\n");
2567 }
2568
2569 /*
2570 * Use global stat_config that is zero meaning aggr_mode is AGGR_NONE
2571 * and hybrid_merge is false.
2572 */
2573 evlist__uniquify_evsel_names(rec->evlist, &stat_config);
2574
2575 evlist__config(rec->evlist, opts, &callchain_param);
2576
2577 /* Debug message used by test scripts */
2578 pr_debug3("perf record opening and mmapping events\n");
2579 if (record__open(rec) != 0) {
2580 err = -1;
2581 goto out_free_threads;
2582 }
2583 /* Debug message used by test scripts */
2584 pr_debug3("perf record done opening and mmapping events\n");
2585 env->comp_mmap_len = evlist__core(session->evlist)->mmap_len;
2586
2587 if (rec->opts.kcore) {
2588 err = record__kcore_copy(&session->machines.host, data);
2589 if (err) {
2590 pr_err("ERROR: Failed to copy kcore\n");
2591 goto out_free_threads;
2592 }
2593 }
2594
2595 /*
2596 * Normally perf_session__new would do this, but it doesn't have the
2597 * evlist.
2598 */
2599 if (rec->tool.ordered_events && !evlist__sample_id_all(rec->evlist)) {
2600 pr_warning("WARNING: No sample_id_all support, falling back to unordered processing\n");
2601 rec->tool.ordered_events = false;
2602 }
2603
2604 if (evlist__nr_groups(rec->evlist) == 0)
2605 perf_header__clear_feat(&session->header, HEADER_GROUP_DESC);
2606
2607 if (data->is_pipe) {
2608 err = perf_header__write_pipe(fd);
2609 if (err < 0)
2610 goto out_free_threads;
2611 } else {
2612 err = perf_session__write_header(session, rec->evlist, fd, false);
2613 if (err < 0)
2614 goto out_free_threads;
2615 }
2616
2617 err = -1;
2618 if (!rec->no_buildid
2619 && !perf_header__has_feat(&session->header, HEADER_BUILD_ID)) {
2620 pr_err("Couldn't generate buildids. "
2621 "Use --no-buildid to profile anyway.\n");
2622 goto out_free_threads;
2623 }
2624
2625 if (!evlist__needs_bpf_sb_event(rec->evlist))
2626 opts->no_bpf_event = true;
2627
2628 err = record__setup_sb_evlist(rec);
2629 if (err)
2630 goto out_free_threads;
2631
2632 err = record__synthesize(rec, false);
2633 if (err < 0)
2634 goto out_free_threads;
2635
2636 if (rec->realtime_prio) {
2637 struct sched_param param;
2638
2639 param.sched_priority = rec->realtime_prio;
2640 if (sched_setscheduler(0, SCHED_FIFO, &param)) {
2641 pr_err("Could not set realtime priority.\n");
2642 err = -1;
2643 goto out_free_threads;
2644 }
2645 }
2646
2647 if (record__start_threads(rec))
2648 goto out_free_threads;
2649
2650 /*
2651 * When perf is starting the traced process, all the events
2652 * (apart from group members) have enable_on_exec=1 set,
2653 * so don't spoil it by prematurely enabling them.
2654 */
2655 if (!target__none(&opts->target) && !opts->target.initial_delay)
2656 evlist__enable(rec->evlist);
2657
2658 /*
2659 * offcpu-time does not call execve, so enable_on_exe wouldn't work
2660 * when recording a workload, do it manually
2661 */
2662 if (rec->off_cpu)
2663 evlist__enable_evsel(rec->evlist, (char *)OFFCPU_EVENT);
2664
2665 /*
2666 * Let the child rip
2667 */
2668 if (forks) {
2669 struct machine *machine = &session->machines.host;
2670 union perf_event *event;
2671 pid_t tgid;
2672
2673 event = malloc(sizeof(event->comm) + machine->id_hdr_size);
2674 if (event == NULL) {
2675 err = -ENOMEM;
2676 goto out_child;
2677 }
2678
2679 /*
2680 * Some H/W events are generated before COMM event
2681 * which is emitted during exec(), so perf script
2682 * cannot see a correct process name for those events.
2683 * Synthesize COMM event to prevent it.
2684 */
2685 tgid = perf_event__synthesize_comm(tool, event,
2686 evlist__workload_pid(rec->evlist),
2687 process_synthesized_event,
2688 machine);
2689 free(event);
2690
2691 if (tgid == -1)
2692 goto out_child;
2693
2694 event = malloc(sizeof(event->namespaces) +
2695 (NR_NAMESPACES * sizeof(struct perf_ns_link_info)) +
2696 machine->id_hdr_size);
2697 if (event == NULL) {
2698 err = -ENOMEM;
2699 goto out_child;
2700 }
2701
2702 /*
2703 * Synthesize NAMESPACES event for the command specified.
2704 */
2705 perf_event__synthesize_namespaces(tool, event,
2706 evlist__workload_pid(rec->evlist),
2707 tgid, process_synthesized_event,
2708 machine);
2709 free(event);
2710
2711 evlist__start_workload(rec->evlist);
2712 }
2713
2714 if (opts->target.initial_delay) {
2715 pr_info(EVLIST_DISABLED_MSG);
2716 if (opts->target.initial_delay > 0) {
2717 usleep(opts->target.initial_delay * USEC_PER_MSEC);
2718 evlist__enable(rec->evlist);
2719 pr_info(EVLIST_ENABLED_MSG);
2720 }
2721 }
2722
2723 err = event_enable_timer__start(evlist__event_enable_timer(rec->evlist));
2724 if (err)
2725 goto out_child;
2726
2727 /* Debug message used by test scripts */
2728 pr_debug3("perf record has started\n");
2729 fflush(stderr);
2730
2731 trigger_ready(&auxtrace_snapshot_trigger);
2732 trigger_ready(&switch_output_trigger);
2733 perf_hooks__invoke_record_start();
2734
2735 /*
2736 * Must write FINISHED_INIT so it will be seen after all other
2737 * synthesized user events, but before any regular events.
2738 */
2739 err = write_finished_init(rec, false);
2740 if (err < 0)
2741 goto out_child;
2742
2743 for (;;) {
2744 unsigned long long hits = thread->samples;
2745
2746 /*
2747 * rec->evlist->bkw_mmap_state is possible to be
2748 * BKW_MMAP_EMPTY here: when done == true and
2749 * hits != rec->samples in previous round.
2750 *
2751 * evlist__toggle_bkw_mmap ensure we never
2752 * convert BKW_MMAP_EMPTY to BKW_MMAP_DATA_PENDING.
2753 */
2754 if (trigger_is_hit(&switch_output_trigger) || done || draining)
2755 evlist__toggle_bkw_mmap(rec->evlist, BKW_MMAP_DATA_PENDING);
2756
2757 if (record__mmap_read_all(rec, false) < 0) {
2758 trigger_error(&auxtrace_snapshot_trigger);
2759 trigger_error(&switch_output_trigger);
2760 err = -1;
2761 goto out_child_no_flush;
2762 }
2763
2764 if (auxtrace_record__snapshot_started) {
2765 auxtrace_record__snapshot_started = 0;
2766 if (!trigger_is_error(&auxtrace_snapshot_trigger))
2767 record__read_auxtrace_snapshot(rec, false);
2768 if (trigger_is_error(&auxtrace_snapshot_trigger)) {
2769 pr_err("AUX area tracing snapshot failed\n");
2770 err = -1;
2771 goto out_child;
2772 }
2773 }
2774
2775 if (trigger_is_hit(&switch_output_trigger)) {
2776 /*
2777 * If switch_output_trigger is hit, the data in
2778 * overwritable ring buffer should have been collected,
2779 * so bkw_mmap_state should be set to BKW_MMAP_EMPTY.
2780 *
2781 * If SIGUSR2 raise after or during record__mmap_read_all(),
2782 * record__mmap_read_all() didn't collect data from
2783 * overwritable ring buffer. Read again.
2784 */
2785 if (evlist__bkw_mmap_state(rec->evlist) == BKW_MMAP_RUNNING)
2786 continue;
2787 trigger_ready(&switch_output_trigger);
2788
2789 /*
2790 * Reenable events in overwrite ring buffer after
2791 * record__mmap_read_all(): we should have collected
2792 * data from it.
2793 */
2794 evlist__toggle_bkw_mmap(rec->evlist, BKW_MMAP_RUNNING);
2795
2796 if (!quiet)
2797 fprintf(stderr, "[ perf record: dump data: Woken up %ld times ]\n",
2798 record__waking(rec));
2799 thread->waking = 0;
2800 fd = record__switch_output(rec, false);
2801 if (fd < 0) {
2802 pr_err("Failed to switch to new file\n");
2803 trigger_error(&switch_output_trigger);
2804 err = fd;
2805 goto out_child;
2806 }
2807
2808 /* re-arm the alarm */
2809 if (rec->switch_output.time)
2810 alarm(rec->switch_output.time);
2811 }
2812
2813 if (hits == thread->samples) {
2814 if (done || draining)
2815 break;
2816 err = fdarray__poll(&thread->pollfd, -1);
2817 /*
2818 * Propagate error, only if there's any. Ignore positive
2819 * number of returned events and interrupt error.
2820 */
2821 if (err > 0 || (err < 0 && errno == EINTR))
2822 err = 0;
2823 thread->waking++;
2824
2825 if (fdarray__filter(&thread->pollfd, POLLERR | POLLHUP,
2826 record__thread_munmap_filtered, NULL) == 0)
2827 draining = true;
2828
2829 err = record__update_evlist_pollfd_from_thread(rec, rec->evlist, thread);
2830 if (err)
2831 goto out_child;
2832 }
2833
2834 if (evlist__ctlfd_process(rec->evlist, &cmd) > 0) {
2835 switch (cmd) {
2836 case EVLIST_CTL_CMD_SNAPSHOT:
2837 hit_auxtrace_snapshot_trigger(rec);
2838 evlist__ctlfd_ack(rec->evlist);
2839 break;
2840 case EVLIST_CTL_CMD_STOP:
2841 done = 1;
2842 break;
2843 case EVLIST_CTL_CMD_ACK:
2844 case EVLIST_CTL_CMD_UNSUPPORTED:
2845 case EVLIST_CTL_CMD_ENABLE:
2846 case EVLIST_CTL_CMD_DISABLE:
2847 case EVLIST_CTL_CMD_EVLIST:
2848 case EVLIST_CTL_CMD_PING:
2849 default:
2850 break;
2851 }
2852 }
2853
2854 err = event_enable_timer__process(evlist__event_enable_timer(rec->evlist));
2855 if (err < 0)
2856 goto out_child;
2857 if (err) {
2858 err = 0;
2859 done = 1;
2860 }
2861
2862 /*
2863 * When perf is starting the traced process, at the end events
2864 * die with the process and we wait for that. Thus no need to
2865 * disable events in this case.
2866 */
2867 if (done && !disabled && !target__none(&opts->target)) {
2868 trigger_off(&auxtrace_snapshot_trigger);
2869 evlist__disable(rec->evlist);
2870 disabled = true;
2871 }
2872 }
2873
2874 trigger_off(&auxtrace_snapshot_trigger);
2875 trigger_off(&switch_output_trigger);
2876
2877 record__synthesize_final_bpf_metadata(rec);
2878
2879 if (opts->auxtrace_snapshot_on_exit)
2880 record__auxtrace_snapshot_exit(rec);
2881
2882 if (forks && workload_exec_errno) {
2883 char msg[STRERR_BUFSIZE];
2884 const char *emsg = str_error_r(workload_exec_errno, msg, sizeof(msg));
2885 struct strbuf sb = STRBUF_INIT;
2886
2887 evlist__format_evsels(rec->evlist, &sb, 2048);
2888
2889 pr_err("Failed to collect '%s' for the '%s' workload: %s\n",
2890 sb.buf, argv[0], emsg);
2891 strbuf_release(&sb);
2892 err = -1;
2893 goto out_child;
2894 }
2895
2896 if (!quiet)
2897 fprintf(stderr, "[ perf record: Woken up %ld times to write data ]\n",
2898 record__waking(rec));
2899
2900 write_finished_init(rec, true);
2901
2902 if (target__none(&rec->opts.target))
2903 record__synthesize_workload(rec, true);
2904
2905out_child:
2906 evlist__disable(rec->evlist);
2907 record__stop_threads(rec);
2908 record__mmap_read_all(rec, true);
2909 goto out_free_threads;
2910out_child_no_flush:
2911 /* mmap read already failed — retrying would just fail again */
2912 evlist__disable(rec->evlist);
2913 record__stop_threads(rec);
2914out_free_threads:
2915 record__free_thread_data(rec);
2916 evlist__finalize_ctlfd(rec->evlist);
2917 record__aio_mmap_read_sync(rec);
2918
2919 if (rec->session->bytes_transferred && rec->session->bytes_compressed) {
2920 ratio = (float)rec->session->bytes_transferred/(float)rec->session->bytes_compressed;
2921 env->comp_ratio = ratio + 0.5;
2922 }
2923
2924 if (forks) {
2925 int exit_status;
2926
2927 if (!child_finished)
2928 kill(evlist__workload_pid(rec->evlist), SIGTERM);
2929
2930 wait(&exit_status);
2931
2932 if (err < 0)
2933 status = err;
2934 else if (WIFEXITED(exit_status))
2935 status = WEXITSTATUS(exit_status);
2936 else if (WIFSIGNALED(exit_status))
2937 signr = WTERMSIG(exit_status);
2938 } else
2939 status = err;
2940
2941 if (rec->off_cpu)
2942 rec->bytes_written += off_cpu_write(rec->session);
2943
2944 record__read_lost_samples(rec);
2945 /* this will be recalculated during process_buildids() */
2946 rec->samples = 0;
2947
2948 if (!err) {
2949 record__synthesize(rec, true);
2950 if (!rec->timestamp_filename) {
2951 record__finish_output(rec);
2952 } else {
2953 fd = record__switch_output(rec, true);
2954 if (fd < 0) {
2955 status = fd;
2956 goto out_delete_session;
2957 }
2958 }
2959 }
2960
2961 perf_hooks__invoke_record_end();
2962
2963 if (!err && !quiet) {
2964 char samples[128];
2965 const char *postfix = rec->timestamp_filename ?
2966 ".<timestamp>" : "";
2967
2968 if (rec->samples && !rec->opts.full_auxtrace)
2969 scnprintf(samples, sizeof(samples),
2970 " (%" PRIu64 " samples)", rec->samples);
2971 else
2972 samples[0] = '\0';
2973
2974 fprintf(stderr, "[ perf record: Captured and wrote %.3f MB %s%s%s",
2975 perf_data__size(data) / 1024.0 / 1024.0,
2976 data->path, postfix, samples);
2977 if (ratio) {
2978 fprintf(stderr, ", compressed (original %.3f MB, ratio is %.3f)",
2979 rec->session->bytes_transferred / 1024.0 / 1024.0,
2980 ratio);
2981 }
2982 fprintf(stderr, " ]\n");
2983 }
2984
2985out_delete_session:
2986#ifdef HAVE_EVENTFD_SUPPORT
2987 if (done_fd >= 0) {
2988 fd = done_fd;
2989 done_fd = -1;
2990
2991 close(fd);
2992 }
2993#endif
2994 zstd_fini(&session->zstd_data);
2995 if (!opts->no_bpf_event)
2996 evlist__stop_sb_thread(rec->sb_evlist);
2997
2998 perf_session__delete(session);
2999 return status;
3000}
3001
3002static int record_parse_callchain_opt(const struct option *opt,
3003 const char *arg,
3004 int unset)
3005{
3006 return record_opts__parse_callchain(opt->value, &callchain_param, arg, unset);
3007}
3008
3009static int record_callchain_opt(const struct option *opt,
3010 const char *arg __maybe_unused,
3011 int unset)
3012{
3013 /*
3014 * The -g option only sets the callchain if not already configured by
3015 * .perfconfig. It does, however, enable it.
3016 */
3017 if (callchain_param.record_mode != CALLCHAIN_NONE) {
3018 callchain_param.enabled = true;
3019 return 0;
3020 }
3021
3022 return record_opts__parse_callchain(opt->value, &callchain_param,
3023 EM_HOST != EM_S390 ? "fp" : "dwarf",
3024 unset);
3025}
3026
3027
3028static int perf_record_config(const char *var, const char *value, void *cb)
3029{
3030 struct record *rec = cb;
3031
3032 if (!strcmp(var, "record.build-id")) {
3033 if (!strcmp(value, "cache"))
3034 rec->no_buildid_cache = false;
3035 else if (!strcmp(value, "no-cache"))
3036 rec->no_buildid_cache = true;
3037 else if (!strcmp(value, "skip"))
3038 rec->no_buildid = rec->no_buildid_cache = true;
3039 else if (!strcmp(value, "mmap"))
3040 rec->buildid_mmap = true;
3041 else if (!strcmp(value, "no-mmap"))
3042 rec->buildid_mmap = false;
3043 else
3044 return -1;
3045 return 0;
3046 }
3047 if (!strcmp(var, "record.call-graph")) {
3048 var = "call-graph.record-mode";
3049 return perf_default_config(var, value, cb);
3050 }
3051#ifdef HAVE_AIO_SUPPORT
3052 if (!strcmp(var, "record.aio")) {
3053 rec->opts.nr_cblocks = strtol(value, NULL, 0);
3054 if (!rec->opts.nr_cblocks)
3055 rec->opts.nr_cblocks = nr_cblocks_default;
3056 }
3057#endif
3058 if (!strcmp(var, "record.debuginfod")) {
3059 rec->debuginfod.urls = strdup(value);
3060 if (!rec->debuginfod.urls)
3061 return -ENOMEM;
3062 rec->debuginfod.set = true;
3063 }
3064
3065 return 0;
3066}
3067
3068static int record__parse_event_enable_time(const struct option *opt, const char *str, int unset)
3069{
3070 struct record *rec = (struct record *)opt->value;
3071
3072 return evlist__parse_event_enable_time(rec->evlist, &rec->opts, str, unset);
3073}
3074
3075static int record__parse_affinity(const struct option *opt, const char *str, int unset)
3076{
3077 struct record_opts *opts = (struct record_opts *)opt->value;
3078
3079 if (unset || !str)
3080 return 0;
3081
3082 if (!strcasecmp(str, "node"))
3083 opts->affinity = PERF_AFFINITY_NODE;
3084 else if (!strcasecmp(str, "cpu"))
3085 opts->affinity = PERF_AFFINITY_CPU;
3086
3087 return 0;
3088}
3089
3090static int record__mmap_cpu_mask_alloc(struct mmap_cpu_mask *mask, int nr_bits)
3091{
3092 mask->nbits = nr_bits;
3093 mask->bits = bitmap_zalloc(mask->nbits);
3094 if (!mask->bits)
3095 return -ENOMEM;
3096
3097 return 0;
3098}
3099
3100static void record__mmap_cpu_mask_free(struct mmap_cpu_mask *mask)
3101{
3102 bitmap_free(mask->bits);
3103 mask->bits = NULL;
3104 mask->nbits = 0;
3105}
3106
3107static int record__thread_mask_alloc(struct thread_mask *mask, int nr_bits)
3108{
3109 int ret;
3110
3111 ret = record__mmap_cpu_mask_alloc(&mask->maps, nr_bits);
3112 if (ret) {
3113 mask->affinity.bits = NULL;
3114 return ret;
3115 }
3116
3117 ret = record__mmap_cpu_mask_alloc(&mask->affinity, nr_bits);
3118 if (ret) {
3119 record__mmap_cpu_mask_free(&mask->maps);
3120 mask->maps.bits = NULL;
3121 }
3122
3123 return ret;
3124}
3125
3126static void record__thread_mask_free(struct thread_mask *mask)
3127{
3128 record__mmap_cpu_mask_free(&mask->maps);
3129 record__mmap_cpu_mask_free(&mask->affinity);
3130}
3131
3132static int record__parse_threads(const struct option *opt, const char *str, int unset)
3133{
3134 int s;
3135 struct record_opts *opts = opt->value;
3136
3137 if (unset || !str || !strlen(str)) {
3138 opts->threads_spec = THREAD_SPEC__CPU;
3139 } else {
3140 for (s = 1; s < THREAD_SPEC__MAX; s++) {
3141 if (s == THREAD_SPEC__USER) {
3142 opts->threads_user_spec = strdup(str);
3143 if (!opts->threads_user_spec)
3144 return -ENOMEM;
3145 opts->threads_spec = THREAD_SPEC__USER;
3146 break;
3147 }
3148 if (!strncasecmp(str, thread_spec_tags[s], strlen(thread_spec_tags[s]))) {
3149 opts->threads_spec = s;
3150 break;
3151 }
3152 }
3153 }
3154
3155 if (opts->threads_spec == THREAD_SPEC__USER)
3156 pr_debug("threads_spec: %s\n", opts->threads_user_spec);
3157 else
3158 pr_debug("threads_spec: %s\n", thread_spec_tags[opts->threads_spec]);
3159
3160 return 0;
3161}
3162
3163static int parse_output_max_size(const struct option *opt,
3164 const char *str, int unset)
3165{
3166 unsigned long *s = (unsigned long *)opt->value;
3167 static struct parse_tag tags_size[] = {
3168 { .tag = 'B', .mult = 1 },
3169 { .tag = 'K', .mult = 1 << 10 },
3170 { .tag = 'M', .mult = 1 << 20 },
3171 { .tag = 'G', .mult = 1 << 30 },
3172 { .tag = 0 },
3173 };
3174 unsigned long val;
3175
3176 if (unset) {
3177 *s = 0;
3178 return 0;
3179 }
3180
3181 val = parse_tag_value(str, tags_size);
3182 if (val != (unsigned long) -1) {
3183 *s = val;
3184 return 0;
3185 }
3186
3187 return -1;
3188}
3189
3190static int record__parse_mmap_pages(const struct option *opt,
3191 const char *str,
3192 int unset __maybe_unused)
3193{
3194 struct record_opts *opts = opt->value;
3195 char *s, *p;
3196 unsigned int mmap_pages;
3197 int ret;
3198
3199 if (!str)
3200 return -EINVAL;
3201
3202 s = strdup(str);
3203 if (!s)
3204 return -ENOMEM;
3205
3206 p = strchr(s, ',');
3207 if (p)
3208 *p = '\0';
3209
3210 if (*s) {
3211 ret = __evlist__parse_mmap_pages(&mmap_pages, s);
3212 if (ret)
3213 goto out_free;
3214 opts->mmap_pages = mmap_pages;
3215 }
3216
3217 if (!p) {
3218 ret = 0;
3219 goto out_free;
3220 }
3221
3222 ret = __evlist__parse_mmap_pages(&mmap_pages, p + 1);
3223 if (ret)
3224 goto out_free;
3225
3226 opts->auxtrace_mmap_pages = mmap_pages;
3227
3228out_free:
3229 free(s);
3230 return ret;
3231}
3232
3233static int record__parse_off_cpu_thresh(const struct option *opt,
3234 const char *str,
3235 int unset __maybe_unused)
3236{
3237 struct record_opts *opts = opt->value;
3238 char *endptr;
3239 u64 off_cpu_thresh_ms;
3240
3241 if (!str)
3242 return -EINVAL;
3243
3244 off_cpu_thresh_ms = strtoull(str, &endptr, 10);
3245
3246 /* the threshold isn't string "0", yet strtoull() returns 0, parsing failed */
3247 if (*endptr || (off_cpu_thresh_ms == 0 && strcmp(str, "0")))
3248 return -EINVAL;
3249 else
3250 opts->off_cpu_thresh_ns = off_cpu_thresh_ms * NSEC_PER_MSEC;
3251
3252 return 0;
3253}
3254
3255static int parse_control_option(const struct option *opt,
3256 const char *str,
3257 int unset __maybe_unused)
3258{
3259 struct record_opts *opts = opt->value;
3260
3261 return evlist__parse_control(str, &opts->ctl_fd, &opts->ctl_fd_ack, &opts->ctl_fd_close);
3262}
3263
3264static void switch_output_size_warn(struct record *rec)
3265{
3266 u64 wakeup_size = evlist__mmap_size(rec->opts.mmap_pages);
3267 struct switch_output *s = &rec->switch_output;
3268
3269 wakeup_size /= 2;
3270
3271 if (s->size < wakeup_size) {
3272 char buf[100];
3273
3274 unit_number__scnprintf(buf, sizeof(buf), wakeup_size);
3275 pr_warning("WARNING: switch-output data size lower than "
3276 "wakeup kernel buffer size (%s) "
3277 "expect bigger perf.data sizes\n", buf);
3278 }
3279}
3280
3281static int switch_output_setup(struct record *rec)
3282{
3283 struct switch_output *s = &rec->switch_output;
3284 static struct parse_tag tags_size[] = {
3285 { .tag = 'B', .mult = 1 },
3286 { .tag = 'K', .mult = 1 << 10 },
3287 { .tag = 'M', .mult = 1 << 20 },
3288 { .tag = 'G', .mult = 1 << 30 },
3289 { .tag = 0 },
3290 };
3291 static struct parse_tag tags_time[] = {
3292 { .tag = 's', .mult = 1 },
3293 { .tag = 'm', .mult = 60 },
3294 { .tag = 'h', .mult = 60*60 },
3295 { .tag = 'd', .mult = 60*60*24 },
3296 { .tag = 0 },
3297 };
3298 unsigned long val;
3299
3300 /*
3301 * If we're using --switch-output-events, then we imply its
3302 * --switch-output=signal, as we'll send a SIGUSR2 from the side band
3303 * thread to its parent.
3304 */
3305 if (rec->switch_output_event_set) {
3306 if (record__threads_enabled(rec)) {
3307 pr_warning("WARNING: --switch-output-event option is not available in parallel streaming mode.\n");
3308 return 0;
3309 }
3310 goto do_signal;
3311 }
3312
3313 if (!s->set)
3314 return 0;
3315
3316 if (record__threads_enabled(rec)) {
3317 pr_warning("WARNING: --switch-output option is not available in parallel streaming mode.\n");
3318 return 0;
3319 }
3320
3321 if (!strcmp(s->str, "signal")) {
3322do_signal:
3323 s->signal = true;
3324 pr_debug("switch-output with SIGUSR2 signal\n");
3325 goto enabled;
3326 }
3327
3328 val = parse_tag_value(s->str, tags_size);
3329 if (val != (unsigned long) -1) {
3330 s->size = val;
3331 pr_debug("switch-output with %s size threshold\n", s->str);
3332 goto enabled;
3333 }
3334
3335 val = parse_tag_value(s->str, tags_time);
3336 if (val != (unsigned long) -1) {
3337 s->time = val;
3338 pr_debug("switch-output with %s time threshold (%lu seconds)\n",
3339 s->str, s->time);
3340 goto enabled;
3341 }
3342
3343 return -1;
3344
3345enabled:
3346 rec->timestamp_filename = true;
3347 s->enabled = true;
3348
3349 if (s->size && !rec->opts.no_buffering)
3350 switch_output_size_warn(rec);
3351
3352 return 0;
3353}
3354
3355static const char * const __record_usage[] = {
3356 "perf record [<options>] [<command>]",
3357 "perf record [<options>] -- <command> [<options>]",
3358 NULL
3359};
3360const char * const *record_usage = __record_usage;
3361
3362static int build_id__process_mmap(const struct perf_tool *tool, union perf_event *event,
3363 struct perf_sample *sample, struct machine *machine)
3364{
3365 /*
3366 * We already have the kernel maps, put in place via perf_session__create_kernel_maps()
3367 * no need to add them twice.
3368 */
3369 if (!(event->header.misc & PERF_RECORD_MISC_USER))
3370 return 0;
3371 return perf_event__process_mmap(tool, event, sample, machine);
3372}
3373
3374static int build_id__process_mmap2(const struct perf_tool *tool, union perf_event *event,
3375 struct perf_sample *sample, struct machine *machine)
3376{
3377 /*
3378 * We already have the kernel maps, put in place via perf_session__create_kernel_maps()
3379 * no need to add them twice.
3380 */
3381 if (!(event->header.misc & PERF_RECORD_MISC_USER))
3382 return 0;
3383
3384 return perf_event__process_mmap2(tool, event, sample, machine);
3385}
3386
3387static int process_timestamp_boundary(const struct perf_tool *tool,
3388 union perf_event *event __maybe_unused,
3389 struct perf_sample *sample,
3390 struct machine *machine __maybe_unused)
3391{
3392 struct record *rec = container_of(tool, struct record, tool);
3393
3394 set_timestamp_boundary(rec, sample->time);
3395 return 0;
3396}
3397
3398static int parse_record_synth_option(const struct option *opt,
3399 const char *str,
3400 int unset __maybe_unused)
3401{
3402 struct record_opts *opts = opt->value;
3403 char *p = strdup(str);
3404
3405 if (p == NULL)
3406 return -1;
3407
3408 opts->synth = parse_synth_opt(p);
3409 free(p);
3410
3411 if (opts->synth < 0) {
3412 pr_err("Invalid synth option: %s\n", str);
3413 return -1;
3414 }
3415 return 0;
3416}
3417
3418/*
3419 * XXX Ideally would be local to cmd_record() and passed to a record__new
3420 * because we need to have access to it in record__exit, that is called
3421 * after cmd_record() exits, but since record_options need to be accessible to
3422 * builtin-script, leave it here.
3423 *
3424 * At least we don't ouch it in all the other functions here directly.
3425 *
3426 * Just say no to tons of global variables, sigh.
3427 */
3428static struct record record = {
3429 .opts = {
3430 .sample_time = true,
3431 .mmap_pages = UINT_MAX,
3432 .user_freq = UINT_MAX,
3433 .user_interval = ULLONG_MAX,
3434 .freq = 4000,
3435 .target = {
3436 .uses_mmap = true,
3437 .default_per_cpu = true,
3438 },
3439 .mmap_flush = MMAP_FLUSH_DEFAULT,
3440 .nr_threads_synthesize = 1,
3441 .ctl_fd = -1,
3442 .ctl_fd_ack = -1,
3443 .synth = PERF_SYNTH_ALL,
3444 .off_cpu_thresh_ns = OFFCPU_THRESH,
3445 },
3446 .buildid_mmap = true,
3447};
3448
3449const char record_callchain_help[] = CALLCHAIN_RECORD_HELP
3450 "\n\t\t\t\tDefault: fp";
3451
3452static bool dry_run;
3453
3454static struct parse_events_option_args parse_events_option_args = {
3455 .evlistp = &record.evlist,
3456};
3457
3458static struct parse_events_option_args switch_output_parse_events_option_args = {
3459 .evlistp = &record.sb_evlist,
3460};
3461
3462/*
3463 * XXX Will stay a global variable till we fix builtin-script.c to stop messing
3464 * with it and switch to use the library functions in perf_evlist that came
3465 * from builtin-record.c, i.e. use record_opts,
3466 * evlist__prepare_workload, etc instead of fork+exec'in 'perf record',
3467 * using pipes, etc.
3468 */
3469static struct option __record_options[] = {
3470 OPT_CALLBACK('e', "event", &parse_events_option_args, "event",
3471 "event selector. use 'perf list' to list available events",
3472 parse_events_option),
3473 OPT_CALLBACK(0, "filter", &record.evlist, "filter",
3474 "event filter", parse_filter),
3475 OPT_BOOLEAN(0, "latency", &record.latency,
3476 "Enable data collection for latency profiling.\n"
3477 "\t\t\t Use perf report --latency for latency-centric profile."),
3478 OPT_CALLBACK_NOOPT(0, "exclude-perf", &record.evlist,
3479 NULL, "don't record events from perf itself",
3480 exclude_perf),
3481 OPT_STRING('p', "pid", &record.opts.target.pid, "pid",
3482 "record events on existing process id"),
3483 OPT_STRING('t', "tid", &record.opts.target.tid, "tid",
3484 "record events on existing thread id"),
3485 OPT_INTEGER('r', "realtime", &record.realtime_prio,
3486 "collect data with this RT SCHED_FIFO priority"),
3487 OPT_BOOLEAN(0, "no-buffering", &record.opts.no_buffering,
3488 "collect data without buffering"),
3489 OPT_BOOLEAN('R', "raw-samples", &record.opts.raw_samples,
3490 "collect raw sample records from all opened counters"),
3491 OPT_BOOLEAN('a', "all-cpus", &record.opts.target.system_wide,
3492 "system-wide collection from all CPUs"),
3493 OPT_STRING('C', "cpu", &record.opts.target.cpu_list, "cpu",
3494 "list of cpus to monitor"),
3495 OPT_U64('c', "count", &record.opts.user_interval, "event period to sample"),
3496 OPT_STRING('o', "output", &record.data.path, "file",
3497 "output file name"),
3498 OPT_BOOLEAN_SET('i', "no-inherit", &record.opts.no_inherit,
3499 &record.opts.no_inherit_set,
3500 "child tasks do not inherit counters"),
3501 OPT_BOOLEAN(0, "tail-synthesize", &record.opts.tail_synthesize,
3502 "synthesize non-sample events at the end of output"),
3503 OPT_BOOLEAN(0, "overwrite", &record.opts.overwrite, "use overwrite mode"),
3504 OPT_BOOLEAN(0, "no-bpf-event", &record.opts.no_bpf_event, "do not record bpf events"),
3505 OPT_BOOLEAN(0, "strict-freq", &record.opts.strict_freq,
3506 "Fail if the specified frequency can't be used"),
3507 OPT_CALLBACK('F', "freq", &record.opts, "freq or 'max'",
3508 "profile at this frequency",
3509 record__parse_freq),
3510 OPT_CALLBACK('m', "mmap-pages", &record.opts, "pages[,pages]",
3511 "number of mmap data pages and AUX area tracing mmap pages",
3512 record__parse_mmap_pages),
3513 OPT_CALLBACK(0, "mmap-flush", &record.opts, "number",
3514 "Minimal number of bytes that is extracted from mmap data pages (default: 1)",
3515 record__mmap_flush_parse),
3516 OPT_CALLBACK_NOOPT('g', NULL, &record.opts,
3517 NULL, "enables call-graph recording" ,
3518 &record_callchain_opt),
3519 OPT_CALLBACK(0, "call-graph", &record.opts,
3520 "record_mode[,record_size]", record_callchain_help,
3521 &record_parse_callchain_opt),
3522 OPT_INCR('v', "verbose", &verbose,
3523 "be more verbose (show counter open errors, etc)"),
3524 OPT_BOOLEAN('q', "quiet", &quiet, "don't print any warnings or messages"),
3525 OPT_BOOLEAN('s', "stat", &record.opts.inherit_stat,
3526 "per thread counts"),
3527 OPT_BOOLEAN('d', "data", &record.opts.sample_address, "Record the sample addresses"),
3528 OPT_BOOLEAN(0, "phys-data", &record.opts.sample_phys_addr,
3529 "Record the sample physical addresses"),
3530 OPT_BOOLEAN(0, "data-page-size", &record.opts.sample_data_page_size,
3531 "Record the sampled data address data page size"),
3532 OPT_BOOLEAN(0, "code-page-size", &record.opts.sample_code_page_size,
3533 "Record the sampled code address (ip) page size"),
3534 OPT_BOOLEAN(0, "sample-mem-info", &record.opts.sample_data_src,
3535 "Record the data source for memory operations"),
3536 OPT_BOOLEAN(0, "sample-cpu", &record.opts.sample_cpu, "Record the sample cpu"),
3537 OPT_BOOLEAN(0, "sample-identifier", &record.opts.sample_identifier,
3538 "Record the sample identifier"),
3539 OPT_BOOLEAN_SET('T', "timestamp", &record.opts.sample_time,
3540 &record.opts.sample_time_set,
3541 "Record the sample timestamps"),
3542 OPT_BOOLEAN_SET('P', "period", &record.opts.period, &record.opts.period_set,
3543 "Record the sample period"),
3544 OPT_BOOLEAN('n', "no-samples", &record.opts.no_samples,
3545 "don't sample"),
3546 OPT_BOOLEAN_SET('N', "no-buildid-cache", &record.no_buildid_cache,
3547 &record.no_buildid_cache_set,
3548 "do not update the buildid cache"),
3549 OPT_BOOLEAN_SET('B', "no-buildid", &record.no_buildid,
3550 &record.no_buildid_set,
3551 "do not collect buildids in perf.data"),
3552 OPT_CALLBACK('G', "cgroup", &record.evlist, "name",
3553 "monitor event in cgroup name only",
3554 parse_cgroups),
3555 OPT_CALLBACK('D', "delay", &record, "ms",
3556 "ms to wait before starting measurement after program start (-1: start with events disabled), "
3557 "or ranges of time to enable events e.g. '-D 10-20,30-40'",
3558 record__parse_event_enable_time),
3559 OPT_BOOLEAN(0, "kcore", &record.opts.kcore, "copy /proc/kcore"),
3560 OPT_STRING('u', "uid", &record.uid_str, "user", "user to profile"),
3561
3562 OPT_CALLBACK_NOOPT('b', "branch-any", &record.opts.branch_stack,
3563 "branch any", "sample any taken branches",
3564 parse_branch_stack),
3565
3566 OPT_CALLBACK('j', "branch-filter", &record.opts.branch_stack,
3567 "branch filter mask", "branch stack filter modes",
3568 parse_branch_stack),
3569 OPT_BOOLEAN('W', "weight", &record.opts.sample_weight,
3570 "sample by weight (on special events only)"),
3571 OPT_BOOLEAN(0, "transaction", &record.opts.sample_transaction,
3572 "sample transaction flags (special events only)"),
3573 OPT_BOOLEAN(0, "per-thread", &record.opts.target.per_thread,
3574 "use per-thread mmaps"),
3575 OPT_CALLBACK_OPTARG('I', "intr-regs", &record.opts.sample_intr_regs, NULL, "any register",
3576 "sample selected machine registers on interrupt,"
3577 " use '-I?' to list register names", parse_intr_regs),
3578 OPT_CALLBACK_OPTARG(0, "user-regs", &record.opts.sample_user_regs, NULL, "any register",
3579 "sample selected machine registers in user space,"
3580 " use '--user-regs=?' to list register names", parse_user_regs),
3581 OPT_BOOLEAN(0, "running-time", &record.opts.running_time,
3582 "Record running/enabled time of read (:S) events"),
3583 OPT_CALLBACK('k', "clockid", &record.opts,
3584 "clockid", "clockid to use for events, see clock_gettime()",
3585 parse_clockid),
3586 OPT_STRING_OPTARG('S', "snapshot", &record.opts.auxtrace_snapshot_opts,
3587 "opts", "AUX area tracing Snapshot Mode", ""),
3588 OPT_STRING_OPTARG(0, "aux-sample", &record.opts.auxtrace_sample_opts,
3589 "opts", "sample AUX area", ""),
3590 OPT_UINTEGER(0, "proc-map-timeout", &proc_map_timeout,
3591 "per thread proc mmap processing timeout in ms"),
3592 OPT_BOOLEAN(0, "namespaces", &record.opts.record_namespaces,
3593 "Record namespaces events"),
3594 OPT_BOOLEAN(0, "all-cgroups", &record.opts.record_cgroup,
3595 "Record cgroup events"),
3596 OPT_BOOLEAN_SET(0, "switch-events", &record.opts.record_switch_events,
3597 &record.opts.record_switch_events_set,
3598 "Record context switch events"),
3599 OPT_BOOLEAN_FLAG(0, "all-kernel", &record.opts.all_kernel,
3600 "Configure all used events to run in kernel space.",
3601 PARSE_OPT_EXCLUSIVE),
3602 OPT_BOOLEAN_FLAG(0, "all-user", &record.opts.all_user,
3603 "Configure all used events to run in user space.",
3604 PARSE_OPT_EXCLUSIVE),
3605 OPT_BOOLEAN(0, "kernel-callchains", &record.opts.kernel_callchains,
3606 "collect kernel callchains"),
3607 OPT_BOOLEAN(0, "user-callchains", &record.opts.user_callchains,
3608 "collect user callchains"),
3609 OPT_STRING(0, "vmlinux", &symbol_conf.vmlinux_name,
3610 "file", "vmlinux pathname"),
3611 OPT_BOOLEAN(0, "buildid-all", &record.buildid_all,
3612 "Record build-id of all DSOs regardless of hits"),
3613 OPT_BOOLEAN_SET(0, "buildid-mmap", &record.buildid_mmap, &record.buildid_mmap_set,
3614 "Record build-id in mmap events and skip build-id processing."),
3615 OPT_BOOLEAN(0, "timestamp-filename", &record.timestamp_filename,
3616 "append timestamp to output filename"),
3617 OPT_BOOLEAN(0, "timestamp-boundary", &record.timestamp_boundary,
3618 "Record timestamp boundary (time of first/last samples)"),
3619 OPT_STRING_OPTARG_SET(0, "switch-output", &record.switch_output.str,
3620 &record.switch_output.set, "signal or size[BKMG] or time[smhd]",
3621 "Switch output when receiving SIGUSR2 (signal) or cross a size or time threshold",
3622 "signal"),
3623 OPT_CALLBACK_SET(0, "switch-output-event", &switch_output_parse_events_option_args,
3624 &record.switch_output_event_set, "switch output event",
3625 "switch output event selector. use 'perf list' to list available events",
3626 parse_events_option_new_evlist),
3627 OPT_INTEGER(0, "switch-max-files", &record.switch_output.num_files,
3628 "Limit number of switch output generated files"),
3629 OPT_BOOLEAN(0, "dry-run", &dry_run,
3630 "Parse options then exit"),
3631#ifdef HAVE_AIO_SUPPORT
3632 OPT_CALLBACK_OPTARG(0, "aio", &record.opts,
3633 &nr_cblocks_default, "n", "Use <n> control blocks in asynchronous trace writing mode (default: 1, max: 4)",
3634 record__aio_parse),
3635#endif
3636 OPT_CALLBACK(0, "affinity", &record.opts, "node|cpu",
3637 "Set affinity mask of trace reading thread to NUMA node cpu mask or cpu of processed mmap buffer",
3638 record__parse_affinity),
3639#ifdef HAVE_ZSTD_SUPPORT
3640 OPT_CALLBACK_OPTARG('z', "compression-level", &record.opts, &comp_level_default, "n",
3641 "Compress records using specified level (default: 1 - fastest compression, 22 - greatest compression)",
3642 record__parse_comp_level),
3643#endif
3644 OPT_CALLBACK(0, "max-size", &record.output_max_size,
3645 "size", "Limit the maximum size of the output file", parse_output_max_size),
3646 OPT_UINTEGER(0, "num-thread-synthesize",
3647 &record.opts.nr_threads_synthesize,
3648 "number of threads to run for event synthesis"),
3649#ifdef HAVE_LIBPFM
3650 OPT_CALLBACK(0, "pfm-events", &record.evlist, "event",
3651 "libpfm4 event selector. use 'perf list' to list available events",
3652 parse_libpfm_events_option),
3653#endif
3654 OPT_CALLBACK(0, "control", &record.opts, "fd:ctl-fd[,ack-fd] or fifo:ctl-fifo[,ack-fifo]",
3655 "Listen on ctl-fd descriptor for command to control measurement ('enable': enable events, 'disable': disable events,\n"
3656 "\t\t\t 'snapshot': AUX area tracing snapshot).\n"
3657 "\t\t\t Optionally send control command completion ('ack\\n') to ack-fd descriptor.\n"
3658 "\t\t\t Alternatively, ctl-fifo / ack-fifo will be opened and used as ctl-fd / ack-fd.",
3659 parse_control_option),
3660 OPT_CALLBACK(0, "synth", &record.opts, "no|all|task|mmap|cgroup",
3661 "Fine-tune event synthesis: default=all", parse_record_synth_option),
3662 OPT_STRING_OPTARG_SET(0, "debuginfod", &record.debuginfod.urls,
3663 &record.debuginfod.set, "debuginfod urls",
3664 "Enable debuginfod data retrieval from DEBUGINFOD_URLS or specified urls",
3665 "system"),
3666 OPT_CALLBACK_OPTARG(0, "threads", &record.opts, NULL, "spec",
3667 "write collected trace data into several data files using parallel threads",
3668 record__parse_threads),
3669 OPT_BOOLEAN(0, "off-cpu", &record.off_cpu, "Enable off-cpu analysis"),
3670 OPT_STRING(0, "setup-filter", &record.filter_action, "pin|unpin",
3671 "BPF filter action"),
3672 OPT_CALLBACK(0, "off-cpu-thresh", &record.opts, "ms",
3673 "Dump off-cpu samples if off-cpu time exceeds this threshold (in milliseconds). (Default: 500ms)",
3674 record__parse_off_cpu_thresh),
3675 OPT_BOOLEAN_SET(0, "data-mmap", &record.opts.record_data_mmap,
3676 &record.opts.record_data_mmap_set,
3677 "Record mmap events for non-executable mappings"),
3678 OPT_END()
3679};
3680
3681struct option *record_options = __record_options;
3682
3683static int record__mmap_cpu_mask_init(struct mmap_cpu_mask *mask, struct perf_cpu_map *cpus)
3684{
3685 struct perf_cpu cpu;
3686 unsigned int idx;
3687
3688 if (cpu_map__is_dummy(cpus))
3689 return 0;
3690
3691 perf_cpu_map__for_each_cpu_skip_any(cpu, idx, cpus) {
3692 /* Return ENODEV is input cpu is greater than max cpu */
3693 if ((unsigned long)cpu.cpu > mask->nbits)
3694 return -ENODEV;
3695 __set_bit(cpu.cpu, mask->bits);
3696 }
3697
3698 return 0;
3699}
3700
3701static int record__mmap_cpu_mask_init_spec(struct mmap_cpu_mask *mask, const char *mask_spec)
3702{
3703 struct perf_cpu_map *cpus;
3704
3705 cpus = perf_cpu_map__new(mask_spec);
3706 if (!cpus)
3707 return -ENOMEM;
3708
3709 bitmap_zero(mask->bits, mask->nbits);
3710 if (record__mmap_cpu_mask_init(mask, cpus))
3711 return -ENODEV;
3712
3713 perf_cpu_map__put(cpus);
3714
3715 return 0;
3716}
3717
3718static void record__free_thread_masks(struct record *rec, int nr_threads)
3719{
3720 int t;
3721
3722 if (rec->thread_masks)
3723 for (t = 0; t < nr_threads; t++)
3724 record__thread_mask_free(&rec->thread_masks[t]);
3725
3726 zfree(&rec->thread_masks);
3727}
3728
3729static int record__alloc_thread_masks(struct record *rec, int nr_threads, int nr_bits)
3730{
3731 int t, ret;
3732
3733 rec->thread_masks = calloc(nr_threads, sizeof(*(rec->thread_masks)));
3734 if (!rec->thread_masks) {
3735 pr_err("Failed to allocate thread masks\n");
3736 return -ENOMEM;
3737 }
3738
3739 for (t = 0; t < nr_threads; t++) {
3740 ret = record__thread_mask_alloc(&rec->thread_masks[t], nr_bits);
3741 if (ret) {
3742 pr_err("Failed to allocate thread masks[%d]\n", t);
3743 goto out_free;
3744 }
3745 }
3746
3747 return 0;
3748
3749out_free:
3750 record__free_thread_masks(rec, nr_threads);
3751
3752 return ret;
3753}
3754
3755static int record__init_thread_cpu_masks(struct record *rec, struct perf_cpu_map *cpus)
3756{
3757 int t, ret, nr_cpus = perf_cpu_map__nr(cpus);
3758
3759 ret = record__alloc_thread_masks(rec, nr_cpus, cpu__max_cpu().cpu);
3760 if (ret)
3761 return ret;
3762
3763 rec->nr_threads = nr_cpus;
3764 pr_debug("nr_threads: %d\n", rec->nr_threads);
3765
3766 for (t = 0; t < rec->nr_threads; t++) {
3767 __set_bit(perf_cpu_map__cpu(cpus, t).cpu, rec->thread_masks[t].maps.bits);
3768 __set_bit(perf_cpu_map__cpu(cpus, t).cpu, rec->thread_masks[t].affinity.bits);
3769 if (verbose > 0) {
3770 pr_debug("thread_masks[%d]: ", t);
3771 mmap_cpu_mask__scnprintf(&rec->thread_masks[t].maps, "maps");
3772 pr_debug("thread_masks[%d]: ", t);
3773 mmap_cpu_mask__scnprintf(&rec->thread_masks[t].affinity, "affinity");
3774 }
3775 }
3776
3777 return 0;
3778}
3779
3780static int record__init_thread_masks_spec(struct record *rec, struct perf_cpu_map *cpus,
3781 const char **maps_spec, const char **affinity_spec,
3782 u32 nr_spec)
3783{
3784 u32 s;
3785 int ret = 0, t = 0;
3786 struct mmap_cpu_mask cpus_mask;
3787 struct thread_mask thread_mask, full_mask, *thread_masks;
3788
3789 ret = record__mmap_cpu_mask_alloc(&cpus_mask, cpu__max_cpu().cpu);
3790 if (ret) {
3791 pr_err("Failed to allocate CPUs mask\n");
3792 return ret;
3793 }
3794
3795 ret = record__mmap_cpu_mask_init(&cpus_mask, cpus);
3796 if (ret) {
3797 pr_err("Failed to init cpu mask\n");
3798 goto out_free_cpu_mask;
3799 }
3800
3801 ret = record__thread_mask_alloc(&full_mask, cpu__max_cpu().cpu);
3802 if (ret) {
3803 pr_err("Failed to allocate full mask\n");
3804 goto out_free_cpu_mask;
3805 }
3806
3807 ret = record__thread_mask_alloc(&thread_mask, cpu__max_cpu().cpu);
3808 if (ret) {
3809 pr_err("Failed to allocate thread mask\n");
3810 goto out_free_full_and_cpu_masks;
3811 }
3812
3813 for (s = 0; s < nr_spec; s++) {
3814 ret = record__mmap_cpu_mask_init_spec(&thread_mask.maps, maps_spec[s]);
3815 if (ret) {
3816 pr_err("Failed to initialize maps thread mask\n");
3817 goto out_free;
3818 }
3819 ret = record__mmap_cpu_mask_init_spec(&thread_mask.affinity, affinity_spec[s]);
3820 if (ret) {
3821 pr_err("Failed to initialize affinity thread mask\n");
3822 goto out_free;
3823 }
3824
3825 /* ignore invalid CPUs but do not allow empty masks */
3826 if (!bitmap_and(thread_mask.maps.bits, thread_mask.maps.bits,
3827 cpus_mask.bits, thread_mask.maps.nbits)) {
3828 pr_err("Empty maps mask: %s\n", maps_spec[s]);
3829 ret = -EINVAL;
3830 goto out_free;
3831 }
3832 if (!bitmap_and(thread_mask.affinity.bits, thread_mask.affinity.bits,
3833 cpus_mask.bits, thread_mask.affinity.nbits)) {
3834 pr_err("Empty affinity mask: %s\n", affinity_spec[s]);
3835 ret = -EINVAL;
3836 goto out_free;
3837 }
3838
3839 /* do not allow intersection with other masks (full_mask) */
3840 if (bitmap_intersects(thread_mask.maps.bits, full_mask.maps.bits,
3841 thread_mask.maps.nbits)) {
3842 pr_err("Intersecting maps mask: %s\n", maps_spec[s]);
3843 ret = -EINVAL;
3844 goto out_free;
3845 }
3846 if (bitmap_intersects(thread_mask.affinity.bits, full_mask.affinity.bits,
3847 thread_mask.affinity.nbits)) {
3848 pr_err("Intersecting affinity mask: %s\n", affinity_spec[s]);
3849 ret = -EINVAL;
3850 goto out_free;
3851 }
3852
3853 bitmap_or(full_mask.maps.bits, full_mask.maps.bits,
3854 thread_mask.maps.bits, full_mask.maps.nbits);
3855 bitmap_or(full_mask.affinity.bits, full_mask.affinity.bits,
3856 thread_mask.affinity.bits, full_mask.maps.nbits);
3857
3858 thread_masks = realloc(rec->thread_masks, (t + 1) * sizeof(struct thread_mask));
3859 if (!thread_masks) {
3860 pr_err("Failed to reallocate thread masks\n");
3861 ret = -ENOMEM;
3862 goto out_free;
3863 }
3864 rec->thread_masks = thread_masks;
3865 rec->thread_masks[t] = thread_mask;
3866 if (verbose > 0) {
3867 pr_debug("thread_masks[%d]: ", t);
3868 mmap_cpu_mask__scnprintf(&rec->thread_masks[t].maps, "maps");
3869 pr_debug("thread_masks[%d]: ", t);
3870 mmap_cpu_mask__scnprintf(&rec->thread_masks[t].affinity, "affinity");
3871 }
3872 t++;
3873 ret = record__thread_mask_alloc(&thread_mask, cpu__max_cpu().cpu);
3874 if (ret) {
3875 pr_err("Failed to allocate thread mask\n");
3876 goto out_free_full_and_cpu_masks;
3877 }
3878 }
3879 rec->nr_threads = t;
3880 pr_debug("nr_threads: %d\n", rec->nr_threads);
3881 if (!rec->nr_threads)
3882 ret = -EINVAL;
3883
3884out_free:
3885 record__thread_mask_free(&thread_mask);
3886out_free_full_and_cpu_masks:
3887 record__thread_mask_free(&full_mask);
3888out_free_cpu_mask:
3889 record__mmap_cpu_mask_free(&cpus_mask);
3890
3891 return ret;
3892}
3893
3894static int record__init_thread_core_masks(struct record *rec, struct perf_cpu_map *cpus)
3895{
3896 int ret;
3897 struct cpu_topology *topo;
3898
3899 topo = cpu_topology__new();
3900 if (!topo) {
3901 pr_err("Failed to allocate CPU topology\n");
3902 return -ENOMEM;
3903 }
3904
3905 ret = record__init_thread_masks_spec(rec, cpus, topo->core_cpus_list,
3906 topo->core_cpus_list, topo->core_cpus_lists);
3907 cpu_topology__delete(topo);
3908
3909 return ret;
3910}
3911
3912static int record__init_thread_package_masks(struct record *rec, struct perf_cpu_map *cpus)
3913{
3914 int ret;
3915 struct cpu_topology *topo;
3916
3917 topo = cpu_topology__new();
3918 if (!topo) {
3919 pr_err("Failed to allocate CPU topology\n");
3920 return -ENOMEM;
3921 }
3922
3923 ret = record__init_thread_masks_spec(rec, cpus, topo->package_cpus_list,
3924 topo->package_cpus_list, topo->package_cpus_lists);
3925 cpu_topology__delete(topo);
3926
3927 return ret;
3928}
3929
3930static int record__init_thread_numa_masks(struct record *rec, struct perf_cpu_map *cpus)
3931{
3932 u32 s;
3933 int ret;
3934 const char **spec;
3935 struct numa_topology *topo;
3936
3937 topo = numa_topology__new();
3938 if (!topo) {
3939 pr_err("Failed to allocate NUMA topology\n");
3940 return -ENOMEM;
3941 }
3942
3943 spec = calloc(topo->nr, sizeof(char *));
3944 if (!spec) {
3945 pr_err("Failed to allocate NUMA spec\n");
3946 ret = -ENOMEM;
3947 goto out_delete_topo;
3948 }
3949 for (s = 0; s < topo->nr; s++)
3950 spec[s] = topo->nodes[s].cpus;
3951
3952 ret = record__init_thread_masks_spec(rec, cpus, spec, spec, topo->nr);
3953
3954 zfree(&spec);
3955
3956out_delete_topo:
3957 numa_topology__delete(topo);
3958
3959 return ret;
3960}
3961
3962static int record__init_thread_user_masks(struct record *rec, struct perf_cpu_map *cpus)
3963{
3964 int t, ret;
3965 u32 s, nr_spec = 0;
3966 char **maps_spec = NULL, **affinity_spec = NULL, **tmp_spec;
3967 char *user_spec, *spec, *spec_ptr, *mask, *mask_ptr, *dup_mask = NULL;
3968
3969 for (t = 0, user_spec = (char *)rec->opts.threads_user_spec; ; t++, user_spec = NULL) {
3970 spec = strtok_r(user_spec, ":", &spec_ptr);
3971 if (spec == NULL)
3972 break;
3973 pr_debug2("threads_spec[%d]: %s\n", t, spec);
3974 mask = strtok_r(spec, "/", &mask_ptr);
3975 if (mask == NULL)
3976 break;
3977 pr_debug2(" maps mask: %s\n", mask);
3978 tmp_spec = realloc(maps_spec, (nr_spec + 1) * sizeof(char *));
3979 if (!tmp_spec) {
3980 pr_err("Failed to reallocate maps spec\n");
3981 ret = -ENOMEM;
3982 goto out_free;
3983 }
3984 maps_spec = tmp_spec;
3985 maps_spec[nr_spec] = dup_mask = strdup(mask);
3986 if (!maps_spec[nr_spec]) {
3987 pr_err("Failed to allocate maps spec[%d]\n", nr_spec);
3988 ret = -ENOMEM;
3989 goto out_free;
3990 }
3991 mask = strtok_r(NULL, "/", &mask_ptr);
3992 if (mask == NULL) {
3993 pr_err("Invalid thread maps or affinity specs\n");
3994 ret = -EINVAL;
3995 goto out_free;
3996 }
3997 pr_debug2(" affinity mask: %s\n", mask);
3998 tmp_spec = realloc(affinity_spec, (nr_spec + 1) * sizeof(char *));
3999 if (!tmp_spec) {
4000 pr_err("Failed to reallocate affinity spec\n");
4001 ret = -ENOMEM;
4002 goto out_free;
4003 }
4004 affinity_spec = tmp_spec;
4005 affinity_spec[nr_spec] = strdup(mask);
4006 if (!affinity_spec[nr_spec]) {
4007 pr_err("Failed to allocate affinity spec[%d]\n", nr_spec);
4008 ret = -ENOMEM;
4009 goto out_free;
4010 }
4011 dup_mask = NULL;
4012 nr_spec++;
4013 }
4014
4015 ret = record__init_thread_masks_spec(rec, cpus, (const char **)maps_spec,
4016 (const char **)affinity_spec, nr_spec);
4017
4018out_free:
4019 free(dup_mask);
4020 for (s = 0; s < nr_spec; s++) {
4021 if (maps_spec)
4022 free(maps_spec[s]);
4023 if (affinity_spec)
4024 free(affinity_spec[s]);
4025 }
4026 free(affinity_spec);
4027 free(maps_spec);
4028
4029 return ret;
4030}
4031
4032static int record__init_thread_default_masks(struct record *rec, struct perf_cpu_map *cpus)
4033{
4034 int ret;
4035
4036 ret = record__alloc_thread_masks(rec, 1, cpu__max_cpu().cpu);
4037 if (ret)
4038 return ret;
4039
4040 if (record__mmap_cpu_mask_init(&rec->thread_masks->maps, cpus))
4041 return -ENODEV;
4042
4043 rec->nr_threads = 1;
4044
4045 return 0;
4046}
4047
4048static int record__init_thread_masks(struct record *rec)
4049{
4050 int ret = 0;
4051 struct perf_cpu_map *cpus = evlist__core(rec->evlist)->all_cpus;
4052
4053 if (!record__threads_enabled(rec))
4054 return record__init_thread_default_masks(rec, cpus);
4055
4056 if (evlist__per_thread(rec->evlist)) {
4057 pr_err("--per-thread option is mutually exclusive to parallel streaming mode.\n");
4058 return -EINVAL;
4059 }
4060
4061 switch (rec->opts.threads_spec) {
4062 case THREAD_SPEC__CPU:
4063 ret = record__init_thread_cpu_masks(rec, cpus);
4064 break;
4065 case THREAD_SPEC__CORE:
4066 ret = record__init_thread_core_masks(rec, cpus);
4067 break;
4068 case THREAD_SPEC__PACKAGE:
4069 ret = record__init_thread_package_masks(rec, cpus);
4070 break;
4071 case THREAD_SPEC__NUMA:
4072 ret = record__init_thread_numa_masks(rec, cpus);
4073 break;
4074 case THREAD_SPEC__USER:
4075 ret = record__init_thread_user_masks(rec, cpus);
4076 break;
4077 default:
4078 break;
4079 }
4080
4081 return ret;
4082}
4083
4084int cmd_record(int argc, const char **argv)
4085{
4086 int err;
4087 struct record *rec = &record;
4088 char errbuf[BUFSIZ];
4089
4090 setlocale(LC_ALL, "");
4091
4092#ifndef HAVE_BPF_SKEL
4093# define set_nobuild(s, l, m, c) set_option_nobuild(record_options, s, l, m, c)
4094 set_nobuild('\0', "off-cpu", "no BUILD_BPF_SKEL=1", true);
4095# undef set_nobuild
4096#endif
4097
4098 /* Disable eager loading of kernel symbols that adds overhead to perf record. */
4099 symbol_conf.lazy_load_kernel_maps = true;
4100 rec->opts.affinity = PERF_AFFINITY_SYS;
4101
4102 rec->evlist = evlist__new();
4103 if (rec->evlist == NULL)
4104 return -ENOMEM;
4105
4106 err = perf_config(perf_record_config, rec);
4107 if (err)
4108 return err;
4109
4110 argc = parse_options(argc, argv, record_options, record_usage,
4111 PARSE_OPT_STOP_AT_NON_OPTION);
4112 if (quiet)
4113 perf_quiet_option();
4114
4115 err = symbol__validate_sym_arguments();
4116 if (err)
4117 return err;
4118
4119 perf_debuginfod_setup(&record.debuginfod);
4120
4121 /*
4122 * Use system wide (-a) for the default target (ie. when no
4123 * workload). User ID filtering also implies system-wide.
4124 */
4125 if ((!argc && target__none(&rec->opts.target)) || rec->uid_str)
4126 rec->opts.target.system_wide = true;
4127
4128 if (nr_cgroups && !rec->opts.target.system_wide) {
4129 usage_with_options_msg(record_usage, record_options,
4130 "cgroup monitoring only available in system-wide mode");
4131
4132 }
4133
4134 if (record.latency) {
4135 /*
4136 * There is no fundamental reason why latency profiling
4137 * can't work for system-wide mode, but exact semantics
4138 * and details are to be defined.
4139 * See the following thread for details:
4140 * https://lore.kernel.org/all/[email protected]/
4141 */
4142 if (record.opts.target.system_wide) {
4143 pr_err("Failed: latency profiling is not supported with system-wide collection.\n");
4144 err = -EINVAL;
4145 goto out_opts;
4146 }
4147 record.opts.record_switch_events = true;
4148 }
4149
4150 if (rec->buildid_mmap && !perf_can_record_build_id()) {
4151 pr_warning("Missing support for build id in kernel mmap events.\n"
4152 "Disable this warning with --no-buildid-mmap\n");
4153 rec->buildid_mmap = false;
4154 }
4155
4156 if (rec->buildid_mmap) {
4157 /* Enable perf_event_attr::build_id bit. */
4158 rec->opts.build_id = true;
4159 /* Disable build-ID table in the header. */
4160 rec->no_buildid = true;
4161 } else {
4162 pr_debug("Disabling build id in synthesized mmap2 events.\n");
4163 symbol_conf.no_buildid_mmap2 = true;
4164 }
4165
4166 if (rec->no_buildid_set && rec->no_buildid) {
4167 /* -B implies -N for historic reasons. */
4168 rec->no_buildid_cache = true;
4169 }
4170
4171 if (rec->opts.record_cgroup && !perf_can_record_cgroup()) {
4172 pr_err("Kernel has no cgroup sampling support.\n");
4173 err = -EINVAL;
4174 goto out_opts;
4175 }
4176
4177 if (rec->opts.kcore)
4178 rec->opts.text_poke = true;
4179
4180 if (rec->opts.kcore || record__threads_enabled(rec))
4181 rec->data.is_dir = true;
4182
4183 if (record__threads_enabled(rec)) {
4184 if (rec->opts.affinity != PERF_AFFINITY_SYS) {
4185 pr_err("--affinity option is mutually exclusive to parallel streaming mode.\n");
4186 goto out_opts;
4187 }
4188 if (record__aio_enabled(rec)) {
4189 pr_err("Asynchronous streaming mode (--aio) is mutually exclusive to parallel streaming mode.\n");
4190 goto out_opts;
4191 }
4192 }
4193
4194 if (rec->opts.comp_level != 0) {
4195 pr_debug("Compression enabled, disabling build id collection at the end of the session.\n");
4196 rec->no_buildid = true;
4197 }
4198
4199 if (rec->opts.record_switch_events &&
4200 !perf_can_record_switch_events()) {
4201 ui__error("kernel does not support recording context switch events\n");
4202 parse_options_usage(record_usage, record_options, "switch-events", 0);
4203 err = -EINVAL;
4204 goto out_opts;
4205 }
4206
4207 if (switch_output_setup(rec)) {
4208 parse_options_usage(record_usage, record_options, "switch-output", 0);
4209 err = -EINVAL;
4210 goto out_opts;
4211 }
4212
4213 if (rec->switch_output.time) {
4214 signal(SIGALRM, alarm_sig_handler);
4215 alarm(rec->switch_output.time);
4216 }
4217
4218 if (rec->switch_output.num_files) {
4219 rec->switch_output.filenames = calloc(rec->switch_output.num_files,
4220 sizeof(char *));
4221 if (!rec->switch_output.filenames) {
4222 err = -EINVAL;
4223 goto out_opts;
4224 }
4225 }
4226
4227 if (rec->timestamp_filename && record__threads_enabled(rec)) {
4228 rec->timestamp_filename = false;
4229 pr_warning("WARNING: --timestamp-filename option is not available in parallel streaming mode.\n");
4230 }
4231
4232 if (rec->filter_action) {
4233 if (!strcmp(rec->filter_action, "pin"))
4234 err = perf_bpf_filter__pin();
4235 else if (!strcmp(rec->filter_action, "unpin"))
4236 err = perf_bpf_filter__unpin();
4237 else {
4238 pr_warning("Unknown BPF filter action: %s\n", rec->filter_action);
4239 err = -EINVAL;
4240 }
4241 goto out_opts;
4242 }
4243
4244 /* For backward compatibility, -d implies --mem-info and --data-mmap */
4245 if (rec->opts.sample_address) {
4246 rec->opts.sample_data_src = true;
4247 if (!rec->opts.record_data_mmap_set)
4248 rec->opts.record_data_mmap = true;
4249 }
4250
4251 /*
4252 * Allow aliases to facilitate the lookup of symbols for address
4253 * filters. Refer to auxtrace_parse_filters().
4254 */
4255 symbol_conf.allow_aliases = true;
4256
4257 symbol__init(NULL);
4258
4259 err = record__auxtrace_init(rec);
4260 if (err)
4261 goto out;
4262
4263 if (dry_run)
4264 goto out;
4265
4266 err = -ENOMEM;
4267
4268 if (rec->no_buildid_cache) {
4269 disable_buildid_cache();
4270 } else if (rec->switch_output.enabled) {
4271 /*
4272 * In 'perf record --switch-output', disable buildid
4273 * generation by default to reduce data file switching
4274 * overhead. Still generate buildid if they are required
4275 * explicitly using
4276 *
4277 * perf record --switch-output --no-no-buildid \
4278 * --no-no-buildid-cache
4279 *
4280 * Following code equals to:
4281 *
4282 * if ((rec->no_buildid || !rec->no_buildid_set) &&
4283 * (rec->no_buildid_cache || !rec->no_buildid_cache_set))
4284 * disable_buildid_cache();
4285 */
4286 bool disable = true;
4287
4288 if (rec->no_buildid_set && !rec->no_buildid)
4289 disable = false;
4290 if (rec->no_buildid_cache_set && !rec->no_buildid_cache)
4291 disable = false;
4292 if (disable) {
4293 rec->no_buildid = true;
4294 rec->no_buildid_cache = true;
4295 disable_buildid_cache();
4296 }
4297 }
4298
4299 if (record.opts.overwrite)
4300 record.opts.tail_synthesize = true;
4301
4302 if (evlist__nr_entries(rec->evlist) == 0) {
4303 struct evlist *def_evlist = evlist__new_default(&rec->opts.target,
4304 callchain_param.enabled);
4305
4306 if (!def_evlist)
4307 goto out;
4308
4309 evlist__splice_list_tail(rec->evlist, &evlist__core(def_evlist)->entries);
4310 evlist__put(def_evlist);
4311 }
4312
4313 if (rec->opts.target.tid && !rec->opts.no_inherit_set)
4314 rec->opts.no_inherit = true;
4315
4316 err = target__validate(&rec->opts.target);
4317 if (err) {
4318 target__strerror(&rec->opts.target, err, errbuf, BUFSIZ);
4319 ui__warning("%s\n", errbuf);
4320 }
4321
4322 if (rec->uid_str) {
4323 uid_t uid = parse_uid(rec->uid_str);
4324
4325 if (uid == UINT_MAX) {
4326 ui__error("Invalid User: %s", rec->uid_str);
4327 err = -EINVAL;
4328 goto out;
4329 }
4330 err = parse_uid_filter(rec->evlist, uid);
4331 if (err)
4332 goto out;
4333 }
4334
4335 /* Enable ignoring missing threads when -p option is defined. */
4336 rec->opts.ignore_missing_thread = rec->opts.target.pid;
4337
4338 evlist__warn_user_requested_cpus(rec->evlist, rec->opts.target.cpu_list);
4339
4340 if (callchain_param.enabled && callchain_param.record_mode == CALLCHAIN_FP) {
4341 if (EM_HOST == EM_AARCH64)
4342 add_leaf_frame_caller_opts_aarch64(&rec->opts);
4343 }
4344
4345 err = -ENOMEM;
4346 if (evlist__create_maps(rec->evlist, &rec->opts.target) < 0) {
4347 if (rec->opts.target.pid != NULL) {
4348 pr_err("Couldn't create thread/CPU maps: %s\n",
4349 errno == ENOENT ? "No such process" : str_error_r(errno, errbuf, sizeof(errbuf)));
4350 goto out;
4351 }
4352 else
4353 usage_with_options(record_usage, record_options);
4354 }
4355
4356 err = auxtrace_record__options(rec->itr, rec->evlist, &rec->opts);
4357 if (err)
4358 goto out;
4359
4360 /*
4361 * We take all buildids when the file contains
4362 * AUX area tracing data because we do not decode the
4363 * trace because it would take too long.
4364 */
4365 if (rec->opts.full_auxtrace)
4366 rec->buildid_all = true;
4367
4368 if (rec->opts.text_poke) {
4369 err = record__config_text_poke(rec->evlist);
4370 if (err) {
4371 pr_err("record__config_text_poke failed, error %d\n", err);
4372 goto out;
4373 }
4374 }
4375
4376 if (rec->off_cpu) {
4377 err = record__config_off_cpu(rec);
4378 if (err) {
4379 pr_err("record__config_off_cpu failed, error %d\n", err);
4380 goto out;
4381 }
4382 }
4383
4384 if (record_opts__config(&rec->opts)) {
4385 err = -EINVAL;
4386 goto out;
4387 }
4388
4389 err = record__config_tracking_events(rec);
4390 if (err) {
4391 pr_err("record__config_tracking_events failed, error %d\n", err);
4392 goto out;
4393 }
4394
4395 err = record__init_thread_masks(rec);
4396 if (err) {
4397 pr_err("Failed to initialize parallel data streaming masks\n");
4398 goto out;
4399 }
4400
4401 if (rec->opts.nr_cblocks > nr_cblocks_max)
4402 rec->opts.nr_cblocks = nr_cblocks_max;
4403 pr_debug("nr_cblocks: %d\n", rec->opts.nr_cblocks);
4404
4405 pr_debug("affinity: %s\n", affinity_tags[rec->opts.affinity]);
4406 pr_debug("mmap flush: %d\n", rec->opts.mmap_flush);
4407
4408 if (rec->opts.comp_level > comp_level_max)
4409 rec->opts.comp_level = comp_level_max;
4410 pr_debug("comp level: %d\n", rec->opts.comp_level);
4411
4412 err = __cmd_record(&record, argc, argv);
4413out:
4414 record__free_thread_masks(rec, rec->nr_threads);
4415 rec->nr_threads = 0;
4416 symbol__exit();
4417 auxtrace_record__free(rec->itr);
4418out_opts:
4419 evlist__close_control(rec->opts.ctl_fd, rec->opts.ctl_fd_ack, &rec->opts.ctl_fd_close);
4420 evlist__put(rec->evlist);
4421 return err;
4422}
4423
4424static void snapshot_sig_handler(int sig __maybe_unused)
4425{
4426 struct record *rec = &record;
4427
4428 hit_auxtrace_snapshot_trigger(rec);
4429
4430 if (switch_output_signal(rec))
4431 trigger_hit(&switch_output_trigger);
4432}
4433
4434static void alarm_sig_handler(int sig __maybe_unused)
4435{
4436 struct record *rec = &record;
4437
4438 if (switch_output_time(rec))
4439 trigger_hit(&switch_output_trigger);
4440}