linux - tools/perf/builtin-stat.c

3103 lines · 85.0 KiB

1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* builtin-stat.c
4	*
5	* Builtin stat command: Give a precise performance counters summary
6	* overview about any workload, CPU or specific PID.
7	*
8	* Sample output:
9
10	$ perf stat ./hackbench 10
11
12	Time: 0.118
13
14	Performance counter stats for './hackbench 10':
15
16	1708.761321 task-clock # 11.037 CPUs utilized
17	41,190 context-switches # 0.024 M/sec
18	6,735 CPU-migrations # 0.004 M/sec
19	17,318 page-faults # 0.010 M/sec
20	5,205,202,243 cycles # 3.046 GHz
21	3,856,436,920 stalled-cycles-frontend # 74.09% frontend cycles idle
22	1,600,790,871 stalled-cycles-backend # 30.75% backend cycles idle
23	2,603,501,247 instructions # 0.50 insns per cycle
24	# 1.48 stalled cycles per insn
25	484,357,498 branches # 283.455 M/sec
26	6,388,934 branch-misses # 1.32% of all branches
27
28	0.154822978 seconds time elapsed
29
30	*
31	* Copyright (C) 2008-2011, Red Hat Inc, Ingo Molnar <[email protected]>
32	*
33	* Improvements and fixes by:
34	*
35	* Arjan van de Ven <[email protected]>
36	* Yanmin Zhang <[email protected]>
37	* Wu Fengguang <[email protected]>
38	* Mike Galbraith <[email protected]>
39	* Paul Mackerras <[email protected]>
40	* Jaswinder Singh Rajput <[email protected]>
41	*/
42
43	#include "builtin.h"
44	#include "util/cgroup.h"
45	#include <subcmd/parse-options.h>
46	#include "util/parse-events.h"
47	#include "util/pmus.h"
48	#include "util/pmu.h"
49	#include "util/tool_pmu.h"
50	#include "util/event.h"
51	#include "util/evlist.h"
52	#include "util/evsel.h"
53	#include "util/debug.h"
54	#include "util/color.h"
55	#include "util/stat.h"
56	#include "util/header.h"
57	#include "util/cpumap.h"
58	#include "util/thread_map.h"
59	#include "util/counts.h"
60	#include "util/topdown.h"
61	#include "util/session.h"
62	#include "util/tool.h"
63	#include "util/string2.h"
64	#include "util/metricgroup.h"
65	#include "util/synthetic-events.h"
66	#include "util/target.h"
67	#include "util/time-utils.h"
68	#include "util/top.h"
69	#include "util/affinity.h"
70	#include "util/pfm.h"
71	#include "util/bpf_counter.h"
72	#include "util/iostat.h"
73	#include "util/util.h"
74	#include "util/intel-tpebs.h"
75	#include "asm/bug.h"
76
77	#include <linux/list_sort.h>
78	#include <linux/time64.h>
79	#include <linux/zalloc.h>
80	#include <api/fs/fs.h>
81	#include <errno.h>
82	#include <signal.h>
83	#include <stdlib.h>
84	#include <sys/prctl.h>
85	#include <inttypes.h>
86	#include <locale.h>
87	#include <math.h>
88	#include <sys/types.h>
89	#include <sys/stat.h>
90	#include <sys/wait.h>
91	#include <unistd.h>
92	#include <sys/time.h>
93	#include <sys/resource.h>
94	#include <linux/err.h>
95
96	#include <linux/ctype.h>
97	#include <perf/evlist.h>
98	#include <internal/threadmap.h>
99
100	#ifdef HAVE_BPF_SKEL
101	#include "util/bpf_skel/bperf_cgroup.h"
102	#endif
103
104	#define DEFAULT_SEPARATOR " "
105	#define FREEZE_ON_SMI_PATH "bus/event_source/devices/cpu/freeze_on_smi"
106
107	struct rusage_stats {
108	struct stats ru_utime_usec_stat;
109	struct stats ru_stime_usec_stat;
110	};
111
112	static void print_counters(struct timespec ts, int argc, const char *argv);
113
114	static struct evlist *evsel_list;
115	static struct parse_events_option_args parse_events_option_args = {
116	.evlistp = &evsel_list,
117	};
118
119	static bool all_counters_use_bpf = true;
120
121	static struct target target;
122
123	static volatile sig_atomic_t child_pid = -1;
124	static int detailed_run = 0;
125	static bool transaction_run;
126	static bool topdown_run = false;
127	static bool smi_cost = false;
128	static bool smi_reset = false;
129	static int big_num_opt = -1;
130	static const char *pre_cmd = NULL;
131	static const char *post_cmd = NULL;
132	static bool sync_run = false;
133	static bool forever = false;
134	static bool force_metric_only = false;
135	static struct timespec ref_time;
136	static bool append_file;
137	static bool interval_count;
138	static const char *output_name;
139	static int output_fd;
140	static char *metrics;
141	static struct rusage_stats ru_stats;
142
143	struct perf_stat {
144	bool record;
145	struct perf_data data;
146	struct perf_session *session;
147	u64 bytes_written;
148	struct perf_tool tool;
149	bool maps_allocated;
150	struct perf_cpu_map *cpus;
151	struct perf_thread_map *threads;
152	enum aggr_mode aggr_mode;
153	u32 aggr_level;
154	};
155
156	static struct perf_stat perf_stat;
157	#define STAT_RECORD perf_stat.record
158
159	static volatile sig_atomic_t done = 0;
160
161	/* Options set from the command line. */
162	struct opt_aggr_mode {
163	bool node, socket, die, cluster, cache, core, thread, no_aggr;
164	};
165
166	/* Turn command line option into most generic aggregation mode setting. */
167	static enum aggr_mode opt_aggr_mode_to_aggr_mode(const struct opt_aggr_mode *opt_mode)
168	{
169	enum aggr_mode mode = AGGR_GLOBAL;
170
171	if (opt_mode->node)
172	mode = AGGR_NODE;
173	if (opt_mode->socket)
174	mode = AGGR_SOCKET;
175	if (opt_mode->die)
176	mode = AGGR_DIE;
177	if (opt_mode->cluster)
178	mode = AGGR_CLUSTER;
179	if (opt_mode->cache)
180	mode = AGGR_CACHE;
181	if (opt_mode->core)
182	mode = AGGR_CORE;
183	if (opt_mode->thread)
184	mode = AGGR_THREAD;
185	if (opt_mode->no_aggr)
186	mode = AGGR_NONE;
187	return mode;
188	}
189
190	static void evlist__check_cpu_maps(struct evlist *evlist)
191	{
192	struct evsel evsel, warned_leader = NULL;
193
194	evlist__for_each_entry(evlist, evsel) {
195	struct evsel *leader = evsel__leader(evsel);
196
197	/* Check that leader matches cpus with each member. */
198	if (leader == evsel)
199	continue;
200	if (perf_cpu_map__equal(leader->core.cpus, evsel->core.cpus))
201	continue;
202
203	/* If there's mismatch disable the group and warn user. */
204	if (warned_leader != leader) {
205	char buf[200];
206
207	pr_warning("WARNING: grouped events cpus do not match.\n"
208	"Events with CPUs not matching the leader will "
209	"be removed from the group.\n");
210	evsel__group_desc(leader, buf, sizeof(buf));
211	pr_warning(" %s\n", buf);
212	warned_leader = leader;
213	}
214	if (verbose > 0) {
215	char buf[200];
216
217	cpu_map__snprint(leader->core.cpus, buf, sizeof(buf));
218	pr_warning(" %s: %s\n", leader->name, buf);
219	cpu_map__snprint(evsel->core.cpus, buf, sizeof(buf));
220	pr_warning(" %s: %s\n", evsel->name, buf);
221	}
222
223	evsel__remove_from_group(evsel, leader);
224	}
225	}
226
227	static inline void diff_timespec(struct timespec r, struct timespec a,
228	struct timespec *b)
229	{
230	r->tv_sec = a->tv_sec - b->tv_sec;
231	if (a->tv_nsec < b->tv_nsec) {
232	r->tv_nsec = a->tv_nsec + NSEC_PER_SEC - b->tv_nsec;
233	r->tv_sec--;
234	} else {
235	r->tv_nsec = a->tv_nsec - b->tv_nsec ;
236	}
237	}
238
239	static void perf_stat__reset_stats(void)
240	{
241	evlist__reset_stats(evsel_list);
242	memset(stat_config.walltime_nsecs_stats, 0, sizeof(*stat_config.walltime_nsecs_stats));
243	}
244
245	static int process_synthesized_event(const struct perf_tool *tool __maybe_unused,
246	union perf_event *event,
247	struct perf_sample *sample __maybe_unused,
248	struct machine *machine __maybe_unused)
249	{
250	if (perf_data__write(&perf_stat.data, event, event->header.size) < 0) {
251	pr_err("failed to write perf data, error: %m\n");
252	return -1;
253	}
254
255	perf_stat.bytes_written += event->header.size;
256	return 0;
257	}
258
259	static int write_stat_round_event(u64 tm, u64 type)
260	{
261	return perf_event__synthesize_stat_round(NULL, tm, type,
262	process_synthesized_event,
263	NULL);
264	}
265
266	#define WRITE_STAT_ROUND_EVENT(time, interval) \
267	write_stat_round_event(time, PERF_STAT_ROUND_TYPE__ ## interval)
268
269	#define SID(e, x, y) xyarray__entry(e->core.sample_id, x, y)
270
271	static int evsel__write_stat_event(struct evsel *counter, int cpu_map_idx, u32 thread,
272	struct perf_counts_values *count)
273	{
274	struct perf_sample_id *sid = SID(counter, cpu_map_idx, thread);
275	struct perf_cpu cpu = perf_cpu_map__cpu(evsel__cpus(counter), cpu_map_idx);
276
277	return perf_event__synthesize_stat(NULL, cpu, thread, sid->id, count,
278	process_synthesized_event, NULL);
279	}
280
281	static int read_single_counter(struct evsel *counter, int cpu_map_idx, int thread)
282	{
283	int err = evsel__read_counter(counter, cpu_map_idx, thread);
284
285	/*
286	* Reading user and system time will fail when the process
287	* terminates. Use the wait4 values in that case.
288	*/
289	if (err && cpu_map_idx == 0 &&
290	(evsel__tool_event(counter) == TOOL_PMU__EVENT_USER_TIME \|\|
291	evsel__tool_event(counter) == TOOL_PMU__EVENT_SYSTEM_TIME)) {
292	struct perf_counts_values *count =
293	perf_counts(counter->counts, cpu_map_idx, thread);
294	struct perf_counts_values *old_count = NULL;
295	u64 val;
296
297	if (counter->prev_raw_counts)
298	old_count = perf_counts(counter->prev_raw_counts, cpu_map_idx, thread);
299
300	if (evsel__tool_event(counter) == TOOL_PMU__EVENT_USER_TIME)
301	val = ru_stats.ru_utime_usec_stat.mean;
302	else
303	val = ru_stats.ru_stime_usec_stat.mean;
304
305	count->val = val;
306	if (old_count) {
307	count->run = old_count->run + 1;
308	count->ena = old_count->ena + 1;
309	} else {
310	count->run++;
311	count->ena++;
312	}
313	return 0;
314	}
315	return err;
316	}
317
318	/*
319	* Read out the results of a single counter:
320	* do not aggregate counts across CPUs in system-wide mode
321	*/
322	static int read_counter_cpu(struct evsel *counter, int cpu_map_idx)
323	{
324	int nthreads = perf_thread_map__nr(evlist__core(evsel_list)->threads);
325	int thread;
326
327	if (!counter->supported)
328	return -ENOENT;
329
330	for (thread = 0; thread < nthreads; thread++) {
331	struct perf_counts_values *count;
332
333	count = perf_counts(counter->counts, cpu_map_idx, thread);
334
335	/*
336	* The leader's group read loads data into its group members
337	* (via evsel__read_counter()) and sets their count->loaded.
338	*/
339	if (!perf_counts__is_loaded(counter->counts, cpu_map_idx, thread) &&
340	read_single_counter(counter, cpu_map_idx, thread)) {
341	counter->counts->scaled = -1;
342	perf_counts(counter->counts, cpu_map_idx, thread)->ena = 0;
343	perf_counts(counter->counts, cpu_map_idx, thread)->run = 0;
344	return -1;
345	}
346
347	perf_counts__set_loaded(counter->counts, cpu_map_idx, thread, false);
348
349	if (STAT_RECORD) {
350	if (evsel__write_stat_event(counter, cpu_map_idx, thread, count)) {
351	pr_err("failed to write stat event\n");
352	return -1;
353	}
354	}
355
356	if (verbose > 1) {
357	fprintf(stat_config.output,
358	"%s: %d: %" PRIu64 " %" PRIu64 " %" PRIu64 "\n",
359	evsel__name(counter),
360	perf_cpu_map__cpu(evsel__cpus(counter),
361	cpu_map_idx).cpu,
362	count->val, count->ena, count->run);
363	}
364	}
365
366	return 0;
367	}
368
369	static int read_counters_with_affinity(void)
370	{
371	struct evlist_cpu_iterator evlist_cpu_itr;
372
373	if (all_counters_use_bpf)
374	return 0;
375
376	evlist__for_each_cpu(evlist_cpu_itr, evsel_list) {
377	struct evsel *counter = evlist_cpu_itr.evsel;
378
379	if (evsel__is_bpf(counter))
380	continue;
381
382	if (evsel__is_tool(counter))
383	continue;
384
385	if (!counter->err)
386	counter->err = read_counter_cpu(counter, evlist_cpu_itr.cpu_map_idx);
387	}
388
389	return 0;
390	}
391
392	static int read_bpf_map_counters(void)
393	{
394	struct evsel *counter;
395	int err;
396
397	evlist__for_each_entry(evsel_list, counter) {
398	if (!evsel__is_bpf(counter))
399	continue;
400
401	err = bpf_counter__read(counter);
402	if (err)
403	return err;
404	}
405	return 0;
406	}
407
408	static int read_tool_counters(void)
409	{
410	struct evsel *counter;
411
412	evlist__for_each_entry(evsel_list, counter) {
413	unsigned int idx;
414
415	if (!evsel__is_tool(counter))
416	continue;
417
418	perf_cpu_map__for_each_idx(idx, counter->core.cpus) {
419	if (!counter->err)
420	counter->err = read_counter_cpu(counter, idx);
421	}
422	}
423	return 0;
424	}
425
426	static int read_counters(void)
427	{
428	int ret;
429
430	if (stat_config.stop_read_counter)
431	return 0;
432
433	// Read all BPF counters first.
434	ret = read_bpf_map_counters();
435	if (ret)
436	return ret;
437
438	// Read non-BPF and non-tool counters next.
439	ret = read_counters_with_affinity();
440	if (ret)
441	return ret;
442
443	// Read the tool counters last. This way the duration_time counter
444	// should always be greater than any other counter's enabled time.
445	return read_tool_counters();
446	}
447
448	static void process_counters(void)
449	{
450	struct evsel *counter;
451
452	evlist__for_each_entry(evsel_list, counter) {
453	if (counter->err)
454	pr_debug("failed to read counter %s\n", counter->name);
455	if (counter->err == 0 && perf_stat_process_counter(&stat_config, counter))
456	pr_warning("failed to process counter %s\n", counter->name);
457	counter->err = 0;
458	}
459
460	perf_stat_merge_counters(&stat_config, evsel_list);
461	perf_stat_process_percore(&stat_config, evsel_list);
462	}
463
464	static void process_interval(void)
465	{
466	struct timespec ts, rs;
467
468	clock_gettime(CLOCK_MONOTONIC, &ts);
469	diff_timespec(&rs, &ts, &ref_time);
470
471	evlist__reset_aggr_stats(evsel_list);
472
473	if (read_counters() == 0)
474	process_counters();
475
476	if (STAT_RECORD) {
477	if (WRITE_STAT_ROUND_EVENT(rs.tv_sec * NSEC_PER_SEC + rs.tv_nsec, INTERVAL))
478	pr_err("failed to write stat round event\n");
479	}
480
481	init_stats(stat_config.walltime_nsecs_stats);
482	update_stats(stat_config.walltime_nsecs_stats, stat_config.interval * 1000000ULL);
483	print_counters(&rs, 0, NULL);
484	}
485
486	static bool handle_interval(unsigned int interval, int *times)
487	{
488	if (interval) {
489	process_interval();
490	if (interval_count && !(--(*times)))
491	return true;
492	}
493	return false;
494	}
495
496	static int enable_counters(void)
497	{
498	struct evsel *evsel;
499	int err;
500
501	evlist__for_each_entry(evsel_list, evsel) {
502	if (!evsel__is_bpf(evsel))
503	continue;
504
505	err = bpf_counter__enable(evsel);
506	if (err)
507	return err;
508	}
509
510	if (!target__enable_on_exec(&target)) {
511	if (!all_counters_use_bpf)
512	evlist__enable(evsel_list);
513	}
514	return 0;
515	}
516
517	static void disable_counters(void)
518	{
519	struct evsel *counter;
520
521	/*
522	* If we don't have tracee (attaching to task or cpu), counters may
523	* still be running. To get accurate group ratios, we must stop groups
524	* from counting before reading their constituent counters.
525	*/
526	if (!target__none(&target)) {
527	evlist__for_each_entry(evsel_list, counter)
528	bpf_counter__disable(counter);
529	if (!all_counters_use_bpf)
530	evlist__disable(evsel_list);
531	}
532	}
533
534	static volatile sig_atomic_t workload_exec_errno;
535
536	/*
537	* evlist__prepare_workload will send a SIGUSR1
538	* if the fork fails, since we asked by setting its
539	* want_signal to true.
540	*/
541	static void workload_exec_failed_signal(int signo __maybe_unused, siginfo_t *info,
542	void *ucontext __maybe_unused)
543	{
544	workload_exec_errno = info->si_value.sival_int;
545	}
546
547	static bool evsel__should_store_id(struct evsel *counter)
548	{
549	return STAT_RECORD \|\| counter->core.attr.read_format & PERF_FORMAT_ID;
550	}
551
552	static bool is_target_alive(struct target *_target,
553	struct perf_thread_map *threads)
554	{
555	struct stat st;
556	int i;
557
558	if (!target__has_task(_target))
559	return true;
560
561	for (i = 0; i < threads->nr; i++) {
562	char path[PATH_MAX];
563
564	scnprintf(path, PATH_MAX, "%s/%d", procfs__mountpoint(),
565	threads->map[i].pid);
566
567	if (!stat(path, &st))
568	return true;
569	}
570
571	return false;
572	}
573
574	static void process_evlist(struct evlist *evlist, unsigned int interval)
575	{
576	enum evlist_ctl_cmd cmd = EVLIST_CTL_CMD_UNSUPPORTED;
577
578	if (evlist__ctlfd_process(evlist, &cmd) > 0) {
579	switch (cmd) {
580	case EVLIST_CTL_CMD_ENABLE:
581	fallthrough;
582	case EVLIST_CTL_CMD_DISABLE:
583	if (interval)
584	process_interval();
585	break;
586	case EVLIST_CTL_CMD_SNAPSHOT:
587	case EVLIST_CTL_CMD_ACK:
588	case EVLIST_CTL_CMD_UNSUPPORTED:
589	case EVLIST_CTL_CMD_EVLIST:
590	case EVLIST_CTL_CMD_STOP:
591	case EVLIST_CTL_CMD_PING:
592	default:
593	break;
594	}
595	}
596	}
597
598	static void compute_tts(struct timespec time_start, struct timespec time_stop,
599	int *time_to_sleep)
600	{
601	int tts = *time_to_sleep;
602	struct timespec time_diff;
603
604	diff_timespec(&time_diff, time_stop, time_start);
605
606	tts -= time_diff.tv_sec * MSEC_PER_SEC +
607	time_diff.tv_nsec / NSEC_PER_MSEC;
608
609	if (tts < 0)
610	tts = 0;
611
612	*time_to_sleep = tts;
613	}
614
615	static int dispatch_events(bool forks, int timeout, int interval, int *times)
616	{
617	int child_exited = 0, status = 0;
618	int time_to_sleep, sleep_time;
619	struct timespec time_start, time_stop;
620
621	if (interval)
622	sleep_time = interval;
623	else if (timeout)
624	sleep_time = timeout;
625	else
626	sleep_time = 1000;
627
628	time_to_sleep = sleep_time;
629
630	while (!done) {
631	if (forks) {
632	child_exited = waitpid(child_pid, &status, WNOHANG);
633	} else {
634	child_exited = !is_target_alive(&target,
635	evlist__core(evsel_list)->threads) ? 1 : 0;
636	}
637	if (child_exited)
638	break;
639
640	clock_gettime(CLOCK_MONOTONIC, &time_start);
641	if (!(evlist__poll(evsel_list, time_to_sleep) > 0)) { /* poll timeout or EINTR */
642	if (timeout \|\| handle_interval(interval, times))
643	break;
644	time_to_sleep = sleep_time;
645	} else { /* fd revent */
646	process_evlist(evsel_list, interval);
647	clock_gettime(CLOCK_MONOTONIC, &time_stop);
648	compute_tts(&time_start, &time_stop, &time_to_sleep);
649	}
650	}
651
652	return status;
653	}
654
655	enum counter_recovery {
656	COUNTER_SKIP,
657	COUNTER_RETRY,
658	};
659
660	static enum counter_recovery stat_handle_error(struct evsel *counter, int err)
661	{
662	char msg[BUFSIZ];
663
664	assert(!counter->supported);
665
666	/*
667	* PPC returns ENXIO for HW counters until 2.6.37
668	* (behavior changed with commit b0a873e).
669	*/
670	if (err == EINVAL \|\| err == ENOSYS \|\| err == ENOENT \|\| err == ENXIO) {
671	if (verbose > 0) {
672	evsel__open_strerror(counter, &target, err, msg, sizeof(msg));
673	ui__warning("%s event is not supported by the kernel.\n%s\n",
674	evsel__name(counter), msg);
675	}
676	return COUNTER_SKIP;
677	}
678	if (evsel__fallback(counter, &target, err, msg, sizeof(msg))) {
679	if (verbose > 0)
680	ui__warning("%s\n", msg);
681	counter->supported = true;
682	return COUNTER_RETRY;
683	}
684	if (target__has_per_thread(&target) && err != EOPNOTSUPP &&
685	evlist__core(evsel_list)->threads &&
686	evlist__core(evsel_list)->threads->err_thread != -1) {
687	/*
688	* For global --per-thread case, skip current
689	* error thread.
690	*/
691	if (!thread_map__remove(evlist__core(evsel_list)->threads,
692	evlist__core(evsel_list)->threads->err_thread)) {
693	evlist__core(evsel_list)->threads->err_thread = -1;
694	counter->supported = true;
695	return COUNTER_RETRY;
696	}
697	}
698	if (verbose > 0) {
699	evsel__open_strerror(counter, &target, err, msg, sizeof(msg));
700	ui__warning(err == EOPNOTSUPP
701	? "%s event is not supported by the kernel.\n%s\n"
702	: "skipping event %s that kernel failed to open.\n%s\n",
703	evsel__name(counter), msg);
704	}
705	return COUNTER_SKIP;
706	}
707
708	static int create_perf_stat_counter(struct evsel *evsel,
709	struct perf_stat_config *config,
710	int cpu_map_idx)
711	{
712	struct perf_event_attr *attr = &evsel->core.attr;
713	struct evsel *leader = evsel__leader(evsel);
714
715	/* Reset supported flag as creating a stat counter is retried. */
716	attr->read_format = PERF_FORMAT_TOTAL_TIME_ENABLED \|
717	PERF_FORMAT_TOTAL_TIME_RUNNING;
718
719	/*
720	* The event is part of non trivial group, let's enable
721	* the group read (for leader) and ID retrieval for all
722	* members.
723	*/
724	if (leader->core.nr_members > 1)
725	attr->read_format \|= PERF_FORMAT_ID\|PERF_FORMAT_GROUP;
726
727	attr->inherit = !config->no_inherit && list_empty(&evsel->bpf_counter_list);
728
729	/*
730	* Some events get initialized with sample_(period/type) set,
731	* like tracepoints. Clear it up for counting.
732	*/
733	attr->sample_period = 0;
734
735	if (config->identifier)
736	attr->sample_type = PERF_SAMPLE_IDENTIFIER;
737
738	if (config->all_user) {
739	attr->exclude_kernel = 1;
740	attr->exclude_user = 0;
741	}
742
743	if (config->all_kernel) {
744	attr->exclude_kernel = 0;
745	attr->exclude_user = 1;
746	}
747
748	/*
749	* Disabling all counters initially, they will be enabled
750	* either manually by us or by kernel via enable_on_exec
751	* set later.
752	*/
753	if (evsel__is_group_leader(evsel)) {
754	attr->disabled = 1;
755
756	if (target__enable_on_exec(&target))
757	attr->enable_on_exec = 1;
758	}
759
760	return evsel__open_per_cpu_and_thread(evsel, evsel__cpus(evsel), cpu_map_idx,
761	evsel->core.threads);
762	}
763
764	static void update_rusage_stats(const struct rusage *rusage)
765	{
766	const u64 us_to_ns = 1000;
767	const u64 s_to_ns = 1000000000;
768
769	update_stats(&ru_stats.ru_utime_usec_stat,
770	(rusage->ru_utime.tv_usec * us_to_ns + rusage->ru_utime.tv_sec * s_to_ns));
771	update_stats(&ru_stats.ru_stime_usec_stat,
772	(rusage->ru_stime.tv_usec * us_to_ns + rusage->ru_stime.tv_sec * s_to_ns));
773	}
774
775	static int __run_perf_stat(int argc, const char **argv, int run_idx)
776	{
777	int interval = stat_config.interval;
778	int times = stat_config.times;
779	int timeout = stat_config.timeout;
780	char msg[BUFSIZ];
781	unsigned long long t0, t1;
782	struct evsel *counter;
783	size_t l;
784	int status = 0;
785	const bool forks = (argc > 0);
786	bool is_pipe = STAT_RECORD ? perf_stat.data.is_pipe : false;
787	struct evlist_cpu_iterator evlist_cpu_itr;
788	int err, open_err = 0;
789	bool second_pass = false, has_supported_counters;
790
791	if (forks) {
792	if (evlist__prepare_workload(evsel_list, &target, argv, is_pipe,
793	workload_exec_failed_signal) < 0) {
794	perror("failed to prepare workload");
795	return -1;
796	}
797	child_pid = evlist__workload_pid(evsel_list);
798	}
799
800	evlist__for_each_entry(evsel_list, counter) {
801	counter->reset_group = false;
802	if (bpf_counter__load(counter, &target)) {
803	err = -1;
804	goto err_out;
805	}
806	if (!(evsel__is_bperf(counter)))
807	all_counters_use_bpf = false;
808	}
809
810	evlist__reset_aggr_stats(evsel_list);
811
812	/*
813	* bperf calls evsel__open_per_cpu() in bperf__load(), so
814	* no need to call it again here.
815	*/
816	if (!target.use_bpf) {
817	evlist__for_each_cpu(evlist_cpu_itr, evsel_list) {
818	counter = evlist_cpu_itr.evsel;
819
820	if (counter->reset_group \|\| !counter->supported)
821	continue;
822	if (evsel__is_bperf(counter))
823	continue;
824
825	while (true) {
826	if (create_perf_stat_counter(counter, &stat_config,
827	evlist_cpu_itr.cpu_map_idx) == 0)
828	break;
829
830	open_err = errno;
831	/*
832	* Weak group failed. We cannot just undo this
833	* here because earlier CPUs might be in group
834	* mode, and the kernel doesn't support mixing
835	* group and non group reads. Defer it to later.
836	* Don't close here because we're in the wrong
837	* affinity.
838	*/
839	if ((open_err == EINVAL \|\| open_err == EBADF) &&
840	evsel__leader(counter) != counter &&
841	counter->weak_group) {
842	evlist__reset_weak_group(evsel_list, counter, false);
843	assert(counter->reset_group);
844	counter->supported = true;
845	second_pass = true;
846	break;
847	}
848
849	if (stat_handle_error(counter, open_err) != COUNTER_RETRY)
850	break;
851	}
852	}
853	}
854	if (second_pass) {
855	/*
856	* Now redo all the weak group after closing them,
857	* and also close errored counters.
858	*/
859
860	/* First close errored or weak retry */
861	evlist__for_each_cpu(evlist_cpu_itr, evsel_list) {
862	counter = evlist_cpu_itr.evsel;
863
864	if (!counter->reset_group && counter->supported)
865	continue;
866
867	perf_evsel__close_cpu(&counter->core, evlist_cpu_itr.cpu_map_idx);
868	}
869	/* Now reopen weak */
870	evlist__for_each_cpu(evlist_cpu_itr, evsel_list) {
871	counter = evlist_cpu_itr.evsel;
872
873	if (!counter->reset_group)
874	continue;
875
876	while (true) {
877	pr_debug2("reopening weak %s\n", evsel__name(counter));
878	if (create_perf_stat_counter(counter, &stat_config,
879	evlist_cpu_itr.cpu_map_idx) == 0) {
880	evlist_cpu_iterator__exit(&evlist_cpu_itr);
881	break;
882	}
883	open_err = errno;
884	if (stat_handle_error(counter, open_err) != COUNTER_RETRY) {
885	evlist_cpu_iterator__exit(&evlist_cpu_itr);
886	break;
887	}
888	}
889	}
890	}
891
892	has_supported_counters = false;
893	evlist__for_each_entry(evsel_list, counter) {
894	if (!counter->supported) {
895	perf_evsel__free_fd(&counter->core);
896	continue;
897	}
898	has_supported_counters = true;
899
900	l = strlen(counter->unit);
901	if (l > stat_config.unit_width)
902	stat_config.unit_width = l;
903
904	if (evsel__should_store_id(counter) &&
905	evsel__store_ids(counter, evsel_list)) {
906	err = -1;
907	goto err_out;
908	}
909	}
910	if (!has_supported_counters && !stat_config.null_run) {
911	if (open_err) {
912	evsel__open_strerror(evlist__first(evsel_list), &target, open_err,
913	msg, sizeof(msg));
914	}
915	ui__error("No supported events found.\n%s\n", msg);
916
917	if (child_pid != -1)
918	kill(child_pid, SIGTERM);
919	err = -1;
920	goto err_out;
921	}
922
923	if (evlist__apply_filters(evsel_list, &counter, &target)) {
924	pr_err("failed to set filter \"%s\" on event %s: %m\n",
925	counter->filter, evsel__name(counter));
926	return -1;
927	}
928
929	if (STAT_RECORD) {
930	int fd = perf_data__fd(&perf_stat.data);
931
932	if (is_pipe) {
933	err = perf_header__write_pipe(perf_data__fd(&perf_stat.data));
934	} else {
935	err = perf_session__write_header(perf_stat.session, evsel_list,
936	fd, false);
937	}
938
939	if (err < 0)
940	goto err_out;
941
942	err = perf_event__synthesize_stat_events(&stat_config, NULL, evsel_list,
943	process_synthesized_event, is_pipe);
944	if (err < 0)
945	goto err_out;
946
947	}
948
949	if (target.initial_delay) {
950	pr_info(EVLIST_DISABLED_MSG);
951	} else {
952	err = enable_counters();
953	if (err) {
954	err = -1;
955	goto err_out;
956	}
957	}
958
959	/* Exec the command, if any */
960	if (forks)
961	evlist__start_workload(evsel_list);
962
963	if (target.initial_delay > 0) {
964	usleep(target.initial_delay * USEC_PER_MSEC);
965	err = enable_counters();
966	if (err) {
967	err = -1;
968	goto err_out;
969	}
970
971	pr_info(EVLIST_ENABLED_MSG);
972	}
973
974	t0 = rdclock();
975	clock_gettime(CLOCK_MONOTONIC, &ref_time);
976
977	if (forks) {
978	if (interval \|\| timeout \|\| evlist__ctlfd_initialized(evsel_list))
979	status = dispatch_events(forks, timeout, interval, &times);
980	if (child_pid != -1) {
981	if (timeout)
982	kill(child_pid, SIGTERM);
983	wait4(child_pid, &status, 0, &stat_config.ru_data);
984	}
985
986	if (workload_exec_errno) {
987	errno = workload_exec_errno;
988	pr_err("Workload failed: %m\n");
989	err = -1;
990	goto err_out;
991	}
992
993	if (WIFSIGNALED(status)) {
994	/*
995	* We want to indicate failure to stop a repeat run,
996	* hence negative. We want the value to be the exit code
997	* of perf, which for termination by a signal is 128
998	* plus the signal number.
999	*/
1000	err = 0 - (128 + WTERMSIG(status));
1001	psignal(WTERMSIG(status), argv[0]);
1002	} else {
1003	err = WEXITSTATUS(status);
1004	}
1005	} else {
1006	err = dispatch_events(forks, timeout, interval, &times);
1007	}
1008
1009	disable_counters();
1010
1011	t1 = rdclock();
1012
1013	if (stat_config.walltime_run_table)
1014	stat_config.walltime_run[run_idx] = t1 - t0;
1015
1016	if (interval && stat_config.summary) {
1017	stat_config.interval = 0;
1018	stat_config.stop_read_counter = true;
1019	init_stats(stat_config.walltime_nsecs_stats);
1020	update_stats(stat_config.walltime_nsecs_stats, t1 - t0);
1021
1022	evlist__copy_prev_raw_counts(evsel_list);
1023	evlist__reset_prev_raw_counts(evsel_list);
1024	evlist__reset_aggr_stats(evsel_list);
1025	} else {
1026	update_stats(stat_config.walltime_nsecs_stats, t1 - t0);
1027	update_rusage_stats(&stat_config.ru_data);
1028	}
1029
1030	/*
1031	* Closing a group leader splits the group, and as we only disable
1032	* group leaders, results in remaining events becoming enabled. To
1033	* avoid arbitrary skew, we must read all counters before closing any
1034	* group leaders.
1035	*/
1036	if (read_counters() == 0)
1037	process_counters();
1038
1039	/*
1040	* We need to keep evsel_list alive, because it's processed
1041	* later the evsel_list will be closed after.
1042	*/
1043	if (!STAT_RECORD)
1044	evlist__close(evsel_list);
1045
1046	return err;
1047
1048	err_out:
1049	if (forks)
1050	evlist__cancel_workload(evsel_list);
1051
1052	return err;
1053	}
1054
1055	/*
1056	* Returns -1 for fatal errors which signifies to not continue
1057	* when in repeat mode.
1058	*
1059	* Returns < -1 error codes when stat record is used. These
1060	* result in the stat information being displayed, but writing
1061	* to the file fails and is non fatal.
1062	*/
1063	static int run_perf_stat(int argc, const char **argv, int run_idx)
1064	{
1065	int ret;
1066
1067	if (pre_cmd) {
1068	ret = system(pre_cmd);
1069	if (ret)
1070	return ret;
1071	}
1072
1073	if (sync_run)
1074	sync();
1075
1076	ret = __run_perf_stat(argc, argv, run_idx);
1077	if (ret)
1078	return ret;
1079
1080	if (post_cmd) {
1081	ret = system(post_cmd);
1082	if (ret)
1083	return ret;
1084	}
1085
1086	return ret;
1087	}
1088
1089	static void print_counters(struct timespec ts, int argc, const char *argv)
1090	{
1091	/* Do not print anything if we record to the pipe. */
1092	if (STAT_RECORD && perf_stat.data.is_pipe)
1093	return;
1094	if (quiet)
1095	return;
1096
1097	evlist__print_counters(evsel_list, &stat_config, &target, ts, argc, argv);
1098	}
1099
1100	static volatile sig_atomic_t signr = -1;
1101
1102	static void skip_signal(int signo)
1103	{
1104	if ((child_pid == -1) \|\| stat_config.interval)
1105	done = 1;
1106
1107	signr = signo;
1108	/*
1109	* render child_pid harmless
1110	* won't send SIGTERM to a random
1111	* process in case of race condition
1112	* and fast PID recycling
1113	*/
1114	child_pid = -1;
1115	}
1116
1117	static void sig_atexit(void)
1118	{
1119	sigset_t set, oset;
1120
1121	/*
1122	* avoid race condition with SIGCHLD handler
1123	* in skip_signal() which is modifying child_pid
1124	* goal is to avoid send SIGTERM to a random
1125	* process
1126	*/
1127	sigemptyset(&set);
1128	sigaddset(&set, SIGCHLD);
1129	sigprocmask(SIG_BLOCK, &set, &oset);
1130
1131	if (child_pid != -1)
1132	kill(child_pid, SIGTERM);
1133
1134	sigprocmask(SIG_SETMASK, &oset, NULL);
1135
1136	if (signr == -1)
1137	return;
1138
1139	signal(signr, SIG_DFL);
1140	kill(getpid(), signr);
1141	}
1142
1143	static int stat__set_big_num(const struct option *opt __maybe_unused,
1144	const char *s __maybe_unused, int unset)
1145	{
1146	big_num_opt = unset ? 0 : 1;
1147	perf_stat__set_big_num(!unset);
1148	return 0;
1149	}
1150
1151	static int enable_metric_only(const struct option *opt __maybe_unused,
1152	const char *s __maybe_unused, int unset)
1153	{
1154	force_metric_only = true;
1155	stat_config.metric_only = !unset;
1156	return 0;
1157	}
1158
1159	static int append_metric_groups(const struct option *opt __maybe_unused,
1160	const char *str,
1161	int unset __maybe_unused)
1162	{
1163	if (metrics) {
1164	char *tmp;
1165
1166	if (asprintf(&tmp, "%s,%s", metrics, str) < 0)
1167	return -ENOMEM;
1168	free(metrics);
1169	metrics = tmp;
1170	} else {
1171	metrics = strdup(str);
1172	if (!metrics)
1173	return -ENOMEM;
1174	}
1175	return 0;
1176	}
1177
1178	static int parse_control_option(const struct option *opt,
1179	const char *str,
1180	int unset __maybe_unused)
1181	{
1182	struct perf_stat_config *config = opt->value;
1183
1184	return evlist__parse_control(str, &config->ctl_fd, &config->ctl_fd_ack, &config->ctl_fd_close);
1185	}
1186
1187	static int parse_stat_cgroups(const struct option *opt,
1188	const char *str, int unset)
1189	{
1190	if (stat_config.cgroup_list) {
1191	pr_err("--cgroup and --for-each-cgroup cannot be used together\n");
1192	return -1;
1193	}
1194
1195	return parse_cgroups(opt, str, unset);
1196	}
1197
1198	static int parse_cputype(const struct option *opt,
1199	const char *str,
1200	int unset __maybe_unused)
1201	{
1202	const struct perf_pmu *pmu;
1203	struct evlist evlist = (struct evlist **)opt->value;
1204
1205	if (!list_empty(&evlist__core(evlist)->entries)) {
1206	fprintf(stderr, "Must define cputype before events/metrics\n");
1207	return -1;
1208	}
1209
1210	pmu = perf_pmus__pmu_for_pmu_filter(str);
1211	if (!pmu) {
1212	fprintf(stderr, "--cputype %s is not supported!\n", str);
1213	return -1;
1214	}
1215	parse_events_option_args.pmu_filter = pmu->name;
1216	parse_events_option_args.cputype_filter = true;
1217
1218	return 0;
1219	}
1220
1221	static int parse_pmu_filter(const struct option *opt,
1222	const char *str,
1223	int unset __maybe_unused)
1224	{
1225	struct evlist evlist = (struct evlist **)opt->value;
1226
1227	if (!list_empty(&evlist__core(evlist)->entries)) {
1228	fprintf(stderr, "Must define pmu-filter before events/metrics\n");
1229	return -1;
1230	}
1231
1232	parse_events_option_args.pmu_filter = str;
1233	parse_events_option_args.cputype_filter = false;
1234	return 0;
1235	}
1236
1237	static int parse_cache_level(const struct option *opt,
1238	const char *str,
1239	int unset __maybe_unused)
1240	{
1241	int level;
1242	bool *per_cache = opt->value;
1243	u32 *aggr_level = opt->data;
1244
1245	/*
1246	* If no string is specified, aggregate based on the topology of
1247	* Last Level Cache (LLC). Since the LLC level can change from
1248	* architecture to architecture, set level greater than
1249	* MAX_CACHE_LVL which will be interpreted as LLC.
1250	*/
1251	if (str == NULL) {
1252	level = MAX_CACHE_LVL + 1;
1253	goto out;
1254	}
1255
1256	/*
1257	* The format to specify cache level is LX or lX where X is the
1258	* cache level.
1259	*/
1260	if (strlen(str) != 2 \|\| (str[0] != 'l' && str[0] != 'L')) {
1261	pr_err("Cache level must be of form L[1-%d], or l[1-%d]\n",
1262	MAX_CACHE_LVL,
1263	MAX_CACHE_LVL);
1264	return -EINVAL;
1265	}
1266
1267	level = atoi(&str[1]);
1268	if (level < 1) {
1269	pr_err("Cache level must be of form L[1-%d], or l[1-%d]\n",
1270	MAX_CACHE_LVL,
1271	MAX_CACHE_LVL);
1272	return -EINVAL;
1273	}
1274
1275	if (level > MAX_CACHE_LVL) {
1276	pr_err("perf only supports max cache level of %d.\n"
1277	"Consider increasing MAX_CACHE_LVL\n", MAX_CACHE_LVL);
1278	return -EINVAL;
1279	}
1280	out:
1281	*per_cache = true;
1282	*aggr_level = level;
1283	return 0;
1284	}
1285
1286	/**
1287	* Calculate the cache instance ID from the map in
1288	* /sys/devices/system/cpu/cpuX/cache/indexY/shared_cpu_list
1289	* Cache instance ID is the first CPU reported in the shared_cpu_list file.
1290	*/
1291	static int cpu__get_cache_id_from_map(struct perf_cpu cpu, char *map)
1292	{
1293	int id;
1294	struct perf_cpu_map *cpu_map = perf_cpu_map__new(map);
1295
1296	/*
1297	* If the map contains no CPU, consider the current CPU to
1298	* be the first online CPU in the cache domain else use the
1299	* first online CPU of the cache domain as the ID.
1300	*/
1301	id = perf_cpu_map__min(cpu_map).cpu;
1302	if (id == -1)
1303	id = cpu.cpu;
1304
1305	/* Free the perf_cpu_map used to find the cache ID */
1306	perf_cpu_map__put(cpu_map);
1307
1308	return id;
1309	}
1310
1311	/**
1312	* cpu__get_cache_id - Returns 0 if successful in populating the
1313	* cache level and cache id. Cache level is read from
1314	* /sys/devices/system/cpu/cpuX/cache/indexY/level where as cache instance ID
1315	* is the first CPU reported by
1316	* /sys/devices/system/cpu/cpuX/cache/indexY/shared_cpu_list
1317	*/
1318	static int cpu__get_cache_details(struct perf_cpu cpu, struct perf_cache *cache)
1319	{
1320	int ret = 0;
1321	u32 cache_level = stat_config.aggr_level;
1322	struct cpu_cache_level caches[MAX_CACHE_LVL];
1323	u32 i = 0, caches_cnt = 0;
1324
1325	cache->cache_lvl = (cache_level > MAX_CACHE_LVL) ? 0 : cache_level;
1326	cache->cache = -1;
1327
1328	ret = build_caches_for_cpu(cpu.cpu, caches, &caches_cnt);
1329	if (ret) {
1330	/*
1331	* If caches_cnt is not 0, cpu_cache_level data
1332	* was allocated when building the topology.
1333	* Free the allocated data before returning.
1334	*/
1335	if (caches_cnt)
1336	goto free_caches;
1337
1338	return ret;
1339	}
1340
1341	if (!caches_cnt)
1342	return -1;
1343
1344	/*
1345	* Save the data for the highest level if no
1346	* level was specified by the user.
1347	*/
1348	if (cache_level > MAX_CACHE_LVL) {
1349	int max_level_index = 0;
1350
1351	for (i = 1; i < caches_cnt; ++i) {
1352	if (caches[i].level > caches[max_level_index].level)
1353	max_level_index = i;
1354	}
1355
1356	cache->cache_lvl = caches[max_level_index].level;
1357	cache->cache = cpu__get_cache_id_from_map(cpu, caches[max_level_index].map);
1358
1359	/* Reset i to 0 to free entire caches[] */
1360	i = 0;
1361	goto free_caches;
1362	}
1363
1364	for (i = 0; i < caches_cnt; ++i) {
1365	if (caches[i].level == cache_level) {
1366	cache->cache_lvl = cache_level;
1367	cache->cache = cpu__get_cache_id_from_map(cpu, caches[i].map);
1368	}
1369
1370	cpu_cache_level__free(&caches[i]);
1371	}
1372
1373	free_caches:
1374	/*
1375	* Free all the allocated cpu_cache_level data.
1376	*/
1377	while (i < caches_cnt)
1378	cpu_cache_level__free(&caches[i++]);
1379
1380	return ret;
1381	}
1382
1383	/**
1384	* aggr_cpu_id__cache - Create an aggr_cpu_id with cache instache ID, cache
1385	* level, die and socket populated with the cache instache ID, cache level,
1386	* die and socket for cpu. The function signature is compatible with
1387	* aggr_cpu_id_get_t.
1388	*/
1389	static struct aggr_cpu_id aggr_cpu_id__cache(struct perf_cpu cpu, void *data)
1390	{
1391	int ret;
1392	struct aggr_cpu_id id;
1393	struct perf_cache cache;
1394
1395	id = aggr_cpu_id__die(cpu, data);
1396	if (aggr_cpu_id__is_empty(&id))
1397	return id;
1398
1399	ret = cpu__get_cache_details(cpu, &cache);
1400	if (ret)
1401	return id;
1402
1403	id.cache_lvl = cache.cache_lvl;
1404	id.cache = cache.cache;
1405	return id;
1406	}
1407
1408	static const char *const aggr_mode__string[] = {
1409	[AGGR_CORE] = "core",
1410	[AGGR_CACHE] = "cache",
1411	[AGGR_CLUSTER] = "cluster",
1412	[AGGR_DIE] = "die",
1413	[AGGR_GLOBAL] = "global",
1414	[AGGR_NODE] = "node",
1415	[AGGR_NONE] = "none",
1416	[AGGR_SOCKET] = "socket",
1417	[AGGR_THREAD] = "thread",
1418	[AGGR_UNSET] = "unset",
1419	};
1420
1421	static struct aggr_cpu_id perf_stat__get_socket(struct perf_stat_config *config __maybe_unused,
1422	struct perf_cpu cpu)
1423	{
1424	return aggr_cpu_id__socket(cpu, /data=/NULL);
1425	}
1426
1427	static struct aggr_cpu_id perf_stat__get_die(struct perf_stat_config *config __maybe_unused,
1428	struct perf_cpu cpu)
1429	{
1430	return aggr_cpu_id__die(cpu, /data=/NULL);
1431	}
1432
1433	static struct aggr_cpu_id perf_stat__get_cache_id(struct perf_stat_config *config __maybe_unused,
1434	struct perf_cpu cpu)
1435	{
1436	return aggr_cpu_id__cache(cpu, /data=/NULL);
1437	}
1438
1439	static struct aggr_cpu_id perf_stat__get_cluster(struct perf_stat_config *config __maybe_unused,
1440	struct perf_cpu cpu)
1441	{
1442	return aggr_cpu_id__cluster(cpu, /data=/NULL);
1443	}
1444
1445	static struct aggr_cpu_id perf_stat__get_core(struct perf_stat_config *config __maybe_unused,
1446	struct perf_cpu cpu)
1447	{
1448	return aggr_cpu_id__core(cpu, /data=/NULL);
1449	}
1450
1451	static struct aggr_cpu_id perf_stat__get_node(struct perf_stat_config *config __maybe_unused,
1452	struct perf_cpu cpu)
1453	{
1454	return aggr_cpu_id__node(cpu, /data=/NULL);
1455	}
1456
1457	static struct aggr_cpu_id perf_stat__get_global(struct perf_stat_config *config __maybe_unused,
1458	struct perf_cpu cpu)
1459	{
1460	return aggr_cpu_id__global(cpu, /data=/NULL);
1461	}
1462
1463	static struct aggr_cpu_id perf_stat__get_cpu(struct perf_stat_config *config __maybe_unused,
1464	struct perf_cpu cpu)
1465	{
1466	return aggr_cpu_id__cpu(cpu, /data=/NULL);
1467	}
1468
1469	static struct aggr_cpu_id perf_stat__get_aggr(struct perf_stat_config *config,
1470	aggr_get_id_t get_id, struct perf_cpu cpu)
1471	{
1472	struct aggr_cpu_id id;
1473
1474	/* per-process mode - should use global aggr mode */
1475	if (cpu.cpu == -1 \|\| cpu.cpu >= config->cpus_aggr_map->nr)
1476	return get_id(config, cpu);
1477
1478	if (aggr_cpu_id__is_empty(&config->cpus_aggr_map->map[cpu.cpu]))
1479	config->cpus_aggr_map->map[cpu.cpu] = get_id(config, cpu);
1480
1481	id = config->cpus_aggr_map->map[cpu.cpu];
1482	return id;
1483	}
1484
1485	static struct aggr_cpu_id perf_stat__get_socket_cached(struct perf_stat_config *config,
1486	struct perf_cpu cpu)
1487	{
1488	return perf_stat__get_aggr(config, perf_stat__get_socket, cpu);
1489	}
1490
1491	static struct aggr_cpu_id perf_stat__get_die_cached(struct perf_stat_config *config,
1492	struct perf_cpu cpu)
1493	{
1494	return perf_stat__get_aggr(config, perf_stat__get_die, cpu);
1495	}
1496
1497	static struct aggr_cpu_id perf_stat__get_cluster_cached(struct perf_stat_config *config,
1498	struct perf_cpu cpu)
1499	{
1500	return perf_stat__get_aggr(config, perf_stat__get_cluster, cpu);
1501	}
1502
1503	static struct aggr_cpu_id perf_stat__get_cache_id_cached(struct perf_stat_config *config,
1504	struct perf_cpu cpu)
1505	{
1506	return perf_stat__get_aggr(config, perf_stat__get_cache_id, cpu);
1507	}
1508
1509	static struct aggr_cpu_id perf_stat__get_core_cached(struct perf_stat_config *config,
1510	struct perf_cpu cpu)
1511	{
1512	return perf_stat__get_aggr(config, perf_stat__get_core, cpu);
1513	}
1514
1515	static struct aggr_cpu_id perf_stat__get_node_cached(struct perf_stat_config *config,
1516	struct perf_cpu cpu)
1517	{
1518	return perf_stat__get_aggr(config, perf_stat__get_node, cpu);
1519	}
1520
1521	static struct aggr_cpu_id perf_stat__get_global_cached(struct perf_stat_config *config,
1522	struct perf_cpu cpu)
1523	{
1524	return perf_stat__get_aggr(config, perf_stat__get_global, cpu);
1525	}
1526
1527	static struct aggr_cpu_id perf_stat__get_cpu_cached(struct perf_stat_config *config,
1528	struct perf_cpu cpu)
1529	{
1530	return perf_stat__get_aggr(config, perf_stat__get_cpu, cpu);
1531	}
1532
1533	static aggr_cpu_id_get_t aggr_mode__get_aggr(enum aggr_mode aggr_mode)
1534	{
1535	switch (aggr_mode) {
1536	case AGGR_SOCKET:
1537	return aggr_cpu_id__socket;
1538	case AGGR_DIE:
1539	return aggr_cpu_id__die;
1540	case AGGR_CLUSTER:
1541	return aggr_cpu_id__cluster;
1542	case AGGR_CACHE:
1543	return aggr_cpu_id__cache;
1544	case AGGR_CORE:
1545	return aggr_cpu_id__core;
1546	case AGGR_NODE:
1547	return aggr_cpu_id__node;
1548	case AGGR_NONE:
1549	return aggr_cpu_id__cpu;
1550	case AGGR_GLOBAL:
1551	return aggr_cpu_id__global;
1552	case AGGR_THREAD:
1553	case AGGR_UNSET:
1554	case AGGR_MAX:
1555	default:
1556	return NULL;
1557	}
1558	}
1559
1560	static aggr_get_id_t aggr_mode__get_id(enum aggr_mode aggr_mode)
1561	{
1562	switch (aggr_mode) {
1563	case AGGR_SOCKET:
1564	return perf_stat__get_socket_cached;
1565	case AGGR_DIE:
1566	return perf_stat__get_die_cached;
1567	case AGGR_CLUSTER:
1568	return perf_stat__get_cluster_cached;
1569	case AGGR_CACHE:
1570	return perf_stat__get_cache_id_cached;
1571	case AGGR_CORE:
1572	return perf_stat__get_core_cached;
1573	case AGGR_NODE:
1574	return perf_stat__get_node_cached;
1575	case AGGR_NONE:
1576	return perf_stat__get_cpu_cached;
1577	case AGGR_GLOBAL:
1578	return perf_stat__get_global_cached;
1579	case AGGR_THREAD:
1580	case AGGR_UNSET:
1581	case AGGR_MAX:
1582	default:
1583	return NULL;
1584	}
1585	}
1586
1587	static int perf_stat_init_aggr_mode(void)
1588	{
1589	int nr;
1590	aggr_cpu_id_get_t get_id = aggr_mode__get_aggr(stat_config.aggr_mode);
1591
1592	if (get_id) {
1593	bool needs_sort = stat_config.aggr_mode != AGGR_NONE;
1594	stat_config.aggr_map = cpu_aggr_map__new(
1595	evlist__core(evsel_list)->user_requested_cpus,
1596	get_id, /data=/NULL, needs_sort);
1597	if (!stat_config.aggr_map) {
1598	pr_err("cannot build %s map\n", aggr_mode__string[stat_config.aggr_mode]);
1599	return -1;
1600	}
1601	stat_config.aggr_get_id = aggr_mode__get_id(stat_config.aggr_mode);
1602	}
1603
1604	if (stat_config.aggr_mode == AGGR_THREAD) {
1605	nr = perf_thread_map__nr(evlist__core(evsel_list)->threads);
1606	stat_config.aggr_map = cpu_aggr_map__empty_new(nr);
1607	if (stat_config.aggr_map == NULL)
1608	return -ENOMEM;
1609
1610	for (int s = 0; s < nr; s++) {
1611	struct aggr_cpu_id id = aggr_cpu_id__empty();
1612
1613	id.thread_idx = s;
1614	stat_config.aggr_map->map[s] = id;
1615	}
1616	return 0;
1617	}
1618
1619	/*
1620	* The evsel_list->cpus is the base we operate on,
1621	* taking the highest cpu number to be the size of
1622	* the aggregation translate cpumap.
1623	*/
1624	nr = perf_cpu_map__max(evlist__core(evsel_list)->all_cpus).cpu + 1;
1625	stat_config.cpus_aggr_map = cpu_aggr_map__empty_new(nr);
1626	return stat_config.cpus_aggr_map ? 0 : -ENOMEM;
1627	}
1628
1629	static void cpu_aggr_map__delete(struct cpu_aggr_map *map)
1630	{
1631	free(map);
1632	}
1633
1634	static void perf_stat__exit_aggr_mode(void)
1635	{
1636	cpu_aggr_map__delete(stat_config.aggr_map);
1637	cpu_aggr_map__delete(stat_config.cpus_aggr_map);
1638	stat_config.aggr_map = NULL;
1639	stat_config.cpus_aggr_map = NULL;
1640	}
1641
1642	static struct aggr_cpu_id perf_env__get_socket_aggr_by_cpu(struct perf_cpu cpu, void *data)
1643	{
1644	struct perf_env *env = data;
1645	struct aggr_cpu_id id = aggr_cpu_id__empty();
1646	struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu);
1647
1648	if (topo)
1649	id.socket = topo->socket_id;
1650
1651	return id;
1652	}
1653
1654	static struct aggr_cpu_id perf_env__get_die_aggr_by_cpu(struct perf_cpu cpu, void *data)
1655	{
1656	struct perf_env *env = data;
1657	struct aggr_cpu_id id = aggr_cpu_id__empty();
1658	struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu);
1659
1660	if (topo) {
1661	/*
1662	* die_id is relative to socket, so start
1663	* with the socket ID and then add die to
1664	* make a unique ID.
1665	*/
1666	id.socket = topo->socket_id;
1667	id.die = topo->die_id;
1668	}
1669
1670	return id;
1671	}
1672
1673	static void perf_env__get_cache_id_for_cpu(struct perf_cpu cpu, struct perf_env *env,
1674	u32 cache_level, struct aggr_cpu_id *id)
1675	{
1676	int i;
1677	int caches_cnt = env->caches_cnt;
1678	struct cpu_cache_level *caches = env->caches;
1679
1680	id->cache_lvl = (cache_level > MAX_CACHE_LVL) ? 0 : cache_level;
1681	id->cache = -1;
1682
1683	if (!caches_cnt)
1684	return;
1685
1686	for (i = caches_cnt - 1; i > -1; --i) {
1687	struct perf_cpu_map *cpu_map;
1688	int map_contains_cpu;
1689
1690	/*
1691	* If user has not specified a level, find the fist level with
1692	* the cpu in the map. Since building the map is expensive, do
1693	* this only if levels match.
1694	*/
1695	if (cache_level <= MAX_CACHE_LVL && caches[i].level != cache_level)
1696	continue;
1697
1698	cpu_map = perf_cpu_map__new(caches[i].map);
1699	map_contains_cpu = perf_cpu_map__idx(cpu_map, cpu);
1700	perf_cpu_map__put(cpu_map);
1701
1702	if (map_contains_cpu != -1) {
1703	id->cache_lvl = caches[i].level;
1704	id->cache = cpu__get_cache_id_from_map(cpu, caches[i].map);
1705	return;
1706	}
1707	}
1708	}
1709
1710	static struct aggr_cpu_id perf_env__get_cache_aggr_by_cpu(struct perf_cpu cpu,
1711	void *data)
1712	{
1713	struct perf_env *env = data;
1714	struct aggr_cpu_id id = aggr_cpu_id__empty();
1715	struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu);
1716
1717	if (topo) {
1718	u32 cache_level = (perf_stat.aggr_level) ?: stat_config.aggr_level;
1719
1720	id.socket = topo->socket_id;
1721	id.die = topo->die_id;
1722	perf_env__get_cache_id_for_cpu(cpu, env, cache_level, &id);
1723	}
1724
1725	return id;
1726	}
1727
1728	static struct aggr_cpu_id perf_env__get_cluster_aggr_by_cpu(struct perf_cpu cpu,
1729	void *data)
1730	{
1731	struct perf_env *env = data;
1732	struct aggr_cpu_id id = aggr_cpu_id__empty();
1733	struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu);
1734
1735	if (topo) {
1736	id.socket = topo->socket_id;
1737	id.die = topo->die_id;
1738	id.cluster = topo->cluster_id;
1739	}
1740
1741	return id;
1742	}
1743
1744	static struct aggr_cpu_id perf_env__get_core_aggr_by_cpu(struct perf_cpu cpu, void *data)
1745	{
1746	struct perf_env *env = data;
1747	struct aggr_cpu_id id = aggr_cpu_id__empty();
1748	struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu);
1749
1750	if (topo) {
1751	/*
1752	* core_id is relative to socket, die and cluster, we need a
1753	* global id. So we set socket, die id, cluster id and core id.
1754	*/
1755	id.socket = topo->socket_id;
1756	id.die = topo->die_id;
1757	id.cluster = topo->cluster_id;
1758	id.core = topo->core_id;
1759	}
1760
1761	return id;
1762	}
1763
1764	static struct aggr_cpu_id perf_env__get_cpu_aggr_by_cpu(struct perf_cpu cpu, void *data)
1765	{
1766	struct perf_env *env = data;
1767	struct aggr_cpu_id id = aggr_cpu_id__empty();
1768	struct cpu_topology_map *topo = perf_env__get_cpu_topology(env, cpu);
1769
1770	if (topo) {
1771	/*
1772	* core_id is relative to socket and die,
1773	* we need a global id. So we set
1774	* socket, die id and core id
1775	*/
1776	id.socket = topo->socket_id;
1777	id.die = topo->die_id;
1778	id.core = topo->core_id;
1779	}
1780	id.cpu = cpu;
1781
1782	return id;
1783	}
1784
1785	static struct aggr_cpu_id perf_env__get_node_aggr_by_cpu(struct perf_cpu cpu, void *data)
1786	{
1787	struct aggr_cpu_id id = aggr_cpu_id__empty();
1788
1789	id.node = perf_env__numa_node(data, cpu);
1790	return id;
1791	}
1792
1793	static struct aggr_cpu_id perf_env__get_global_aggr_by_cpu(struct perf_cpu cpu __maybe_unused,
1794	void *data __maybe_unused)
1795	{
1796	struct aggr_cpu_id id = aggr_cpu_id__empty();
1797
1798	/* it always aggregates to the cpu 0 */
1799	id.cpu = (struct perf_cpu){ .cpu = 0 };
1800	return id;
1801	}
1802
1803	static struct aggr_cpu_id perf_stat__get_socket_file(struct perf_stat_config *config __maybe_unused,
1804	struct perf_cpu cpu)
1805	{
1806	return perf_env__get_socket_aggr_by_cpu(cpu, perf_session__env(perf_stat.session));
1807	}
1808	static struct aggr_cpu_id perf_stat__get_die_file(struct perf_stat_config *config __maybe_unused,
1809	struct perf_cpu cpu)
1810	{
1811	return perf_env__get_die_aggr_by_cpu(cpu, perf_session__env(perf_stat.session));
1812	}
1813
1814	static struct aggr_cpu_id perf_stat__get_cluster_file(struct perf_stat_config *config __maybe_unused,
1815	struct perf_cpu cpu)
1816	{
1817	return perf_env__get_cluster_aggr_by_cpu(cpu, perf_session__env(perf_stat.session));
1818	}
1819
1820	static struct aggr_cpu_id perf_stat__get_cache_file(struct perf_stat_config *config __maybe_unused,
1821	struct perf_cpu cpu)
1822	{
1823	return perf_env__get_cache_aggr_by_cpu(cpu, perf_session__env(perf_stat.session));
1824	}
1825
1826	static struct aggr_cpu_id perf_stat__get_core_file(struct perf_stat_config *config __maybe_unused,
1827	struct perf_cpu cpu)
1828	{
1829	return perf_env__get_core_aggr_by_cpu(cpu, perf_session__env(perf_stat.session));
1830	}
1831
1832	static struct aggr_cpu_id perf_stat__get_cpu_file(struct perf_stat_config *config __maybe_unused,
1833	struct perf_cpu cpu)
1834	{
1835	return perf_env__get_cpu_aggr_by_cpu(cpu, perf_session__env(perf_stat.session));
1836	}
1837
1838	static struct aggr_cpu_id perf_stat__get_node_file(struct perf_stat_config *config __maybe_unused,
1839	struct perf_cpu cpu)
1840	{
1841	return perf_env__get_node_aggr_by_cpu(cpu, perf_session__env(perf_stat.session));
1842	}
1843
1844	static struct aggr_cpu_id perf_stat__get_global_file(struct perf_stat_config *config __maybe_unused,
1845	struct perf_cpu cpu)
1846	{
1847	return perf_env__get_global_aggr_by_cpu(cpu, perf_session__env(perf_stat.session));
1848	}
1849
1850	static aggr_cpu_id_get_t aggr_mode__get_aggr_file(enum aggr_mode aggr_mode)
1851	{
1852	switch (aggr_mode) {
1853	case AGGR_SOCKET:
1854	return perf_env__get_socket_aggr_by_cpu;
1855	case AGGR_DIE:
1856	return perf_env__get_die_aggr_by_cpu;
1857	case AGGR_CLUSTER:
1858	return perf_env__get_cluster_aggr_by_cpu;
1859	case AGGR_CACHE:
1860	return perf_env__get_cache_aggr_by_cpu;
1861	case AGGR_CORE:
1862	return perf_env__get_core_aggr_by_cpu;
1863	case AGGR_NODE:
1864	return perf_env__get_node_aggr_by_cpu;
1865	case AGGR_GLOBAL:
1866	return perf_env__get_global_aggr_by_cpu;
1867	case AGGR_NONE:
1868	return perf_env__get_cpu_aggr_by_cpu;
1869	case AGGR_THREAD:
1870	case AGGR_UNSET:
1871	case AGGR_MAX:
1872	default:
1873	return NULL;
1874	}
1875	}
1876
1877	static aggr_get_id_t aggr_mode__get_id_file(enum aggr_mode aggr_mode)
1878	{
1879	switch (aggr_mode) {
1880	case AGGR_SOCKET:
1881	return perf_stat__get_socket_file;
1882	case AGGR_DIE:
1883	return perf_stat__get_die_file;
1884	case AGGR_CLUSTER:
1885	return perf_stat__get_cluster_file;
1886	case AGGR_CACHE:
1887	return perf_stat__get_cache_file;
1888	case AGGR_CORE:
1889	return perf_stat__get_core_file;
1890	case AGGR_NODE:
1891	return perf_stat__get_node_file;
1892	case AGGR_GLOBAL:
1893	return perf_stat__get_global_file;
1894	case AGGR_NONE:
1895	return perf_stat__get_cpu_file;
1896	case AGGR_THREAD:
1897	case AGGR_UNSET:
1898	case AGGR_MAX:
1899	default:
1900	return NULL;
1901	}
1902	}
1903
1904	static int perf_stat_init_aggr_mode_file(struct perf_stat *st)
1905	{
1906	struct perf_env *env = perf_session__env(st->session);
1907	aggr_cpu_id_get_t get_id = aggr_mode__get_aggr_file(stat_config.aggr_mode);
1908	bool needs_sort = stat_config.aggr_mode != AGGR_NONE;
1909
1910	if (stat_config.aggr_mode == AGGR_THREAD) {
1911	int nr = perf_thread_map__nr(evlist__core(evsel_list)->threads);
1912
1913	stat_config.aggr_map = cpu_aggr_map__empty_new(nr);
1914	if (stat_config.aggr_map == NULL)
1915	return -ENOMEM;
1916
1917	for (int s = 0; s < nr; s++) {
1918	struct aggr_cpu_id id = aggr_cpu_id__empty();
1919
1920	id.thread_idx = s;
1921	stat_config.aggr_map->map[s] = id;
1922	}
1923	return 0;
1924	}
1925
1926	if (!get_id)
1927	return 0;
1928
1929	stat_config.aggr_map = cpu_aggr_map__new(evlist__core(evsel_list)->user_requested_cpus,
1930	get_id, env, needs_sort);
1931	if (!stat_config.aggr_map) {
1932	pr_err("cannot build %s map\n", aggr_mode__string[stat_config.aggr_mode]);
1933	return -1;
1934	}
1935	stat_config.aggr_get_id = aggr_mode__get_id_file(stat_config.aggr_mode);
1936	return 0;
1937	}
1938
1939	static int default_evlist_evsel_cmp(void *priv __maybe_unused,
1940	const struct list_head *l,
1941	const struct list_head *r)
1942	{
1943	const struct perf_evsel *lhs_core = container_of(l, struct perf_evsel, node);
1944	const struct evsel *lhs = container_of(lhs_core, struct evsel, core);
1945	const struct perf_evsel *rhs_core = container_of(r, struct perf_evsel, node);
1946	const struct evsel *rhs = container_of(rhs_core, struct evsel, core);
1947	const struct evsel *lhs_leader = evsel__leader(lhs);
1948	const struct evsel *rhs_leader = evsel__leader(rhs);
1949
1950	if (lhs_leader == rhs_leader) {
1951	/* Within the same group, respect the original order. */
1952	return lhs_core->idx - rhs_core->idx;
1953	}
1954
1955	/*
1956	* Compare using leader's attributes so that all members of a group
1957	* stay together. This ensures leaders are opened before their members.
1958	*/
1959
1960	/* Sort default metrics evsels first, and default show events before those. */
1961	if (lhs_leader->default_metricgroup != rhs_leader->default_metricgroup)
1962	return lhs_leader->default_metricgroup ? -1 : 1;
1963
1964	if (lhs_leader->default_show_events != rhs_leader->default_show_events)
1965	return lhs_leader->default_show_events ? -1 : 1;
1966
1967	/* Sort by PMU type (prefers legacy types first). */
1968	if (lhs_leader->pmu != rhs_leader->pmu)
1969	return lhs_leader->pmu->type - rhs_leader->pmu->type;
1970
1971	/* Sort by leader's name. */
1972	return strcmp(evsel__name((struct evsel *)lhs_leader),
1973	evsel__name((struct evsel *)rhs_leader));
1974	}
1975
1976	/*
1977	* Add default events, if there were no attributes specified or
1978	* if -d/--detailed, -d -d or -d -d -d is used:
1979	*/
1980	static int add_default_events(void)
1981	{
1982	const char *pmu = parse_events_option_args.pmu_filter ?: "all";
1983	struct parse_events_error err;
1984	struct evlist *evlist = evlist__new();
1985	struct evsel *evsel;
1986	int ret = 0;
1987
1988	if (!evlist)
1989	return -ENOMEM;
1990
1991	parse_events_error__init(&err);
1992
1993	/* Set attrs if no event is selected and !null_run: */
1994	if (stat_config.null_run)
1995	goto out;
1996
1997	if (transaction_run) {
1998	/* Handle -T as -M transaction. Once platform specific metrics
1999	* support has been added to the json files, all architectures
2000	* will use this approach. To determine transaction support
2001	* on an architecture test for such a metric name.
2002	*/
2003	if (!metricgroup__has_metric_or_groups(pmu, "transaction")) {
2004	pr_err("Missing transaction metrics\n");
2005	ret = -1;
2006	goto out;
2007	}
2008	ret = metricgroup__parse_groups(evlist, pmu,
2009	parse_events_option_args.cputype_filter,
2010	"transaction",
2011	stat_config.metric_no_group,
2012	stat_config.metric_no_merge,
2013	stat_config.metric_no_threshold,
2014	stat_config.user_requested_cpu_list,
2015	stat_config.system_wide,
2016	stat_config.hardware_aware_grouping);
2017	goto out;
2018	}
2019
2020	if (smi_cost) {
2021	int smi;
2022
2023	if (sysfs__read_int(FREEZE_ON_SMI_PATH, &smi) < 0) {
2024	pr_err("freeze_on_smi is not supported.\n");
2025	ret = -1;
2026	goto out;
2027	}
2028
2029	if (!smi) {
2030	if (sysfs__write_int(FREEZE_ON_SMI_PATH, 1) < 0) {
2031	pr_err("Failed to set freeze_on_smi.\n");
2032	ret = -1;
2033	goto out;
2034	}
2035	smi_reset = true;
2036	}
2037
2038	if (!metricgroup__has_metric_or_groups(pmu, "smi")) {
2039	pr_err("Missing smi metrics\n");
2040	ret = -1;
2041	goto out;
2042	}
2043
2044	if (!force_metric_only)
2045	stat_config.metric_only = true;
2046
2047	ret = metricgroup__parse_groups(evlist, pmu,
2048	parse_events_option_args.cputype_filter,
2049	"smi",
2050	stat_config.metric_no_group,
2051	stat_config.metric_no_merge,
2052	stat_config.metric_no_threshold,
2053	stat_config.user_requested_cpu_list,
2054	stat_config.system_wide,
2055	stat_config.hardware_aware_grouping);
2056	goto out;
2057	}
2058
2059	if (topdown_run) {
2060	unsigned int max_level = metricgroups__topdown_max_level();
2061	char str[] = "TopdownL1";
2062
2063	if (!force_metric_only)
2064	stat_config.metric_only = true;
2065
2066	if (!max_level) {
2067	pr_err("Topdown requested but the topdown metric groups aren't present.\n"
2068	"(See perf list the metric groups have names like TopdownL1)\n");
2069	ret = -1;
2070	goto out;
2071	}
2072	if (stat_config.topdown_level > max_level) {
2073	pr_err("Invalid top-down metrics level. The max level is %u.\n", max_level);
2074	ret = -1;
2075	goto out;
2076	} else if (!stat_config.topdown_level) {
2077	stat_config.topdown_level = 1;
2078	}
2079	if (!stat_config.interval && !stat_config.metric_only) {
2080	fprintf(stat_config.output,
2081	"Topdown accuracy may decrease when measuring long periods.\n"
2082	"Please print the result regularly, e.g. -I1000\n");
2083	}
2084	str[8] = stat_config.topdown_level + '0';
2085	if (metricgroup__parse_groups(evlist,
2086	pmu, parse_events_option_args.cputype_filter, str,
2087	/metric_no_group=/false,
2088	/metric_no_merge=/false,
2089	/metric_no_threshold=/true,
2090	stat_config.user_requested_cpu_list,
2091	stat_config.system_wide,
2092	stat_config.hardware_aware_grouping) < 0) {
2093	ret = -1;
2094	goto out;
2095	}
2096	}
2097
2098	if (!stat_config.topdown_level)
2099	stat_config.topdown_level = 1;
2100
2101	if (!evlist__nr_entries(evlist) && !evlist__nr_entries(evsel_list)) {
2102	/*
2103	* Add Default metrics. To minimize multiplexing, don't request
2104	* threshold computation, but it will be computed if the events
2105	* are present.
2106	*/
2107	const char *default_metricgroup_names[] = {
2108	"Default", "Default2", "Default3", "Default4",
2109	};
2110
2111	for (size_t i = 0; i < ARRAY_SIZE(default_metricgroup_names); i++) {
2112	struct evlist *metric_evlist;
2113
2114	if (!metricgroup__has_metric_or_groups(pmu, default_metricgroup_names[i]))
2115	continue;
2116
2117	if ((int)i > detailed_run)
2118	break;
2119
2120	metric_evlist = evlist__new();
2121	if (!metric_evlist) {
2122	ret = -ENOMEM;
2123	break;
2124	}
2125	if (metricgroup__parse_groups(metric_evlist, pmu,
2126	parse_events_option_args.cputype_filter,
2127	default_metricgroup_names[i],
2128	/metric_no_group=/false,
2129	/metric_no_merge=/false,
2130	/metric_no_threshold=/true,
2131	stat_config.user_requested_cpu_list,
2132	stat_config.system_wide,
2133	stat_config.hardware_aware_grouping) < 0) {
2134	evlist__put(metric_evlist);
2135	ret = -1;
2136	break;
2137	}
2138
2139	evlist__for_each_entry(metric_evlist, evsel)
2140	evsel->default_metricgroup = true;
2141
2142	evlist__splice_list_tail(evlist, &evlist__core(metric_evlist)->entries);
2143	metricgroup__copy_metric_events(evlist, /cgrp=/NULL,
2144	evlist__metric_events(evlist),
2145	evlist__metric_events(metric_evlist));
2146	evlist__put(metric_evlist);
2147	}
2148	list_sort(/priv=/NULL, &evlist__core(evlist)->entries, default_evlist_evsel_cmp);
2149
2150	}
2151	out:
2152	if (!ret) {
2153	evlist__for_each_entry(evlist, evsel) {
2154	/*
2155	* Make at least one event non-skippable so fatal errors are visible.
2156	* 'cycles' always used to be default and non-skippable, so use that.
2157	*/
2158	if (!evsel__match(evsel, HARDWARE, HW_CPU_CYCLES))
2159	evsel->skippable = true;
2160	}
2161	}
2162	parse_events_error__exit(&err);
2163	evlist__splice_list_tail(evsel_list, &evlist__core(evlist)->entries);
2164	metricgroup__copy_metric_events(evsel_list, /cgrp=/NULL,
2165	evlist__metric_events(evsel_list),
2166	evlist__metric_events(evlist));
2167	evlist__put(evlist);
2168	return ret;
2169	}
2170
2171	static const char * const stat_record_usage[] = {
2172	"perf stat record [<options>]",
2173	NULL,
2174	};
2175
2176	static void init_features(struct perf_session *session)
2177	{
2178	int feat;
2179
2180	for (feat = HEADER_FIRST_FEATURE; feat < HEADER_LAST_FEATURE; feat++)
2181	perf_header__set_feat(&session->header, feat);
2182
2183	perf_header__clear_feat(&session->header, HEADER_DIR_FORMAT);
2184	perf_header__clear_feat(&session->header, HEADER_BUILD_ID);
2185	perf_header__clear_feat(&session->header, HEADER_TRACING_DATA);
2186	perf_header__clear_feat(&session->header, HEADER_BRANCH_STACK);
2187	perf_header__clear_feat(&session->header, HEADER_AUXTRACE);
2188	}
2189
2190	static int __cmd_record(const struct option stat_options[], struct opt_aggr_mode *opt_mode,
2191	int argc, const char **argv)
2192	{
2193	struct perf_session *session;
2194	struct perf_data *data = &perf_stat.data;
2195
2196	argc = parse_options(argc, argv, stat_options, stat_record_usage,
2197	PARSE_OPT_STOP_AT_NON_OPTION);
2198	stat_config.aggr_mode = opt_aggr_mode_to_aggr_mode(opt_mode);
2199
2200	if (output_name)
2201	data->path = output_name;
2202
2203	if (stat_config.run_count != 1 \|\| forever) {
2204	pr_err("Cannot use -r option with perf stat record.\n");
2205	return -1;
2206	}
2207
2208	session = perf_session__new(data, NULL);
2209	if (IS_ERR(session)) {
2210	pr_err("Perf session creation failed\n");
2211	return PTR_ERR(session);
2212	}
2213
2214	init_features(session);
2215
2216	session->evlist = evsel_list;
2217	perf_stat.session = session;
2218	perf_stat.record = true;
2219	return argc;
2220	}
2221
2222	static int process_stat_round_event(const struct perf_tool *tool __maybe_unused,
2223	struct perf_session *session,
2224	union perf_event *event)
2225	{
2226	struct perf_record_stat_round *stat_round = &event->stat_round;
2227	struct timespec tsh, *ts = NULL;
2228	struct perf_env *env = perf_session__env(session);
2229	const char **argv = env->cmdline_argv;
2230	int argc = env->nr_cmdline;
2231
2232	process_counters();
2233
2234	if (stat_round->type == PERF_STAT_ROUND_TYPE__FINAL)
2235	update_stats(stat_config.walltime_nsecs_stats, stat_round->time);
2236
2237	if (stat_config.interval && stat_round->time) {
2238	tsh.tv_sec = stat_round->time / NSEC_PER_SEC;
2239	tsh.tv_nsec = stat_round->time % NSEC_PER_SEC;
2240	ts = &tsh;
2241	}
2242
2243	print_counters(ts, argc, argv);
2244	return 0;
2245	}
2246
2247	static
2248	int process_stat_config_event(const struct perf_tool *tool,
2249	struct perf_session *session,
2250	union perf_event *event)
2251	{
2252	struct perf_stat *st = container_of(tool, struct perf_stat, tool);
2253
2254	perf_event__read_stat_config(&stat_config, &event->stat_config);
2255
2256	if (perf_cpu_map__is_empty(st->cpus)) {
2257	if (st->aggr_mode != AGGR_UNSET)
2258	pr_warning("warning: processing task data, aggregation mode not set\n");
2259	} else if (st->aggr_mode != AGGR_UNSET) {
2260	stat_config.aggr_mode = st->aggr_mode;
2261	}
2262
2263	if (perf_stat.data.is_pipe)
2264	perf_stat_init_aggr_mode();
2265	else
2266	perf_stat_init_aggr_mode_file(st);
2267
2268	if (stat_config.aggr_map) {
2269	int nr_aggr = stat_config.aggr_map->nr;
2270
2271	if (evlist__alloc_aggr_stats(session->evlist, nr_aggr) < 0) {
2272	pr_err("cannot allocate aggr counts\n");
2273	return -1;
2274	}
2275	}
2276	return 0;
2277	}
2278
2279	static int set_maps(struct perf_stat *st)
2280	{
2281	if (!st->cpus \|\| !st->threads)
2282	return 0;
2283
2284	if (WARN_ONCE(st->maps_allocated, "stats double allocation\n"))
2285	return -EINVAL;
2286
2287	perf_evlist__set_maps(evlist__core(evsel_list), st->cpus, st->threads);
2288
2289	if (evlist__alloc_stats(&stat_config, evsel_list, /alloc_raw=/true))
2290	return -ENOMEM;
2291
2292	st->maps_allocated = true;
2293	return 0;
2294	}
2295
2296	static
2297	int process_thread_map_event(const struct perf_tool *tool,
2298	struct perf_session *session __maybe_unused,
2299	union perf_event *event)
2300	{
2301	struct perf_stat *st = container_of(tool, struct perf_stat, tool);
2302
2303	if (st->threads) {
2304	pr_warning("Extra thread map event, ignoring.\n");
2305	return 0;
2306	}
2307
2308	st->threads = thread_map__new_event(&event->thread_map);
2309	if (!st->threads)
2310	return -ENOMEM;
2311
2312	return set_maps(st);
2313	}
2314
2315	static
2316	int process_cpu_map_event(const struct perf_tool *tool,
2317	struct perf_session *session __maybe_unused,
2318	union perf_event *event)
2319	{
2320	struct perf_stat *st = container_of(tool, struct perf_stat, tool);
2321	struct perf_cpu_map *cpus;
2322
2323	if (st->cpus) {
2324	pr_warning("Extra cpu map event, ignoring.\n");
2325	return 0;
2326	}
2327
2328	cpus = cpu_map__new_data(&event->cpu_map.data);
2329	if (!cpus)
2330	return -ENOMEM;
2331
2332	st->cpus = cpus;
2333	return set_maps(st);
2334	}
2335
2336	static const char * const stat_report_usage[] = {
2337	"perf stat report [<options>]",
2338	NULL,
2339	};
2340
2341	static struct perf_stat perf_stat = {
2342	.aggr_mode = AGGR_UNSET,
2343	.aggr_level = 0,
2344	};
2345
2346	static int __cmd_report(int argc, const char **argv)
2347	{
2348	struct perf_session *session;
2349	struct opt_aggr_mode opt_mode = {};
2350	const struct option options[] = {
2351	OPT_STRING('i', "input", &input_name, "file", "input file name"),
2352	OPT_BOOLEAN(0, "per-thread", &opt_mode.thread, "aggregate counts per thread"),
2353	OPT_BOOLEAN(0, "per-socket", &opt_mode.socket,
2354	"aggregate counts per processor socket"),
2355	OPT_BOOLEAN(0, "per-die", &opt_mode.die, "aggregate counts per processor die"),
2356	OPT_BOOLEAN(0, "per-cluster", &opt_mode.cluster,
2357	"aggregate counts per processor cluster"),
2358	OPT_CALLBACK_OPTARG(0, "per-cache", &opt_mode.cache, &perf_stat.aggr_level,
2359	"cache level", "aggregate count at this cache level (Default: LLC)",
2360	parse_cache_level),
2361	OPT_BOOLEAN(0, "per-core", &opt_mode.core,
2362	"aggregate counts per physical processor core"),
2363	OPT_BOOLEAN(0, "per-node", &opt_mode.node, "aggregate counts per numa node"),
2364	OPT_BOOLEAN('A', "no-aggr", &opt_mode.no_aggr,
2365	"disable aggregation across CPUs or PMUs"),
2366	OPT_END()
2367	};
2368	struct stat st;
2369	int ret;
2370
2371	argc = parse_options(argc, argv, options, stat_report_usage, 0);
2372
2373	perf_stat.aggr_mode = opt_aggr_mode_to_aggr_mode(&opt_mode);
2374	if (perf_stat.aggr_mode == AGGR_GLOBAL)
2375	perf_stat.aggr_mode = AGGR_UNSET; /* No option found so leave unset. */
2376
2377	if (!input_name \|\| !strlen(input_name)) {
2378	if (!fstat(STDIN_FILENO, &st) && S_ISFIFO(st.st_mode))
2379	input_name = "-";
2380	else
2381	input_name = "perf.data";
2382	}
2383
2384	perf_stat.data.path = input_name;
2385	perf_stat.data.mode = PERF_DATA_MODE_READ;
2386
2387	perf_tool__init(&perf_stat.tool, /ordered_events=/false);
2388	perf_stat.tool.attr = perf_event__process_attr;
2389	perf_stat.tool.event_update = perf_event__process_event_update;
2390	perf_stat.tool.thread_map = process_thread_map_event;
2391	perf_stat.tool.cpu_map = process_cpu_map_event;
2392	perf_stat.tool.stat_config = process_stat_config_event;
2393	perf_stat.tool.stat = perf_event__process_stat_event;
2394	perf_stat.tool.stat_round = process_stat_round_event;
2395
2396	session = perf_session__new(&perf_stat.data, &perf_stat.tool);
2397	if (IS_ERR(session))
2398	return PTR_ERR(session);
2399
2400	perf_stat.session = session;
2401	stat_config.output = stderr;
2402	evlist__put(evsel_list);
2403	evsel_list = session->evlist;
2404
2405	ret = perf_session__process_events(session);
2406	if (ret)
2407	return ret;
2408
2409	perf_session__delete(session);
2410	return 0;
2411	}
2412
2413	static void setup_system_wide(int forks)
2414	{
2415	/*
2416	* Make system wide (-a) the default target if
2417	* no target was specified and one of following
2418	* conditions is met:
2419	*
2420	* - there's no workload specified
2421	* - there is workload specified but all requested
2422	* events are system wide events
2423	*/
2424	if (!target__none(&target))
2425	return;
2426
2427	if (!forks)
2428	target.system_wide = true;
2429	else {
2430	struct evsel *counter;
2431
2432	evlist__for_each_entry(evsel_list, counter) {
2433	if (!counter->core.requires_cpu &&
2434	!evsel__name_is(counter, "duration_time")) {
2435	return;
2436	}
2437	}
2438
2439	if (evlist__nr_entries(evsel_list))
2440	target.system_wide = true;
2441	}
2442	}
2443
2444	#ifdef HAVE_ARCH_X86_64_SUPPORT
2445	static int parse_tpebs_mode(const struct option opt, const char str,
2446	int unset __maybe_unused)
2447	{
2448	enum tpebs_mode *mode = opt->value;
2449
2450	if (!strcasecmp("mean", str)) {
2451	*mode = TPEBS_MODE__MEAN;
2452	return 0;
2453	}
2454	if (!strcasecmp("min", str)) {
2455	*mode = TPEBS_MODE__MIN;
2456	return 0;
2457	}
2458	if (!strcasecmp("max", str)) {
2459	*mode = TPEBS_MODE__MAX;
2460	return 0;
2461	}
2462	if (!strcasecmp("last", str)) {
2463	*mode = TPEBS_MODE__LAST;
2464	return 0;
2465	}
2466	return -1;
2467	}
2468	#endif // HAVE_ARCH_X86_64_SUPPORT
2469
2470	int cmd_stat(int argc, const char **argv)
2471	{
2472	struct opt_aggr_mode opt_mode = {};
2473	bool affinity = true, affinity_set = false;
2474	struct option stat_options[] = {
2475	OPT_BOOLEAN('T', "transaction", &transaction_run,
2476	"hardware transaction statistics"),
2477	OPT_CALLBACK('e', "event", &parse_events_option_args, "event",
2478	"event selector. use 'perf list' to list available events",
2479	parse_events_option),
2480	OPT_CALLBACK(0, "filter", &evsel_list, "filter",
2481	"event filter", parse_filter),
2482	OPT_BOOLEAN('i', "no-inherit", &stat_config.no_inherit,
2483	"child tasks do not inherit counters"),
2484	OPT_STRING('p', "pid", &target.pid, "pid",
2485	"stat events on existing process id"),
2486	OPT_STRING('t', "tid", &target.tid, "tid",
2487	"stat events on existing thread id"),
2488	#ifdef HAVE_BPF_SKEL
2489	OPT_STRING('b', "bpf-prog", &target.bpf_str, "bpf-prog-id",
2490	"stat events on existing bpf program id"),
2491	OPT_BOOLEAN(0, "bpf-counters", &target.use_bpf,
2492	"use bpf program to count events"),
2493	OPT_STRING(0, "bpf-attr-map", &target.attr_map, "attr-map-path",
2494	"path to perf_event_attr map"),
2495	#endif
2496	OPT_BOOLEAN('a', "all-cpus", &target.system_wide,
2497	"system-wide collection from all CPUs"),
2498	OPT_BOOLEAN(0, "scale", &stat_config.scale,
2499	"Use --no-scale to disable counter scaling for multiplexing"),
2500	OPT_INCR('v', "verbose", &verbose,
2501	"be more verbose (show counter open errors, etc)"),
2502	OPT_INTEGER('r', "repeat", &stat_config.run_count,
2503	"repeat command and print average + stddev (max: 100, forever: 0)"),
2504	OPT_BOOLEAN(0, "table", &stat_config.walltime_run_table,
2505	"display details about each run (only with -r option)"),
2506	OPT_BOOLEAN('n', "null", &stat_config.null_run,
2507	"null run - dont start any counters"),
2508	OPT_BOOLEAN(0, "hide-zero-events", &stat_config.hide_zero,
2509	"Do not show events with a zero count"),
2510	OPT_INCR('d', "detailed", &detailed_run,
2511	"detailed run - start a lot of events"),
2512	OPT_BOOLEAN('S', "sync", &sync_run,
2513	"call sync() before starting a run"),
2514	OPT_CALLBACK_NOOPT('B', "big-num", NULL, NULL,
2515	"print large numbers with thousands\' separators",
2516	stat__set_big_num),
2517	OPT_STRING('C', "cpu", &target.cpu_list, "cpu",
2518	"list of cpus to monitor in system-wide"),
2519	OPT_BOOLEAN('A', "no-aggr", &opt_mode.no_aggr,
2520	"disable aggregation across CPUs or PMUs"),
2521	OPT_BOOLEAN(0, "no-merge", &opt_mode.no_aggr,
2522	"disable aggregation the same as -A or -no-aggr"),
2523	OPT_BOOLEAN(0, "hybrid-merge", &stat_config.hybrid_merge,
2524	"Merge identical named hybrid events"),
2525	OPT_STRING('x', "field-separator", &stat_config.csv_sep, "separator",
2526	"print counts with custom separator"),
2527	OPT_BOOLEAN('j', "json-output", &stat_config.json_output,
2528	"print counts in JSON format"),
2529	OPT_CALLBACK('G', "cgroup", &evsel_list, "name",
2530	"monitor event in cgroup name only", parse_stat_cgroups),
2531	OPT_STRING(0, "for-each-cgroup", &stat_config.cgroup_list, "name",
2532	"expand events for each cgroup"),
2533	OPT_STRING('o', "output", &output_name, "file", "output file name"),
2534	OPT_BOOLEAN(0, "append", &append_file, "append to the output file"),
2535	OPT_INTEGER(0, "log-fd", &output_fd,
2536	"log output to fd, instead of stderr"),
2537	OPT_STRING(0, "pre", &pre_cmd, "command",
2538	"command to run prior to the measured command"),
2539	OPT_STRING(0, "post", &post_cmd, "command",
2540	"command to run after to the measured command"),
2541	OPT_UINTEGER('I', "interval-print", &stat_config.interval,
2542	"print counts at regular interval in ms "
2543	"(overhead is possible for values <= 100ms)"),
2544	OPT_INTEGER(0, "interval-count", &stat_config.times,
2545	"print counts for fixed number of times"),
2546	OPT_BOOLEAN(0, "interval-clear", &stat_config.interval_clear,
2547	"clear screen in between new interval"),
2548	OPT_UINTEGER(0, "timeout", &stat_config.timeout,
2549	"stop workload and print counts after a timeout period in ms (>= 10ms)"),
2550	OPT_BOOLEAN(0, "per-socket", &opt_mode.socket,
2551	"aggregate counts per processor socket"),
2552	OPT_BOOLEAN(0, "per-die", &opt_mode.die, "aggregate counts per processor die"),
2553	OPT_BOOLEAN(0, "per-cluster", &opt_mode.cluster,
2554	"aggregate counts per processor cluster"),
2555	OPT_CALLBACK_OPTARG(0, "per-cache", &opt_mode.cache, &stat_config.aggr_level,
2556	"cache level", "aggregate count at this cache level (Default: LLC)",
2557	parse_cache_level),
2558	OPT_BOOLEAN(0, "per-core", &opt_mode.core,
2559	"aggregate counts per physical processor core"),
2560	OPT_BOOLEAN(0, "per-thread", &opt_mode.thread, "aggregate counts per thread"),
2561	OPT_BOOLEAN(0, "per-node", &opt_mode.node, "aggregate counts per numa node"),
2562	OPT_INTEGER('D', "delay", &target.initial_delay,
2563	"ms to wait before starting measurement after program start (-1: start with events disabled)"),
2564	OPT_CALLBACK_NOOPT(0, "metric-only", &stat_config.metric_only, NULL,
2565	"Only print computed metrics. No raw values", enable_metric_only),
2566	OPT_BOOLEAN(0, "metric-no-group", &stat_config.metric_no_group,
2567	"don't group metric events, impacts multiplexing"),
2568	OPT_BOOLEAN(0, "metric-no-merge", &stat_config.metric_no_merge,
2569	"don't try to share events between metrics in a group"),
2570	OPT_BOOLEAN(0, "metric-no-threshold", &stat_config.metric_no_threshold,
2571	"disable adding events for the metric threshold calculation"),
2572	OPT_BOOLEAN(0, "topdown", &topdown_run,
2573	"measure top-down statistics"),
2574	#ifdef HAVE_ARCH_X86_64_SUPPORT
2575	OPT_BOOLEAN(0, "record-tpebs", &tpebs_recording,
2576	"enable recording for tpebs when retire_latency required"),
2577	OPT_CALLBACK(0, "tpebs-mode", &tpebs_mode, "tpebs-mode",
2578	"Mode of TPEBS recording: mean, min or max",
2579	parse_tpebs_mode),
2580	#endif
2581	OPT_UINTEGER(0, "td-level", &stat_config.topdown_level,
2582	"Set the metrics level for the top-down statistics (0: max level)"),
2583	OPT_BOOLEAN(0, "smi-cost", &smi_cost,
2584	"measure SMI cost"),
2585	OPT_CALLBACK('M', "metrics", &evsel_list, "metric/metric group list",
2586	"monitor specified metrics or metric groups (separated by ,)",
2587	append_metric_groups),
2588	OPT_BOOLEAN_FLAG(0, "all-kernel", &stat_config.all_kernel,
2589	"Configure all used events to run in kernel space.",
2590	PARSE_OPT_EXCLUSIVE),
2591	OPT_BOOLEAN_FLAG(0, "all-user", &stat_config.all_user,
2592	"Configure all used events to run in user space.",
2593	PARSE_OPT_EXCLUSIVE),
2594	OPT_BOOLEAN(0, "percore-show-thread", &stat_config.percore_show_thread,
2595	"Use with 'percore' event qualifier to show the event "
2596	"counts of one hardware thread by sum up total hardware "
2597	"threads of same physical core"),
2598	OPT_BOOLEAN(0, "summary", &stat_config.summary,
2599	"print summary for interval mode"),
2600	OPT_BOOLEAN(0, "no-csv-summary", &stat_config.no_csv_summary,
2601	"don't print 'summary' for CSV summary output"),
2602	OPT_BOOLEAN(0, "quiet", &quiet,
2603	"don't print any output, messages or warnings (useful with record)"),
2604	OPT_BOOLEAN_SET(0, "affinity", &affinity, &affinity_set,
2605	"enable (default) or disable affinity optimizations to reduce IPIs"),
2606	OPT_CALLBACK(0, "cputype", &evsel_list, "hybrid cpu type",
2607	"Only enable events on applying cpu with this type "
2608	"for hybrid platform (e.g. core or atom)",
2609	parse_cputype),
2610	OPT_CALLBACK(0, "pmu-filter", &evsel_list, "pmu",
2611	"Only enable events on applying pmu with specified "
2612	"for multiple pmus with same type(e.g. hisi_sicl2_cpa0 or hisi_sicl0_cpa0)",
2613	parse_pmu_filter),
2614	#ifdef HAVE_LIBPFM
2615	OPT_CALLBACK(0, "pfm-events", &evsel_list, "event",
2616	"libpfm4 event selector. use 'perf list' to list available events",
2617	parse_libpfm_events_option),
2618	#endif
2619	OPT_CALLBACK(0, "control", &stat_config, "fd:ctl-fd[,ack-fd] or fifo:ctl-fifo[,ack-fifo]",
2620	"Listen on ctl-fd descriptor for command to control measurement ('enable': enable events, 'disable': disable events).\n"
2621	"\t\t\t Optionally send control command completion ('ack\\n') to ack-fd descriptor.\n"
2622	"\t\t\t Alternatively, ctl-fifo / ack-fifo will be opened and used as ctl-fd / ack-fd.",
2623	parse_control_option),
2624	OPT_CALLBACK_OPTARG(0, "iostat", &evsel_list, &stat_config, "default",
2625	"measure I/O performance metrics provided by arch/platform",
2626	iostat_parse),
2627	OPT_END()
2628	};
2629	const char * const stat_usage[] = {
2630	"perf stat [<options>] [<command>]",
2631	NULL
2632	};
2633	int status = -EINVAL, run_idx, err;
2634	const char *mode;
2635	FILE *output = stderr;
2636	unsigned int interval, timeout;
2637	const char * const stat_subcommands[] = { "record", "report" };
2638	char errbuf[BUFSIZ];
2639	struct evsel *counter;
2640
2641	setlocale(LC_ALL, "");
2642
2643	evsel_list = evlist__new();
2644	if (evsel_list == NULL)
2645	return -ENOMEM;
2646
2647	parse_events__shrink_config_terms();
2648
2649	/* String-parsing callback-based options would segfault when negated */
2650	set_option_flag(stat_options, 'e', "event", PARSE_OPT_NONEG);
2651	set_option_flag(stat_options, 'M', "metrics", PARSE_OPT_NONEG);
2652	set_option_flag(stat_options, 'G', "cgroup", PARSE_OPT_NONEG);
2653
2654	argc = parse_options_subcommand(argc, argv, stat_options, stat_subcommands,
2655	(const char **) stat_usage,
2656	PARSE_OPT_STOP_AT_NON_OPTION);
2657
2658	stat_config.aggr_mode = opt_aggr_mode_to_aggr_mode(&opt_mode);
2659
2660	if (stat_config.csv_sep && stat_config.json_output) {
2661	fprintf(stderr, "cannot use both --field-separator and --json-output\n");
2662	parse_options_usage(stat_usage, stat_options, "x", 1);
2663	parse_options_usage(NULL, stat_options, "j", 1);
2664	goto out;
2665	}
2666
2667	if (stat_config.csv_sep) {
2668	stat_config.csv_output = true;
2669	if (!strcmp(stat_config.csv_sep, "\\t"))
2670	stat_config.csv_sep = "\t";
2671	} else
2672	stat_config.csv_sep = DEFAULT_SEPARATOR;
2673
2674	if (affinity_set)
2675	evlist__set_no_affinity(evsel_list, !affinity);
2676
2677	if (argc && strlen(argv[0]) > 2 && strstarts("record", argv[0])) {
2678	argc = __cmd_record(stat_options, &opt_mode, argc, argv);
2679	if (argc < 0)
2680	return -1;
2681	} else if (argc && strlen(argv[0]) > 2 && strstarts("report", argv[0]))
2682	return __cmd_report(argc, argv);
2683
2684	interval = stat_config.interval;
2685	timeout = stat_config.timeout;
2686
2687	/*
2688	* For record command the -o is already taken care of.
2689	*/
2690	if (!STAT_RECORD && output_name && strcmp(output_name, "-"))
2691	output = NULL;
2692
2693	if (output_name && output_fd) {
2694	fprintf(stderr, "cannot use both --output and --log-fd\n");
2695	parse_options_usage(stat_usage, stat_options, "o", 1);
2696	parse_options_usage(NULL, stat_options, "log-fd", 0);
2697	goto out;
2698	}
2699
2700	if (stat_config.metric_only && stat_config.aggr_mode == AGGR_THREAD) {
2701	fprintf(stderr, "--metric-only is not supported with --per-thread\n");
2702	goto out;
2703	}
2704
2705	if (stat_config.metric_only && stat_config.run_count > 1) {
2706	fprintf(stderr, "--metric-only is not supported with -r\n");
2707	goto out;
2708	}
2709
2710	if (stat_config.csv_output \|\| (stat_config.metric_only && stat_config.json_output)) {
2711	/*
2712	* Current CSV and metric-only JSON output doesn't display the
2713	* metric threshold so don't compute it.
2714	*/
2715	stat_config.metric_no_threshold = true;
2716	}
2717
2718	if (stat_config.walltime_run_table && stat_config.run_count <= 1) {
2719	fprintf(stderr, "--table is only supported with -r\n");
2720	parse_options_usage(stat_usage, stat_options, "r", 1);
2721	parse_options_usage(NULL, stat_options, "table", 0);
2722	goto out;
2723	}
2724
2725	if (output_fd < 0) {
2726	fprintf(stderr, "argument to --log-fd must be a > 0\n");
2727	parse_options_usage(stat_usage, stat_options, "log-fd", 0);
2728	goto out;
2729	}
2730
2731	if (!output && !quiet) {
2732	struct timespec tm;
2733	mode = append_file ? "a" : "w";
2734
2735	output = fopen(output_name, mode);
2736	if (!output) {
2737	perror("failed to create output file");
2738	status = -1;
2739	goto out;
2740	}
2741	if (!stat_config.json_output) {
2742	clock_gettime(CLOCK_REALTIME, &tm);
2743	fprintf(output, "# started on %s\n", ctime(&tm.tv_sec));
2744	}
2745	} else if (output_fd > 0) {
2746	mode = append_file ? "a" : "w";
2747	output = fdopen(output_fd, mode);
2748	if (!output) {
2749	perror("Failed opening logfd");
2750	status = -errno;
2751	goto out;
2752	}
2753	}
2754
2755	if (stat_config.interval_clear && !isatty(fileno(output))) {
2756	fprintf(stderr, "--interval-clear does not work with output\n");
2757	parse_options_usage(stat_usage, stat_options, "o", 1);
2758	parse_options_usage(NULL, stat_options, "log-fd", 0);
2759	parse_options_usage(NULL, stat_options, "interval-clear", 0);
2760	status = -1;
2761	goto out;
2762	}
2763
2764	stat_config.output = output;
2765
2766	/*
2767	* let the spreadsheet do the pretty-printing
2768	*/
2769	if (stat_config.csv_output) {
2770	/* User explicitly passed -B? */
2771	if (big_num_opt == 1) {
2772	fprintf(stderr, "-B option not supported with -x\n");
2773	parse_options_usage(stat_usage, stat_options, "B", 1);
2774	parse_options_usage(NULL, stat_options, "x", 1);
2775	goto out;
2776	} else /* Nope, so disable big number formatting */
2777	stat_config.big_num = false;
2778	} else if (big_num_opt == 0) /* User passed --no-big-num */
2779	stat_config.big_num = false;
2780
2781	target.inherit = !stat_config.no_inherit;
2782	err = target__validate(&target);
2783	if (err) {
2784	target__strerror(&target, err, errbuf, BUFSIZ);
2785	pr_warning("%s\n", errbuf);
2786	}
2787
2788	setup_system_wide(argc);
2789
2790	/*
2791	* Display user/system times only for single
2792	* run and when there's specified tracee.
2793	*/
2794	if ((stat_config.run_count == 1) && target__none(&target))
2795	stat_config.ru_display = true;
2796
2797	if (stat_config.run_count < 0) {
2798	pr_err("Run count must be a positive number\n");
2799	parse_options_usage(stat_usage, stat_options, "r", 1);
2800	goto out;
2801	} else if (stat_config.run_count == 0) {
2802	forever = true;
2803	stat_config.run_count = 1;
2804	}
2805
2806	if (stat_config.walltime_run_table) {
2807	stat_config.walltime_run = calloc(stat_config.run_count, sizeof(stat_config.walltime_run[0]));
2808	if (!stat_config.walltime_run) {
2809	pr_err("failed to setup -r option");
2810	goto out;
2811	}
2812	}
2813
2814	if ((stat_config.aggr_mode == AGGR_THREAD) &&
2815	!target__has_task(&target)) {
2816	if (!target.system_wide \|\| target.cpu_list) {
2817	fprintf(stderr, "The --per-thread option is only "
2818	"available when monitoring via -p -t -a "
2819	"options or only --per-thread.\n");
2820	parse_options_usage(NULL, stat_options, "p", 1);
2821	parse_options_usage(NULL, stat_options, "t", 1);
2822	goto out;
2823	}
2824	}
2825
2826	/*
2827	* no_aggr, cgroup are for system-wide only
2828	* --per-thread is aggregated per thread, we dont mix it with cpu mode
2829	*/
2830	if (((stat_config.aggr_mode != AGGR_GLOBAL &&
2831	stat_config.aggr_mode != AGGR_THREAD) \|\|
2832	(nr_cgroups \|\| stat_config.cgroup_list)) &&
2833	!target__has_cpu(&target)) {
2834	fprintf(stderr, "both cgroup and no-aggregation "
2835	"modes only available in system-wide mode\n");
2836
2837	parse_options_usage(stat_usage, stat_options, "G", 1);
2838	parse_options_usage(NULL, stat_options, "A", 1);
2839	parse_options_usage(NULL, stat_options, "a", 1);
2840	parse_options_usage(NULL, stat_options, "for-each-cgroup", 0);
2841	goto out;
2842	}
2843
2844	if (stat_config.iostat_run) {
2845	status = iostat_prepare(&evsel_list, &stat_config);
2846	if (status)
2847	goto out;
2848	if (iostat_mode == IOSTAT_LIST) {
2849	iostat_list(evsel_list, &stat_config);
2850	goto out;
2851	} else if (verbose > 0)
2852	iostat_list(evsel_list, &stat_config);
2853	if (iostat_mode == IOSTAT_RUN && !target__has_cpu(&target))
2854	target.system_wide = true;
2855	}
2856
2857	if ((stat_config.aggr_mode == AGGR_THREAD) && (target.system_wide))
2858	target.per_thread = true;
2859
2860	stat_config.system_wide = target.system_wide;
2861	if (target.cpu_list) {
2862	stat_config.user_requested_cpu_list = strdup(target.cpu_list);
2863	if (!stat_config.user_requested_cpu_list) {
2864	status = -ENOMEM;
2865	goto out;
2866	}
2867	}
2868
2869	/*
2870	* Metric parsing needs to be delayed as metrics may optimize events
2871	* knowing the target is system-wide.
2872	*/
2873	if (metrics) {
2874	const char *pmu = parse_events_option_args.pmu_filter ?: "all";
2875	int ret = metricgroup__parse_groups(evsel_list, pmu,
2876	parse_events_option_args.cputype_filter,
2877	metrics,
2878	stat_config.metric_no_group,
2879	stat_config.metric_no_merge,
2880	stat_config.metric_no_threshold,
2881	stat_config.user_requested_cpu_list,
2882	stat_config.system_wide,
2883	stat_config.hardware_aware_grouping);
2884
2885	zfree(&metrics);
2886	if (ret) {
2887	status = ret;
2888	goto out;
2889	}
2890	}
2891
2892	if (add_default_events())
2893	goto out;
2894
2895	if (stat_config.cgroup_list) {
2896	if (nr_cgroups > 0) {
2897	pr_err("--cgroup and --for-each-cgroup cannot be used together\n");
2898	parse_options_usage(stat_usage, stat_options, "G", 1);
2899	parse_options_usage(NULL, stat_options, "for-each-cgroup", 0);
2900	goto out;
2901	}
2902
2903	if (evlist__expand_cgroup(evsel_list, stat_config.cgroup_list,
2904	!target.use_bpf) < 0) {
2905	parse_options_usage(stat_usage, stat_options,
2906	"for-each-cgroup", 0);
2907	goto out;
2908	}
2909	}
2910	#ifdef HAVE_BPF_SKEL
2911	if (target.use_bpf && nr_cgroups &&
2912	(evlist__nr_entries(evsel_list) / nr_cgroups) > BPERF_CGROUP__MAX_EVENTS) {
2913	pr_warning("Disabling BPF counters due to more events (%d) than the max (%d)\n",
2914	evlist__nr_entries(evsel_list) / nr_cgroups,
2915	BPERF_CGROUP__MAX_EVENTS);
2916	target.use_bpf = false;
2917	}
2918	#endif // HAVE_BPF_SKEL
2919	evlist__warn_user_requested_cpus(evsel_list, target.cpu_list);
2920
2921	evlist__for_each_entry(evsel_list, counter) {
2922	/*
2923	* Setup BPF counters to require CPUs as any(-1) isn't
2924	* supported. evlist__create_maps below will propagate this
2925	* information to the evsels. Note, evsel__is_bperf isn't yet
2926	* set up, and this change must happen early, so directly use
2927	* the bpf_counter variable and target information.
2928	*/
2929	if ((counter->bpf_counter \|\| target.use_bpf) && !target__has_cpu(&target))
2930	counter->core.requires_cpu = true;
2931	}
2932
2933	if (evlist__create_maps(evsel_list, &target) < 0) {
2934	if (target__has_task(&target)) {
2935	pr_err("Problems finding threads of monitor\n");
2936	parse_options_usage(stat_usage, stat_options, "p", 1);
2937	parse_options_usage(NULL, stat_options, "t", 1);
2938	} else if (target__has_cpu(&target)) {
2939	perror("failed to parse CPUs map");
2940	parse_options_usage(stat_usage, stat_options, "C", 1);
2941	parse_options_usage(NULL, stat_options, "a", 1);
2942	}
2943	goto out;
2944	}
2945
2946	evlist__check_cpu_maps(evsel_list);
2947
2948	/*
2949	* Initialize thread_map with comm names,
2950	* so we could print it out on output.
2951	*/
2952	if (stat_config.aggr_mode == AGGR_THREAD) {
2953	thread_map__read_comms(evlist__core(evsel_list)->threads);
2954	}
2955
2956	if (stat_config.aggr_mode == AGGR_NODE)
2957	cpu__setup_cpunode_map();
2958
2959	if (stat_config.times && interval)
2960	interval_count = true;
2961	else if (stat_config.times && !interval) {
2962	pr_err("interval-count option should be used together with "
2963	"interval-print.\n");
2964	parse_options_usage(stat_usage, stat_options, "interval-count", 0);
2965	parse_options_usage(stat_usage, stat_options, "I", 1);
2966	goto out;
2967	}
2968
2969	if (timeout && timeout < 100) {
2970	if (timeout < 10) {
2971	pr_err("timeout must be >= 10ms.\n");
2972	parse_options_usage(stat_usage, stat_options, "timeout", 0);
2973	goto out;
2974	} else
2975	pr_warning("timeout < 100ms. "
2976	"The overhead percentage could be high in some cases. "
2977	"Please proceed with caution.\n");
2978	}
2979	if (timeout && interval) {
2980	pr_err("timeout option is not supported with interval-print.\n");
2981	parse_options_usage(stat_usage, stat_options, "timeout", 0);
2982	parse_options_usage(stat_usage, stat_options, "I", 1);
2983	goto out;
2984	}
2985
2986	if (perf_stat_init_aggr_mode())
2987	goto out;
2988
2989	if (evlist__alloc_stats(&stat_config, evsel_list, interval))
2990	goto out;
2991
2992	/*
2993	* Set sample_type to PERF_SAMPLE_IDENTIFIER, which should be harmless
2994	* while avoiding that older tools show confusing messages.
2995	*
2996	* However for pipe sessions we need to keep it zero,
2997	* because script's perf_evsel__check_attr is triggered
2998	* by attr->sample_type != 0, and we can't run it on
2999	* stat sessions.
3000	*/
3001	stat_config.identifier = !(STAT_RECORD && perf_stat.data.is_pipe);
3002
3003	/*
3004	* We dont want to block the signals - that would cause
3005	* child tasks to inherit that and Ctrl-C would not work.
3006	* What we want is for Ctrl-C to work in the exec()-ed
3007	* task, but being ignored by perf stat itself:
3008	*/
3009	atexit(sig_atexit);
3010	if (!forever)
3011	signal(SIGINT, skip_signal);
3012	signal(SIGCHLD, skip_signal);
3013	signal(SIGALRM, skip_signal);
3014	signal(SIGABRT, skip_signal);
3015
3016	if (evlist__initialize_ctlfd(evsel_list, stat_config.ctl_fd, stat_config.ctl_fd_ack))
3017	goto out;
3018
3019	/* Enable ignoring missing threads when -p option is defined. */
3020	evlist__first(evsel_list)->ignore_missing_thread = target.pid;
3021	status = 0;
3022	for (run_idx = 0; forever \|\| run_idx < stat_config.run_count; run_idx++) {
3023	if (stat_config.run_count != 1 && verbose > 0)
3024	fprintf(output, "[ perf stat: executing run #%d ... ]\n",
3025	run_idx + 1);
3026
3027	if (run_idx != 0)
3028	evlist__reset_prev_raw_counts(evsel_list);
3029
3030	status = run_perf_stat(argc, argv, run_idx);
3031	if (status < 0)
3032	break;
3033
3034	if (forever && !interval) {
3035	print_counters(NULL, argc, argv);
3036	perf_stat__reset_stats();
3037	}
3038	}
3039
3040	if (!forever && status != -1 && (!interval \|\| stat_config.summary)) {
3041	if (stat_config.run_count > 1)
3042	evlist__copy_res_stats(&stat_config, evsel_list);
3043	print_counters(NULL, argc, argv);
3044	}
3045
3046	evlist__finalize_ctlfd(evsel_list);
3047
3048	if (STAT_RECORD) {
3049	/*
3050	* We synthesize the kernel mmap record just so that older tools
3051	* don't emit warnings about not being able to resolve symbols
3052	* due to /proc/sys/kernel/kptr_restrict settings and instead provide
3053	* a saner message about no samples being in the perf.data file.
3054	*
3055	* This also serves to suppress a warning about f_header.data.size == 0
3056	* in header.c at the moment 'perf stat record' gets introduced, which
3057	* is not really needed once we start adding the stat specific PERF_RECORD_
3058	* records, but the need to suppress the kptr_restrict messages in older
3059	* tools remain -acme
3060	*/
3061	int fd = perf_data__fd(&perf_stat.data);
3062
3063	err = perf_event__synthesize_kernel_mmap((void *)&perf_stat,
3064	process_synthesized_event,
3065	&perf_stat.session->machines.host);
3066	if (err) {
3067	pr_warning("Couldn't synthesize the kernel mmap record, harmless, "
3068	"older tools may produce warnings about this file\n.");
3069	}
3070
3071	if (!interval) {
3072	if (WRITE_STAT_ROUND_EVENT(stat_config.walltime_nsecs_stats->max, FINAL))
3073	pr_err("failed to write stat round event\n");
3074	}
3075
3076	if (!perf_stat.data.is_pipe) {
3077	perf_stat.session->header.data_size += perf_stat.bytes_written;
3078	perf_session__write_header(perf_stat.session, evsel_list, fd, true);
3079	}
3080
3081	evlist__close(evsel_list);
3082	perf_session__delete(perf_stat.session);
3083	}
3084
3085	perf_stat__exit_aggr_mode();
3086	evlist__free_stats(evsel_list);
3087	out:
3088	if (stat_config.iostat_run)
3089	iostat_release(evsel_list);
3090
3091	zfree(&stat_config.walltime_run);
3092	zfree(&stat_config.user_requested_cpu_list);
3093
3094	if (smi_cost && smi_reset)
3095	sysfs__write_int(FREEZE_ON_SMI_PATH, 0);
3096
3097	evlist__put(evsel_list);
3098
3099	evlist__close_control(stat_config.ctl_fd, stat_config.ctl_fd_ack, &stat_config.ctl_fd_close);
3100
3101	/* Only the low byte of status becomes the exit code. */
3102	return abs(status);
3103	}