projects/linux·blob master · refs

725 lines · 16.5 KiB

1// SPDX-License-Identifier: GPL-2.0
2#include "stat.h"
3
4#include <errno.h>
5#include <inttypes.h>
6#include <math.h>
7#include <string.h>
8
9#include <linux/err.h>
10#include <linux/zalloc.h>
11
12#include "counts.h"
13#include "cpumap.h"
14#include "debug.h"
15#include "evlist.h"
16#include "evsel.h"
17#include "hashmap.h"
18#include "header.h"
19#include "pmu.h"
20#include "session.h"
21#include "target.h"
22#include "thread_map.h"
23
24void update_stats(struct stats *stats, u64 val)
25{
26 double delta;
27
28 stats->n++;
29 delta = val - stats->mean;
30 stats->mean += delta / stats->n;
31 stats->M2 += delta*(val - stats->mean);
32
33 if (val > stats->max)
34 stats->max = val;
35
36 if (val < stats->min)
37 stats->min = val;
38}
39
40double avg_stats(struct stats *stats)
41{
42 return stats->mean;
43}
44
45/*
46 * http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance
47 *
48 * (\Sum n_i^2) - ((\Sum n_i)^2)/n
49 * s^2 = -------------------------------
50 * n - 1
51 *
52 * http://en.wikipedia.org/wiki/Stddev
53 *
54 * The std dev of the mean is related to the std dev by:
55 *
56 * s
57 * s_mean = -------
58 * sqrt(n)
59 *
60 */
61double stddev_stats(struct stats *stats)
62{
63 double variance, variance_mean;
64
65 if (stats->n < 2)
66 return 0.0;
67
68 variance = stats->M2 / (stats->n - 1);
69 variance_mean = variance / stats->n;
70
71 return sqrt(variance_mean);
72}
73
74double rel_stddev_stats(double stddev, double avg)
75{
76 double pct = 0.0;
77
78 if (avg)
79 pct = 100.0 * stddev/avg;
80
81 return pct;
82}
83
84static void evsel__reset_aggr_stats(struct evsel *evsel)
85{
86 struct perf_stat_evsel *ps = evsel->stats;
87 struct perf_stat_aggr *aggr = ps->aggr;
88
89 if (aggr)
90 memset(aggr, 0, sizeof(*aggr) * ps->nr_aggr);
91}
92
93static void evsel__reset_stat_priv(struct evsel *evsel)
94{
95 struct perf_stat_evsel *ps = evsel->stats;
96
97 init_stats(&ps->res_stats);
98 evsel__reset_aggr_stats(evsel);
99}
100
101static int evsel__alloc_aggr_stats(struct evsel *evsel, int nr_aggr)
102{
103 struct perf_stat_evsel *ps = evsel->stats;
104
105 if (ps == NULL)
106 return 0;
107
108 ps->nr_aggr = nr_aggr;
109 ps->aggr = calloc(nr_aggr, sizeof(*ps->aggr));
110 if (ps->aggr == NULL)
111 return -ENOMEM;
112
113 return 0;
114}
115
116int evlist__alloc_aggr_stats(struct evlist *evlist, int nr_aggr)
117{
118 struct evsel *evsel;
119
120 evlist__for_each_entry(evlist, evsel) {
121 if (evsel__alloc_aggr_stats(evsel, nr_aggr) < 0)
122 return -1;
123 }
124 return 0;
125}
126
127static int evsel__alloc_stat_priv(struct evsel *evsel, int nr_aggr)
128{
129 struct perf_stat_evsel *ps;
130
131 ps = zalloc(sizeof(*ps));
132 if (ps == NULL)
133 return -ENOMEM;
134
135 evsel->stats = ps;
136
137 if (nr_aggr && evsel__alloc_aggr_stats(evsel, nr_aggr) < 0) {
138 evsel->stats = NULL;
139 free(ps);
140 return -ENOMEM;
141 }
142
143 evsel__reset_stat_priv(evsel);
144 return 0;
145}
146
147static void evsel__free_stat_priv(struct evsel *evsel)
148{
149 struct perf_stat_evsel *ps = evsel->stats;
150
151 if (ps) {
152 zfree(&ps->aggr);
153 zfree(&ps->group_data);
154 }
155 zfree(&evsel->stats);
156}
157
158static int evsel__alloc_prev_raw_counts(struct evsel *evsel)
159{
160 int cpu_map_nr = evsel__nr_cpus(evsel);
161 int nthreads = perf_thread_map__nr(evsel->core.threads);
162 struct perf_counts *counts;
163
164 counts = perf_counts__new(cpu_map_nr, nthreads);
165 if (counts)
166 evsel->prev_raw_counts = counts;
167
168 return counts ? 0 : -ENOMEM;
169}
170
171static void evsel__free_prev_raw_counts(struct evsel *evsel)
172{
173 perf_counts__delete(evsel->prev_raw_counts);
174 evsel->prev_raw_counts = NULL;
175}
176
177static void evsel__reset_prev_raw_counts(struct evsel *evsel)
178{
179 if (evsel->prev_raw_counts)
180 perf_counts__reset(evsel->prev_raw_counts);
181}
182
183static int evsel__alloc_stats(struct evsel *evsel, int nr_aggr, bool alloc_raw)
184{
185 if (evsel__alloc_stat_priv(evsel, nr_aggr) < 0 ||
186 evsel__alloc_counts(evsel) < 0 ||
187 (alloc_raw && evsel__alloc_prev_raw_counts(evsel) < 0))
188 return -ENOMEM;
189
190 return 0;
191}
192
193int evlist__alloc_stats(struct perf_stat_config *config,
194 struct evlist *evlist, bool alloc_raw)
195{
196 struct evsel *evsel;
197 int nr_aggr = 0;
198
199 if (config && config->aggr_map)
200 nr_aggr = config->aggr_map->nr;
201
202 evlist__for_each_entry(evlist, evsel) {
203 if (evsel__alloc_stats(evsel, nr_aggr, alloc_raw))
204 goto out_free;
205 }
206
207 return 0;
208
209out_free:
210 evlist__free_stats(evlist);
211 return -1;
212}
213
214void evlist__free_stats(struct evlist *evlist)
215{
216 struct evsel *evsel;
217
218 evlist__for_each_entry(evlist, evsel) {
219 evsel__free_stat_priv(evsel);
220 evsel__free_counts(evsel);
221 evsel__free_prev_raw_counts(evsel);
222 }
223}
224
225void evlist__reset_stats(struct evlist *evlist)
226{
227 struct evsel *evsel;
228
229 evlist__for_each_entry(evlist, evsel) {
230 evsel__reset_stat_priv(evsel);
231 evsel__reset_counts(evsel);
232 }
233}
234
235void evlist__reset_aggr_stats(struct evlist *evlist)
236{
237 struct evsel *evsel;
238
239 evlist__for_each_entry(evlist, evsel)
240 evsel__reset_aggr_stats(evsel);
241}
242
243void evlist__reset_prev_raw_counts(struct evlist *evlist)
244{
245 struct evsel *evsel;
246
247 evlist__for_each_entry(evlist, evsel)
248 evsel__reset_prev_raw_counts(evsel);
249}
250
251static void evsel__copy_prev_raw_counts(struct evsel *evsel)
252{
253 int nthreads = perf_thread_map__nr(evsel->core.threads);
254
255 for (int thread = 0; thread < nthreads; thread++) {
256 unsigned int idx;
257
258 perf_cpu_map__for_each_idx(idx, evsel__cpus(evsel)) {
259 *perf_counts(evsel->counts, idx, thread) =
260 *perf_counts(evsel->prev_raw_counts, idx, thread);
261 }
262 }
263}
264
265void evlist__copy_prev_raw_counts(struct evlist *evlist)
266{
267 struct evsel *evsel;
268
269 evlist__for_each_entry(evlist, evsel)
270 evsel__copy_prev_raw_counts(evsel);
271}
272
273static void evsel__copy_res_stats(struct evsel *evsel)
274{
275 struct perf_stat_evsel *ps = evsel->stats;
276
277 /*
278 * For GLOBAL aggregation mode, it updates the counts for each run
279 * in the evsel->stats.res_stats. See perf_stat_process_counter().
280 */
281 *ps->aggr[0].counts.values = avg_stats(&ps->res_stats);
282}
283
284void evlist__copy_res_stats(struct perf_stat_config *config, struct evlist *evlist)
285{
286 struct evsel *evsel;
287
288 if (config->aggr_mode != AGGR_GLOBAL)
289 return;
290
291 evlist__for_each_entry(evlist, evsel)
292 evsel__copy_res_stats(evsel);
293}
294
295static size_t pkg_id_hash(long __key, void *ctx __maybe_unused)
296{
297 uint64_t *key = (uint64_t *) __key;
298
299 return *key & 0xffffffff;
300}
301
302static bool pkg_id_equal(long __key1, long __key2, void *ctx __maybe_unused)
303{
304 uint64_t *key1 = (uint64_t *) __key1;
305 uint64_t *key2 = (uint64_t *) __key2;
306
307 return *key1 == *key2;
308}
309
310static int check_per_pkg(struct evsel *counter, struct perf_counts_values *vals,
311 int cpu_map_idx, bool *skip)
312{
313 struct hashmap *mask = counter->per_pkg_mask;
314 struct perf_cpu_map *cpus = evsel__cpus(counter);
315 struct perf_cpu cpu = perf_cpu_map__cpu(cpus, cpu_map_idx);
316 int s, d, ret = 0;
317 uint64_t *key;
318
319 *skip = false;
320
321 if (!counter->per_pkg)
322 return 0;
323
324 if (perf_cpu_map__is_any_cpu_or_is_empty(cpus))
325 return 0;
326
327 if (!mask) {
328 mask = hashmap__new(pkg_id_hash, pkg_id_equal, NULL);
329 if (IS_ERR(mask))
330 return -ENOMEM;
331
332 counter->per_pkg_mask = mask;
333 }
334
335 /*
336 * we do not consider an event that has not run as a good
337 * instance to mark a package as used (skip=1). Otherwise
338 * we may run into a situation where the first CPU in a package
339 * is not running anything, yet the second is, and this function
340 * would mark the package as used after the first CPU and would
341 * not read the values from the second CPU.
342 */
343 if (!(vals->run && vals->ena))
344 return 0;
345
346 s = cpu__get_socket_id(cpu);
347 if (s < 0)
348 return -1;
349
350 /*
351 * On multi-die system, die_id > 0. On no-die system, die_id = 0.
352 * We use hashmap(socket, die) to check the used socket+die pair.
353 */
354 d = cpu__get_die_id(cpu);
355 if (d < 0)
356 return -1;
357
358 key = malloc(sizeof(*key));
359 if (!key)
360 return -ENOMEM;
361
362 *key = (uint64_t)d << 32 | s;
363 if (hashmap__find(mask, key, NULL)) {
364 *skip = true;
365 free(key);
366 } else
367 ret = hashmap__add(mask, key, 1);
368
369 return ret;
370}
371
372static bool evsel__count_has_error(struct evsel *evsel,
373 struct perf_counts_values *count,
374 struct perf_stat_config *config)
375{
376 /* the evsel was failed already */
377 if (evsel->err || evsel->counts->scaled == -1)
378 return true;
379
380 /* this is meaningful for CPU aggregation modes only */
381 if (config->aggr_mode == AGGR_GLOBAL)
382 return false;
383
384 /* it's considered ok when it actually ran */
385 if (count->ena != 0 && count->run != 0)
386 return false;
387
388 return true;
389}
390
391static int
392process_counter_values(struct perf_stat_config *config, struct evsel *evsel,
393 int cpu_map_idx, int thread,
394 struct perf_counts_values *count)
395{
396 struct perf_stat_evsel *ps = evsel->stats;
397 static struct perf_counts_values zero;
398 bool skip = false;
399
400 if (check_per_pkg(evsel, count, cpu_map_idx, &skip)) {
401 pr_err("failed to read per-pkg counter\n");
402 return -1;
403 }
404
405 if (skip)
406 count = &zero;
407
408 if (!evsel->snapshot)
409 evsel__compute_deltas(evsel, cpu_map_idx, thread, count);
410 perf_counts_values__scale(count, config->scale, NULL);
411
412 if (config->aggr_mode == AGGR_THREAD) {
413 struct perf_counts_values *aggr_counts = &ps->aggr[thread].counts;
414
415 /*
416 * Skip value 0 when enabling --per-thread globally,
417 * otherwise too many 0 output.
418 */
419 if (count->val == 0 && config->system_wide)
420 return 0;
421
422 ps->aggr[thread].nr++;
423
424 aggr_counts->val += count->val;
425 aggr_counts->ena += count->ena;
426 aggr_counts->run += count->run;
427 return 0;
428 }
429
430 if (ps->aggr) {
431 struct perf_cpu cpu = perf_cpu_map__cpu(evsel->core.cpus, cpu_map_idx);
432 struct aggr_cpu_id aggr_id = config->aggr_get_id(config, cpu);
433 struct perf_stat_aggr *ps_aggr;
434 int i;
435
436 for (i = 0; i < ps->nr_aggr; i++) {
437 if (!aggr_cpu_id__equal(&aggr_id, &config->aggr_map->map[i]))
438 continue;
439
440 ps_aggr = &ps->aggr[i];
441 ps_aggr->nr++;
442
443 /*
444 * When any result is bad, make them all to give consistent output
445 * in interval mode. But per-task counters can have 0 enabled time
446 * when some tasks are idle.
447 */
448 if (evsel__count_has_error(evsel, count, config) && !ps_aggr->failed) {
449 ps_aggr->counts.val = 0;
450 ps_aggr->counts.ena = 0;
451 ps_aggr->counts.run = 0;
452 ps_aggr->failed = true;
453 }
454
455 if (!ps_aggr->failed) {
456 ps_aggr->counts.val += count->val;
457 ps_aggr->counts.ena += count->ena;
458 ps_aggr->counts.run += count->run;
459 }
460 break;
461 }
462 }
463
464 return 0;
465}
466
467static int process_counter_maps(struct perf_stat_config *config,
468 struct evsel *counter)
469{
470 int nthreads = perf_thread_map__nr(counter->core.threads);
471 int ncpus = evsel__nr_cpus(counter);
472 int idx, thread;
473
474 for (thread = 0; thread < nthreads; thread++) {
475 for (idx = 0; idx < ncpus; idx++) {
476 if (process_counter_values(config, counter, idx, thread,
477 perf_counts(counter->counts, idx, thread)))
478 return -1;
479 }
480 }
481
482 return 0;
483}
484
485int perf_stat_process_counter(struct perf_stat_config *config,
486 struct evsel *counter)
487{
488 struct perf_stat_evsel *ps = counter->stats;
489 u64 *count;
490 int ret;
491
492 if (counter->per_pkg)
493 evsel__zero_per_pkg(counter);
494
495 ret = process_counter_maps(config, counter);
496 if (ret)
497 return ret;
498
499 if (config->aggr_mode != AGGR_GLOBAL)
500 return 0;
501
502 /*
503 * GLOBAL aggregation mode only has a single aggr counts,
504 * so we can use ps->aggr[0] as the actual output.
505 */
506 count = ps->aggr[0].counts.values;
507 update_stats(&ps->res_stats, *count);
508
509 if (verbose > 0) {
510 fprintf(config->output, "%s: %" PRIu64 " %" PRIu64 " %" PRIu64 "\n",
511 evsel__name(counter), count[0], count[1], count[2]);
512 }
513
514 return 0;
515}
516
517static int evsel__merge_aggr_counters(struct evsel *evsel, struct evsel *alias)
518{
519 struct perf_stat_evsel *ps_a = evsel->stats;
520 struct perf_stat_evsel *ps_b = alias->stats;
521 int i;
522
523 if (ps_a->aggr == NULL && ps_b->aggr == NULL)
524 return 0;
525
526 if (ps_a->nr_aggr != ps_b->nr_aggr) {
527 pr_err("Unmatched aggregation mode between aliases\n");
528 return -1;
529 }
530
531 for (i = 0; i < ps_a->nr_aggr; i++) {
532 struct perf_counts_values *aggr_counts_a = &ps_a->aggr[i].counts;
533 struct perf_counts_values *aggr_counts_b = &ps_b->aggr[i].counts;
534
535 ps_a->aggr[i].nr += ps_b->aggr[i].nr;
536
537 aggr_counts_a->val += aggr_counts_b->val;
538 aggr_counts_a->ena += aggr_counts_b->ena;
539 aggr_counts_a->run += aggr_counts_b->run;
540 }
541
542 return 0;
543}
544
545static void evsel__merge_aliases(struct evsel *evsel)
546{
547 struct evlist *evlist = evsel->evlist;
548 struct evsel *alias;
549
550 alias = list_prepare_entry(evsel, &(evlist__core(evlist)->entries), core.node);
551 list_for_each_entry_continue(alias, &evlist__core(evlist)->entries, core.node) {
552 if (alias->first_wildcard_match == evsel) {
553 /* Merge the same events on different PMUs. */
554 evsel__merge_aggr_counters(evsel, alias);
555 }
556 }
557}
558
559static bool evsel__should_merge_hybrid(const struct evsel *evsel,
560 const struct perf_stat_config *config)
561{
562 return config->hybrid_merge && evsel__is_hybrid(evsel);
563}
564
565static void evsel__merge_stats(struct evsel *evsel, struct perf_stat_config *config)
566{
567 if (!evsel->pmu || !evsel->pmu->is_core || evsel__should_merge_hybrid(evsel, config))
568 evsel__merge_aliases(evsel);
569}
570
571/* merge the same uncore and hybrid events if requested */
572void perf_stat_merge_counters(struct perf_stat_config *config, struct evlist *evlist)
573{
574 struct evsel *evsel;
575
576 if (config->aggr_mode == AGGR_NONE)
577 return;
578
579 evlist__for_each_entry(evlist, evsel)
580 evsel__merge_stats(evsel, config);
581}
582
583static void evsel__update_percore_stats(struct evsel *evsel, struct aggr_cpu_id *core_id)
584{
585 struct perf_stat_evsel *ps = evsel->stats;
586 struct perf_counts_values counts = { 0, };
587 struct aggr_cpu_id id;
588 struct perf_cpu cpu;
589 unsigned int idx;
590
591 /* collect per-core counts */
592 perf_cpu_map__for_each_cpu(cpu, idx, evsel->core.cpus) {
593 struct perf_stat_aggr *aggr = &ps->aggr[idx];
594
595 id = aggr_cpu_id__core(cpu, NULL);
596 if (!aggr_cpu_id__equal(core_id, &id))
597 continue;
598
599 counts.val += aggr->counts.val;
600 counts.ena += aggr->counts.ena;
601 counts.run += aggr->counts.run;
602 }
603
604 /* update aggregated per-core counts for each CPU */
605 perf_cpu_map__for_each_cpu(cpu, idx, evsel->core.cpus) {
606 struct perf_stat_aggr *aggr = &ps->aggr[idx];
607
608 id = aggr_cpu_id__core(cpu, NULL);
609 if (!aggr_cpu_id__equal(core_id, &id))
610 continue;
611
612 aggr->counts.val = counts.val;
613 aggr->counts.ena = counts.ena;
614 aggr->counts.run = counts.run;
615
616 aggr->used = true;
617 }
618}
619
620/* we have an aggr_map for cpu, but want to aggregate the counters per-core */
621static void evsel__process_percore(struct evsel *evsel)
622{
623 struct perf_stat_evsel *ps = evsel->stats;
624 struct aggr_cpu_id core_id;
625 struct perf_cpu cpu;
626 unsigned int idx;
627
628 if (!evsel->percore)
629 return;
630
631 perf_cpu_map__for_each_cpu(cpu, idx, evsel->core.cpus) {
632 struct perf_stat_aggr *aggr = &ps->aggr[idx];
633
634 if (aggr->used)
635 continue;
636
637 core_id = aggr_cpu_id__core(cpu, NULL);
638 evsel__update_percore_stats(evsel, &core_id);
639 }
640}
641
642/* process cpu stats on per-core events */
643void perf_stat_process_percore(struct perf_stat_config *config, struct evlist *evlist)
644{
645 struct evsel *evsel;
646
647 if (config->aggr_mode != AGGR_NONE)
648 return;
649
650 evlist__for_each_entry(evlist, evsel)
651 evsel__process_percore(evsel);
652}
653
654int perf_event__process_stat_event(const struct perf_tool *tool __maybe_unused,
655 struct perf_session *session,
656 union perf_event *event)
657{
658 struct perf_counts_values count, *ptr;
659 struct perf_record_stat *st = &event->stat;
660 struct evsel *counter;
661 int cpu_map_idx;
662
663 count.val = st->val;
664 count.ena = st->ena;
665 count.run = st->run;
666
667 counter = evlist__id2evsel(session->evlist, st->id);
668 if (!counter) {
669 pr_err("Failed to resolve counter for stat event.\n");
670 return -EINVAL;
671 }
672 cpu_map_idx = perf_cpu_map__idx(evsel__cpus(counter), (struct perf_cpu){.cpu = st->cpu});
673 if (cpu_map_idx == -1) {
674 pr_err("Invalid CPU %d for event %s.\n", st->cpu, evsel__name(counter));
675 return -EINVAL;
676 }
677 ptr = perf_counts(counter->counts, cpu_map_idx, st->thread);
678 if (ptr == NULL) {
679 pr_err("Failed to find perf count for CPU %d thread %d on event %s.\n",
680 st->cpu, st->thread, evsel__name(counter));
681 return -EINVAL;
682 }
683 *ptr = count;
684 counter->supported = true;
685 return 0;
686}
687
688size_t perf_event__fprintf_stat(union perf_event *event, FILE *fp)
689{
690 struct perf_record_stat *st = (struct perf_record_stat *)event;
691 size_t ret;
692
693 ret = fprintf(fp, "\n... id %" PRI_lu64 ", cpu %d, thread %d\n",
694 st->id, st->cpu, st->thread);
695 ret += fprintf(fp, "... value %" PRI_lu64 ", enabled %" PRI_lu64 ", running %" PRI_lu64 "\n",
696 st->val, st->ena, st->run);
697
698 return ret;
699}
700
701size_t perf_event__fprintf_stat_round(union perf_event *event, FILE *fp)
702{
703 struct perf_record_stat_round *rd = (struct perf_record_stat_round *)event;
704 size_t ret;
705
706 ret = fprintf(fp, "\n... time %" PRI_lu64 ", type %s\n", rd->time,
707 rd->type == PERF_STAT_ROUND_TYPE__FINAL ? "FINAL" : "INTERVAL");
708
709 return ret;
710}
711
712size_t perf_event__fprintf_stat_config(union perf_event *event, FILE *fp)
713{
714 struct perf_stat_config sc = {};
715 size_t ret;
716
717 perf_event__read_stat_config(&sc, &event->stat_config);
718
719 ret = fprintf(fp, "\n");
720 ret += fprintf(fp, "... aggr_mode %d\n", sc.aggr_mode);
721 ret += fprintf(fp, "... scale %d\n", sc.scale);
722 ret += fprintf(fp, "... interval %u\n", sc.interval);
723
724 return ret;
725}