projects/linux·blob master · refs

618 lines · 13.5 KiB

1// SPDX-License-Identifier: GPL-2.0
2#include <errno.h>
3#include <unistd.h>
4#include <sys/syscall.h>
5#include <perf/evsel.h>
6#include <perf/cpumap.h>
7#include <perf/threadmap.h>
8#include <linux/hash.h>
9#include <linux/list.h>
10#include <internal/evsel.h>
11#include <linux/zalloc.h>
12#include <stdlib.h>
13#include <internal/xyarray.h>
14#include <internal/cpumap.h>
15#include <internal/mmap.h>
16#include <internal/threadmap.h>
17#include <internal/lib.h>
18#include <linux/string.h>
19#include <sys/ioctl.h>
20#include <sys/mman.h>
21#include <asm/bug.h>
22
23void perf_evsel__init(struct perf_evsel *evsel, struct perf_event_attr *attr,
24 int idx)
25{
26 INIT_LIST_HEAD(&evsel->node);
27 INIT_LIST_HEAD(&evsel->per_stream_periods);
28 evsel->attr = *attr;
29 evsel->idx = idx;
30 evsel->leader = evsel;
31}
32
33struct perf_evsel *perf_evsel__new(struct perf_event_attr *attr)
34{
35 struct perf_evsel *evsel = zalloc(sizeof(*evsel));
36
37 if (evsel != NULL)
38 perf_evsel__init(evsel, attr, 0);
39
40 return evsel;
41}
42
43void perf_evsel__exit(struct perf_evsel *evsel)
44{
45 assert(evsel->fd == NULL); /* If not fds were not closed. */
46 assert(evsel->mmap == NULL); /* If not munmap wasn't called. */
47 assert(evsel->sample_id == NULL); /* If not free_id wasn't called. */
48 perf_cpu_map__put(evsel->cpus);
49 perf_cpu_map__put(evsel->pmu_cpus);
50 perf_thread_map__put(evsel->threads);
51}
52
53void perf_evsel__delete(struct perf_evsel *evsel)
54{
55 perf_evsel__exit(evsel);
56 free(evsel);
57}
58
59#define FD(_evsel, _cpu_map_idx, _thread) \
60 ((int *)xyarray__entry(_evsel->fd, _cpu_map_idx, _thread))
61#define MMAP(_evsel, _cpu_map_idx, _thread) \
62 (_evsel->mmap ? ((struct perf_mmap *) xyarray__entry(_evsel->mmap, _cpu_map_idx, _thread)) \
63 : NULL)
64
65int perf_evsel__alloc_fd(struct perf_evsel *evsel, int ncpus, int nthreads)
66{
67 evsel->fd = xyarray__new(ncpus, nthreads, sizeof(int));
68
69 if (evsel->fd) {
70 int idx, thread;
71
72 for (idx = 0; idx < ncpus; idx++) {
73 for (thread = 0; thread < nthreads; thread++) {
74 int *fd = FD(evsel, idx, thread);
75
76 if (fd)
77 *fd = -1;
78 }
79 }
80 }
81
82 return evsel->fd != NULL ? 0 : -ENOMEM;
83}
84
85static int perf_evsel__alloc_mmap(struct perf_evsel *evsel, int ncpus, int nthreads)
86{
87 evsel->mmap = xyarray__new(ncpus, nthreads, sizeof(struct perf_mmap));
88
89 return evsel->mmap != NULL ? 0 : -ENOMEM;
90}
91
92static int
93sys_perf_event_open(struct perf_event_attr *attr,
94 pid_t pid, struct perf_cpu cpu, int group_fd,
95 unsigned long flags)
96{
97 return syscall(__NR_perf_event_open, attr, pid, cpu.cpu, group_fd, flags);
98}
99
100static int get_group_fd(struct perf_evsel *evsel, int cpu_map_idx, int thread, int *group_fd)
101{
102 struct perf_evsel *leader = evsel->leader;
103 int *fd;
104
105 if (evsel == leader) {
106 *group_fd = -1;
107 return 0;
108 }
109
110 /*
111 * Leader must be already processed/open,
112 * if not it's a bug.
113 */
114 if (!leader->fd)
115 return -ENOTCONN;
116
117 fd = FD(leader, cpu_map_idx, thread);
118 if (fd == NULL || *fd == -1)
119 return -EBADF;
120
121 *group_fd = *fd;
122
123 return 0;
124}
125
126int perf_evsel__open(struct perf_evsel *evsel, struct perf_cpu_map *cpus,
127 struct perf_thread_map *threads)
128{
129 struct perf_cpu cpu;
130 unsigned int idx;
131 int thread, err = 0;
132
133 if (cpus == NULL) {
134 static struct perf_cpu_map *empty_cpu_map;
135
136 if (empty_cpu_map == NULL) {
137 empty_cpu_map = perf_cpu_map__new_any_cpu();
138 if (empty_cpu_map == NULL)
139 return -ENOMEM;
140 }
141
142 cpus = empty_cpu_map;
143 }
144
145 if (threads == NULL) {
146 static struct perf_thread_map *empty_thread_map;
147
148 if (empty_thread_map == NULL) {
149 empty_thread_map = perf_thread_map__new_dummy();
150 if (empty_thread_map == NULL)
151 return -ENOMEM;
152 }
153
154 threads = empty_thread_map;
155 }
156
157 if (evsel->fd == NULL &&
158 perf_evsel__alloc_fd(evsel, perf_cpu_map__nr(cpus), threads->nr) < 0)
159 return -ENOMEM;
160
161 perf_cpu_map__for_each_cpu(cpu, idx, cpus) {
162 for (thread = 0; thread < threads->nr; thread++) {
163 int fd, group_fd, *evsel_fd;
164
165 evsel_fd = FD(evsel, idx, thread);
166 if (evsel_fd == NULL) {
167 err = -EINVAL;
168 goto out;
169 }
170
171 err = get_group_fd(evsel, idx, thread, &group_fd);
172 if (err < 0)
173 goto out;
174
175 fd = sys_perf_event_open(&evsel->attr,
176 threads->map[thread].pid,
177 cpu, group_fd, 0);
178
179 if (fd < 0) {
180 err = -errno;
181 goto out;
182 }
183
184 *evsel_fd = fd;
185 }
186 }
187out:
188 if (err)
189 perf_evsel__close(evsel);
190
191 return err;
192}
193
194static void perf_evsel__close_fd_cpu(struct perf_evsel *evsel, int cpu_map_idx)
195{
196 int thread;
197
198 for (thread = 0; thread < xyarray__max_y(evsel->fd); ++thread) {
199 int *fd = FD(evsel, cpu_map_idx, thread);
200
201 if (fd && *fd >= 0) {
202 close(*fd);
203 *fd = -1;
204 }
205 }
206}
207
208void perf_evsel__close_fd(struct perf_evsel *evsel)
209{
210 for (int idx = 0; idx < xyarray__max_x(evsel->fd); idx++)
211 perf_evsel__close_fd_cpu(evsel, idx);
212}
213
214void perf_evsel__free_fd(struct perf_evsel *evsel)
215{
216 xyarray__delete(evsel->fd);
217 evsel->fd = NULL;
218}
219
220void perf_evsel__close(struct perf_evsel *evsel)
221{
222 if (evsel->fd == NULL)
223 return;
224
225 perf_evsel__close_fd(evsel);
226 perf_evsel__free_fd(evsel);
227}
228
229void perf_evsel__close_cpu(struct perf_evsel *evsel, int cpu_map_idx)
230{
231 if (evsel->fd == NULL)
232 return;
233
234 perf_evsel__close_fd_cpu(evsel, cpu_map_idx);
235}
236
237void perf_evsel__munmap(struct perf_evsel *evsel)
238{
239 int idx, thread;
240
241 if (evsel->fd == NULL || evsel->mmap == NULL)
242 return;
243
244 for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) {
245 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) {
246 int *fd = FD(evsel, idx, thread);
247
248 if (fd == NULL || *fd < 0)
249 continue;
250
251 perf_mmap__munmap(MMAP(evsel, idx, thread));
252 }
253 }
254
255 xyarray__delete(evsel->mmap);
256 evsel->mmap = NULL;
257}
258
259int perf_evsel__mmap(struct perf_evsel *evsel, int pages)
260{
261 int ret, idx, thread;
262 struct perf_mmap_param mp = {
263 .prot = PROT_READ | PROT_WRITE,
264 .mask = (pages * page_size) - 1,
265 };
266
267 if (evsel->fd == NULL || evsel->mmap)
268 return -EINVAL;
269
270 if (perf_evsel__alloc_mmap(evsel, xyarray__max_x(evsel->fd), xyarray__max_y(evsel->fd)) < 0)
271 return -ENOMEM;
272
273 for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) {
274 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) {
275 int *fd = FD(evsel, idx, thread);
276 struct perf_mmap *map;
277 struct perf_cpu cpu = perf_cpu_map__cpu(evsel->cpus, idx);
278
279 if (fd == NULL || *fd < 0)
280 continue;
281
282 map = MMAP(evsel, idx, thread);
283 perf_mmap__init(map, NULL, false, NULL);
284
285 ret = perf_mmap__mmap(map, &mp, *fd, cpu);
286 if (ret) {
287 perf_evsel__munmap(evsel);
288 return ret;
289 }
290 }
291 }
292
293 return 0;
294}
295
296void *perf_evsel__mmap_base(struct perf_evsel *evsel, int cpu_map_idx, int thread)
297{
298 int *fd = FD(evsel, cpu_map_idx, thread);
299
300 if (fd == NULL || *fd < 0 || MMAP(evsel, cpu_map_idx, thread) == NULL)
301 return NULL;
302
303 return MMAP(evsel, cpu_map_idx, thread)->base;
304}
305
306int perf_evsel__read_size(struct perf_evsel *evsel)
307{
308 u64 read_format = evsel->attr.read_format;
309 int entry = sizeof(u64); /* value */
310 int size = 0;
311 int nr = 1;
312
313 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
314 size += sizeof(u64);
315
316 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
317 size += sizeof(u64);
318
319 if (read_format & PERF_FORMAT_ID)
320 entry += sizeof(u64);
321
322 if (read_format & PERF_FORMAT_LOST)
323 entry += sizeof(u64);
324
325 if (read_format & PERF_FORMAT_GROUP) {
326 nr = evsel->nr_members;
327 size += sizeof(u64);
328 }
329
330 size += entry * nr;
331 return size;
332}
333
334/* This only reads values for the leader */
335static int perf_evsel__read_group(struct perf_evsel *evsel, int cpu_map_idx,
336 int thread, struct perf_counts_values *count)
337{
338 size_t size = perf_evsel__read_size(evsel);
339 int *fd = FD(evsel, cpu_map_idx, thread);
340 u64 read_format = evsel->attr.read_format;
341 u64 *data;
342 int idx = 1;
343
344 if (fd == NULL || *fd < 0)
345 return -EINVAL;
346
347 data = calloc(1, size);
348 if (data == NULL)
349 return -ENOMEM;
350
351 if (readn(*fd, data, size) <= 0) {
352 free(data);
353 return -errno;
354 }
355
356 /*
357 * This reads only the leader event intentionally since we don't have
358 * perf counts values for sibling events.
359 */
360 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
361 count->ena = data[idx++];
362 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
363 count->run = data[idx++];
364
365 /* value is always available */
366 count->val = data[idx++];
367 if (read_format & PERF_FORMAT_ID)
368 count->id = data[idx++];
369 if (read_format & PERF_FORMAT_LOST)
370 count->lost = data[idx++];
371
372 free(data);
373 return 0;
374}
375
376/*
377 * The perf read format is very flexible. It needs to set the proper
378 * values according to the read format.
379 */
380static void perf_evsel__adjust_values(struct perf_evsel *evsel, u64 *buf,
381 struct perf_counts_values *count)
382{
383 u64 read_format = evsel->attr.read_format;
384 int n = 0;
385
386 count->val = buf[n++];
387
388 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
389 count->ena = buf[n++];
390
391 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
392 count->run = buf[n++];
393
394 if (read_format & PERF_FORMAT_ID)
395 count->id = buf[n++];
396
397 if (read_format & PERF_FORMAT_LOST)
398 count->lost = buf[n++];
399}
400
401int perf_evsel__read(struct perf_evsel *evsel, int cpu_map_idx, int thread,
402 struct perf_counts_values *count)
403{
404 size_t size = perf_evsel__read_size(evsel);
405 int *fd = FD(evsel, cpu_map_idx, thread);
406 u64 read_format = evsel->attr.read_format;
407 struct perf_counts_values buf;
408
409 memset(count, 0, sizeof(*count));
410
411 if (fd == NULL || *fd < 0)
412 return -EINVAL;
413
414 if (read_format & PERF_FORMAT_GROUP)
415 return perf_evsel__read_group(evsel, cpu_map_idx, thread, count);
416
417 if (MMAP(evsel, cpu_map_idx, thread) &&
418 !(read_format & (PERF_FORMAT_ID | PERF_FORMAT_LOST)) &&
419 !perf_mmap__read_self(MMAP(evsel, cpu_map_idx, thread), count))
420 return 0;
421
422 if (readn(*fd, buf.values, size) <= 0)
423 return -errno;
424
425 perf_evsel__adjust_values(evsel, buf.values, count);
426 return 0;
427}
428
429static int perf_evsel__ioctl(struct perf_evsel *evsel, int ioc, void *arg,
430 int cpu_map_idx, int thread)
431{
432 int *fd = FD(evsel, cpu_map_idx, thread);
433
434 if (fd == NULL || *fd < 0)
435 return -1;
436
437 return ioctl(*fd, ioc, arg);
438}
439
440static int perf_evsel__run_ioctl(struct perf_evsel *evsel,
441 int ioc, void *arg,
442 int cpu_map_idx)
443{
444 int thread;
445
446 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) {
447 int err = perf_evsel__ioctl(evsel, ioc, arg, cpu_map_idx, thread);
448
449 if (err)
450 return err;
451 }
452
453 return 0;
454}
455
456int perf_evsel__enable_cpu(struct perf_evsel *evsel, int cpu_map_idx)
457{
458 return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, cpu_map_idx);
459}
460
461int perf_evsel__enable_thread(struct perf_evsel *evsel, int thread)
462{
463 struct perf_cpu cpu __maybe_unused;
464 unsigned int idx;
465 int err;
466
467 perf_cpu_map__for_each_cpu(cpu, idx, evsel->cpus) {
468 err = perf_evsel__ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, idx, thread);
469 if (err)
470 return err;
471 }
472
473 return 0;
474}
475
476int perf_evsel__enable(struct perf_evsel *evsel)
477{
478 int i;
479 int err = 0;
480
481 for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++)
482 err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, i);
483 return err;
484}
485
486int perf_evsel__disable_cpu(struct perf_evsel *evsel, int cpu_map_idx)
487{
488 return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, cpu_map_idx);
489}
490
491int perf_evsel__disable(struct perf_evsel *evsel)
492{
493 int i;
494 int err = 0;
495
496 for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++)
497 err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, i);
498 return err;
499}
500
501int perf_evsel__apply_filter(struct perf_evsel *evsel, const char *filter)
502{
503 int err = 0;
504
505 for (unsigned int i = 0; i < perf_cpu_map__nr(evsel->cpus) && !err; i++) {
506 err = perf_evsel__run_ioctl(evsel,
507 PERF_EVENT_IOC_SET_FILTER,
508 (void *)filter, i);
509 }
510 return err;
511}
512
513struct perf_cpu_map *perf_evsel__cpus(struct perf_evsel *evsel)
514{
515 return evsel->cpus;
516}
517
518struct perf_thread_map *perf_evsel__threads(struct perf_evsel *evsel)
519{
520 return evsel->threads;
521}
522
523struct perf_event_attr *perf_evsel__attr(struct perf_evsel *evsel)
524{
525 return &evsel->attr;
526}
527
528int perf_evsel__alloc_id(struct perf_evsel *evsel, int ncpus, int nthreads)
529{
530 if (ncpus == 0 || nthreads == 0)
531 return 0;
532
533 evsel->sample_id = xyarray__new(ncpus, nthreads, sizeof(struct perf_sample_id));
534 if (evsel->sample_id == NULL)
535 return -ENOMEM;
536
537 evsel->id = zalloc(ncpus * nthreads * sizeof(u64));
538 if (evsel->id == NULL) {
539 xyarray__delete(evsel->sample_id);
540 evsel->sample_id = NULL;
541 return -ENOMEM;
542 }
543
544 return 0;
545}
546
547void perf_evsel__free_id(struct perf_evsel *evsel)
548{
549 struct perf_sample_id_period *pos, *n;
550
551 xyarray__delete(evsel->sample_id);
552 evsel->sample_id = NULL;
553 zfree(&evsel->id);
554 evsel->ids = 0;
555
556 perf_evsel_for_each_per_thread_period_safe(evsel, n, pos) {
557 list_del_init(&pos->node);
558 free(pos);
559 }
560}
561
562bool perf_evsel__attr_has_per_thread_sample_period(struct perf_evsel *evsel)
563{
564 return (evsel->attr.sample_type & PERF_SAMPLE_READ) &&
565 (evsel->attr.sample_type & PERF_SAMPLE_TID) &&
566 evsel->attr.inherit;
567}
568
569u64 *perf_sample_id__get_period_storage(struct perf_sample_id *sid, u32 tid, bool per_thread)
570{
571 struct hlist_head *head;
572 struct perf_sample_id_period *res;
573 int hash;
574
575 if (!per_thread)
576 return &sid->period;
577
578 hash = hash_32(tid, PERF_SAMPLE_ID__HLIST_BITS);
579 head = &sid->periods[hash];
580
581 hlist_for_each_entry(res, head, hnode)
582 if (res->tid == tid)
583 return &res->period;
584
585 if (sid->evsel == NULL)
586 return NULL;
587
588 res = zalloc(sizeof(struct perf_sample_id_period));
589 if (res == NULL)
590 return NULL;
591
592 INIT_LIST_HEAD(&res->node);
593 res->tid = tid;
594
595 list_add_tail(&res->node, &sid->evsel->per_stream_periods);
596 hlist_add_head(&res->hnode, &sid->periods[hash]);
597
598 return &res->period;
599}
600
601void perf_counts_values__scale(struct perf_counts_values *count,
602 bool scale, __s8 *pscaled)
603{
604 s8 scaled = 0;
605
606 if (scale) {
607 if (count->run == 0) {
608 scaled = -1;
609 count->val = 0;
610 } else if (count->run < count->ena) {
611 scaled = 1;
612 count->val = (u64)((double)count->val * count->ena / count->run);
613 }
614 }
615
616 if (pscaled)
617 *pscaled = scaled;
618}