projects/linux·blob master · refs

787 lines · 22.0 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Branch Record Buffer Extension Driver.
4 *
5 * Copyright (C) 2022-2025 ARM Limited
6 *
7 * Author: Anshuman Khandual <[email protected]>
8 */
9#include <linux/types.h>
10#include <linux/bitmap.h>
11#include <linux/perf/arm_pmu.h>
12#include "arm_brbe.h"
13
14#define BRBFCR_EL1_BRANCH_FILTERS (BRBFCR_EL1_DIRECT | \
15 BRBFCR_EL1_INDIRECT | \
16 BRBFCR_EL1_RTN | \
17 BRBFCR_EL1_INDCALL | \
18 BRBFCR_EL1_DIRCALL | \
19 BRBFCR_EL1_CONDDIR)
20
21/*
22 * BRBTS_EL1 is currently not used for branch stack implementation
23 * purpose but BRBCR_ELx.TS needs to have a valid value from all
24 * available options. BRBCR_ELx_TS_VIRTUAL is selected for this.
25 */
26#define BRBCR_ELx_DEFAULT_TS FIELD_PREP(BRBCR_ELx_TS_MASK, BRBCR_ELx_TS_VIRTUAL)
27
28/*
29 * BRBE Buffer Organization
30 *
31 * BRBE buffer is arranged as multiple banks of 32 branch record
32 * entries each. An individual branch record in a given bank could
33 * be accessed, after selecting the bank in BRBFCR_EL1.BANK and
34 * accessing the registers i.e [BRBSRC, BRBTGT, BRBINF] set with
35 * indices [0..31].
36 *
37 * Bank 0
38 *
39 * --------------------------------- ------
40 * | 00 | BRBSRC | BRBTGT | BRBINF | | 00 |
41 * --------------------------------- ------
42 * | 01 | BRBSRC | BRBTGT | BRBINF | | 01 |
43 * --------------------------------- ------
44 * | .. | BRBSRC | BRBTGT | BRBINF | | .. |
45 * --------------------------------- ------
46 * | 31 | BRBSRC | BRBTGT | BRBINF | | 31 |
47 * --------------------------------- ------
48 *
49 * Bank 1
50 *
51 * --------------------------------- ------
52 * | 32 | BRBSRC | BRBTGT | BRBINF | | 00 |
53 * --------------------------------- ------
54 * | 33 | BRBSRC | BRBTGT | BRBINF | | 01 |
55 * --------------------------------- ------
56 * | .. | BRBSRC | BRBTGT | BRBINF | | .. |
57 * --------------------------------- ------
58 * | 63 | BRBSRC | BRBTGT | BRBINF | | 31 |
59 * --------------------------------- ------
60 */
61#define BRBE_BANK_MAX_ENTRIES 32
62
63struct brbe_regset {
64 u64 brbsrc;
65 u64 brbtgt;
66 u64 brbinf;
67};
68
69#define PERF_BR_ARM64_MAX (PERF_BR_MAX + PERF_BR_NEW_MAX)
70
71struct brbe_hw_attr {
72 int brbe_version;
73 int brbe_cc;
74 int brbe_nr;
75 int brbe_format;
76};
77
78#define BRBE_REGN_CASE(n, case_macro) \
79 case n: case_macro(n); break
80
81#define BRBE_REGN_SWITCH(x, case_macro) \
82 do { \
83 switch (x) { \
84 BRBE_REGN_CASE(0, case_macro); \
85 BRBE_REGN_CASE(1, case_macro); \
86 BRBE_REGN_CASE(2, case_macro); \
87 BRBE_REGN_CASE(3, case_macro); \
88 BRBE_REGN_CASE(4, case_macro); \
89 BRBE_REGN_CASE(5, case_macro); \
90 BRBE_REGN_CASE(6, case_macro); \
91 BRBE_REGN_CASE(7, case_macro); \
92 BRBE_REGN_CASE(8, case_macro); \
93 BRBE_REGN_CASE(9, case_macro); \
94 BRBE_REGN_CASE(10, case_macro); \
95 BRBE_REGN_CASE(11, case_macro); \
96 BRBE_REGN_CASE(12, case_macro); \
97 BRBE_REGN_CASE(13, case_macro); \
98 BRBE_REGN_CASE(14, case_macro); \
99 BRBE_REGN_CASE(15, case_macro); \
100 BRBE_REGN_CASE(16, case_macro); \
101 BRBE_REGN_CASE(17, case_macro); \
102 BRBE_REGN_CASE(18, case_macro); \
103 BRBE_REGN_CASE(19, case_macro); \
104 BRBE_REGN_CASE(20, case_macro); \
105 BRBE_REGN_CASE(21, case_macro); \
106 BRBE_REGN_CASE(22, case_macro); \
107 BRBE_REGN_CASE(23, case_macro); \
108 BRBE_REGN_CASE(24, case_macro); \
109 BRBE_REGN_CASE(25, case_macro); \
110 BRBE_REGN_CASE(26, case_macro); \
111 BRBE_REGN_CASE(27, case_macro); \
112 BRBE_REGN_CASE(28, case_macro); \
113 BRBE_REGN_CASE(29, case_macro); \
114 BRBE_REGN_CASE(30, case_macro); \
115 BRBE_REGN_CASE(31, case_macro); \
116 default: WARN(1, "Invalid BRB* index %d\n", x); \
117 } \
118 } while (0)
119
120#define RETURN_READ_BRBSRCN(n) \
121 return read_sysreg_s(SYS_BRBSRC_EL1(n))
122static inline u64 get_brbsrc_reg(int idx)
123{
124 BRBE_REGN_SWITCH(idx, RETURN_READ_BRBSRCN);
125 return 0;
126}
127
128#define RETURN_READ_BRBTGTN(n) \
129 return read_sysreg_s(SYS_BRBTGT_EL1(n))
130static u64 get_brbtgt_reg(int idx)
131{
132 BRBE_REGN_SWITCH(idx, RETURN_READ_BRBTGTN);
133 return 0;
134}
135
136#define RETURN_READ_BRBINFN(n) \
137 return read_sysreg_s(SYS_BRBINF_EL1(n))
138static u64 get_brbinf_reg(int idx)
139{
140 BRBE_REGN_SWITCH(idx, RETURN_READ_BRBINFN);
141 return 0;
142}
143
144static u64 brbe_record_valid(u64 brbinf)
145{
146 return FIELD_GET(BRBINFx_EL1_VALID_MASK, brbinf);
147}
148
149static bool brbe_invalid(u64 brbinf)
150{
151 return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_NONE;
152}
153
154static bool brbe_record_is_complete(u64 brbinf)
155{
156 return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_FULL;
157}
158
159static bool brbe_record_is_source_only(u64 brbinf)
160{
161 return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_SOURCE;
162}
163
164static bool brbe_record_is_target_only(u64 brbinf)
165{
166 return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_TARGET;
167}
168
169static int brbinf_get_mispredict(u64 brbinf)
170{
171 return FIELD_GET(BRBINFx_EL1_MPRED_MASK, brbinf);
172}
173
174static u16 brbinf_get_cycles(u64 brbinf)
175{
176 u32 exp, mant, cycles;
177 /*
178 * Captured cycle count is unknown and hence
179 * should not be passed on to userspace.
180 */
181 if (brbinf & BRBINFx_EL1_CCU)
182 return 0;
183
184 exp = FIELD_GET(BRBINFx_EL1_CC_EXP_MASK, brbinf);
185 mant = FIELD_GET(BRBINFx_EL1_CC_MANT_MASK, brbinf);
186
187 if (!exp)
188 return mant;
189
190 cycles = (mant | 0x100) << (exp - 1);
191
192 return min(cycles, U16_MAX);
193}
194
195static int brbinf_get_type(u64 brbinf)
196{
197 return FIELD_GET(BRBINFx_EL1_TYPE_MASK, brbinf);
198}
199
200static int brbinf_get_el(u64 brbinf)
201{
202 return FIELD_GET(BRBINFx_EL1_EL_MASK, brbinf);
203}
204
205void brbe_invalidate(void)
206{
207 /* Ensure all branches before this point are recorded */
208 isb();
209 asm volatile(BRB_IALL_INSN);
210 /* Ensure all branch records are invalidated after this point */
211 isb();
212}
213
214static bool valid_brbe_nr(int brbe_nr)
215{
216 return brbe_nr == BRBIDR0_EL1_NUMREC_8 ||
217 brbe_nr == BRBIDR0_EL1_NUMREC_16 ||
218 brbe_nr == BRBIDR0_EL1_NUMREC_32 ||
219 brbe_nr == BRBIDR0_EL1_NUMREC_64;
220}
221
222static bool valid_brbe_cc(int brbe_cc)
223{
224 return brbe_cc == BRBIDR0_EL1_CC_20_BIT;
225}
226
227static bool valid_brbe_format(int brbe_format)
228{
229 return brbe_format == BRBIDR0_EL1_FORMAT_FORMAT_0;
230}
231
232static bool valid_brbidr(u64 brbidr)
233{
234 int brbe_format, brbe_cc, brbe_nr;
235
236 brbe_format = FIELD_GET(BRBIDR0_EL1_FORMAT_MASK, brbidr);
237 brbe_cc = FIELD_GET(BRBIDR0_EL1_CC_MASK, brbidr);
238 brbe_nr = FIELD_GET(BRBIDR0_EL1_NUMREC_MASK, brbidr);
239
240 return valid_brbe_format(brbe_format) && valid_brbe_cc(brbe_cc) && valid_brbe_nr(brbe_nr);
241}
242
243static bool valid_brbe_version(int brbe_version)
244{
245 return brbe_version == ID_AA64DFR0_EL1_BRBE_IMP ||
246 brbe_version == ID_AA64DFR0_EL1_BRBE_BRBE_V1P1;
247}
248
249static void select_brbe_bank(int bank)
250{
251 u64 brbfcr;
252
253 brbfcr = read_sysreg_s(SYS_BRBFCR_EL1);
254 brbfcr &= ~BRBFCR_EL1_BANK_MASK;
255 brbfcr |= SYS_FIELD_PREP(BRBFCR_EL1, BANK, bank);
256 write_sysreg_s(brbfcr, SYS_BRBFCR_EL1);
257 /*
258 * Arm ARM (DDI 0487K.a) D.18.4 rule PPBZP requires explicit sync
259 * between setting BANK and accessing branch records.
260 */
261 isb();
262}
263
264static bool __read_brbe_regset(struct brbe_regset *entry, int idx)
265{
266 entry->brbinf = get_brbinf_reg(idx);
267
268 if (brbe_invalid(entry->brbinf))
269 return false;
270
271 entry->brbsrc = get_brbsrc_reg(idx);
272 entry->brbtgt = get_brbtgt_reg(idx);
273 return true;
274}
275
276/*
277 * Generic perf branch filters supported on BRBE
278 *
279 * New branch filters need to be evaluated whether they could be supported on
280 * BRBE. This ensures that such branch filters would not just be accepted, to
281 * fail silently. PERF_SAMPLE_BRANCH_HV is a special case that is selectively
282 * supported only on platforms where kernel is in hyp mode.
283 */
284#define BRBE_EXCLUDE_BRANCH_FILTERS (PERF_SAMPLE_BRANCH_ABORT_TX | \
285 PERF_SAMPLE_BRANCH_IN_TX | \
286 PERF_SAMPLE_BRANCH_NO_TX | \
287 PERF_SAMPLE_BRANCH_CALL_STACK | \
288 PERF_SAMPLE_BRANCH_COUNTERS)
289
290#define BRBE_ALLOWED_BRANCH_TYPES (PERF_SAMPLE_BRANCH_ANY | \
291 PERF_SAMPLE_BRANCH_ANY_CALL | \
292 PERF_SAMPLE_BRANCH_ANY_RETURN | \
293 PERF_SAMPLE_BRANCH_IND_CALL | \
294 PERF_SAMPLE_BRANCH_COND | \
295 PERF_SAMPLE_BRANCH_IND_JUMP | \
296 PERF_SAMPLE_BRANCH_CALL)
297
298
299#define BRBE_ALLOWED_BRANCH_FILTERS (PERF_SAMPLE_BRANCH_USER | \
300 PERF_SAMPLE_BRANCH_KERNEL | \
301 PERF_SAMPLE_BRANCH_HV | \
302 BRBE_ALLOWED_BRANCH_TYPES | \
303 PERF_SAMPLE_BRANCH_NO_FLAGS | \
304 PERF_SAMPLE_BRANCH_NO_CYCLES | \
305 PERF_SAMPLE_BRANCH_TYPE_SAVE | \
306 PERF_SAMPLE_BRANCH_HW_INDEX | \
307 PERF_SAMPLE_BRANCH_PRIV_SAVE)
308
309#define BRBE_PERF_BRANCH_FILTERS (BRBE_ALLOWED_BRANCH_FILTERS | \
310 BRBE_EXCLUDE_BRANCH_FILTERS)
311
312/*
313 * BRBE supports the following functional branch type filters while
314 * generating branch records. These branch filters can be enabled,
315 * either individually or as a group i.e ORing multiple filters
316 * with each other.
317 *
318 * BRBFCR_EL1_CONDDIR - Conditional direct branch
319 * BRBFCR_EL1_DIRCALL - Direct call
320 * BRBFCR_EL1_INDCALL - Indirect call
321 * BRBFCR_EL1_INDIRECT - Indirect branch
322 * BRBFCR_EL1_DIRECT - Direct branch
323 * BRBFCR_EL1_RTN - Subroutine return
324 */
325static u64 branch_type_to_brbfcr(int branch_type)
326{
327 u64 brbfcr = 0;
328
329 if (branch_type & PERF_SAMPLE_BRANCH_ANY) {
330 brbfcr |= BRBFCR_EL1_BRANCH_FILTERS;
331 return brbfcr;
332 }
333
334 if (branch_type & PERF_SAMPLE_BRANCH_ANY_CALL) {
335 brbfcr |= BRBFCR_EL1_INDCALL;
336 brbfcr |= BRBFCR_EL1_DIRCALL;
337 }
338
339 if (branch_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
340 brbfcr |= BRBFCR_EL1_RTN;
341
342 if (branch_type & PERF_SAMPLE_BRANCH_IND_CALL)
343 brbfcr |= BRBFCR_EL1_INDCALL;
344
345 if (branch_type & PERF_SAMPLE_BRANCH_COND)
346 brbfcr |= BRBFCR_EL1_CONDDIR;
347
348 if (branch_type & PERF_SAMPLE_BRANCH_IND_JUMP)
349 brbfcr |= BRBFCR_EL1_INDIRECT;
350
351 if (branch_type & PERF_SAMPLE_BRANCH_CALL)
352 brbfcr |= BRBFCR_EL1_DIRCALL;
353
354 return brbfcr;
355}
356
357/*
358 * BRBE supports the following privilege mode filters while generating
359 * branch records.
360 *
361 * BRBCR_ELx_E0BRE - EL0 branch records
362 * BRBCR_ELx_ExBRE - EL1/EL2 branch records
363 *
364 * BRBE also supports the following additional functional branch type
365 * filters while generating branch records.
366 *
367 * BRBCR_ELx_EXCEPTION - Exception
368 * BRBCR_ELx_ERTN - Exception return
369 */
370static u64 branch_type_to_brbcr(int branch_type)
371{
372 u64 brbcr = BRBCR_ELx_FZP | BRBCR_ELx_DEFAULT_TS;
373
374 if (branch_type & PERF_SAMPLE_BRANCH_USER)
375 brbcr |= BRBCR_ELx_E0BRE;
376
377 /*
378 * When running in the hyp mode, writing into BRBCR_EL1
379 * actually writes into BRBCR_EL2 instead. Field E2BRE
380 * is also at the same position as E1BRE.
381 */
382 if (branch_type & PERF_SAMPLE_BRANCH_KERNEL)
383 brbcr |= BRBCR_ELx_ExBRE;
384
385 if (branch_type & PERF_SAMPLE_BRANCH_HV) {
386 if (is_kernel_in_hyp_mode())
387 brbcr |= BRBCR_ELx_ExBRE;
388 }
389
390 if (!(branch_type & PERF_SAMPLE_BRANCH_NO_CYCLES))
391 brbcr |= BRBCR_ELx_CC;
392
393 if (!(branch_type & PERF_SAMPLE_BRANCH_NO_FLAGS))
394 brbcr |= BRBCR_ELx_MPRED;
395
396 /*
397 * The exception and exception return branches could be
398 * captured, irrespective of the perf event's privilege.
399 * If the perf event does not have enough privilege for
400 * a given exception level, then addresses which falls
401 * under that exception level will be reported as zero
402 * for the captured branch record, creating source only
403 * or target only records.
404 */
405 if (branch_type & PERF_SAMPLE_BRANCH_KERNEL) {
406 if (branch_type & PERF_SAMPLE_BRANCH_ANY) {
407 brbcr |= BRBCR_ELx_EXCEPTION;
408 brbcr |= BRBCR_ELx_ERTN;
409 }
410
411 if (branch_type & PERF_SAMPLE_BRANCH_ANY_CALL)
412 brbcr |= BRBCR_ELx_EXCEPTION;
413
414 if (branch_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
415 brbcr |= BRBCR_ELx_ERTN;
416 }
417 return brbcr;
418}
419
420bool brbe_branch_attr_valid(struct perf_event *event)
421{
422 u64 branch_type = event->attr.branch_sample_type;
423
424 /*
425 * Ensure both perf branch filter allowed and exclude
426 * masks are always in sync with the generic perf ABI.
427 */
428 BUILD_BUG_ON(BRBE_PERF_BRANCH_FILTERS != (PERF_SAMPLE_BRANCH_MAX - 1));
429
430 if (branch_type & BRBE_EXCLUDE_BRANCH_FILTERS) {
431 pr_debug("requested branch filter not supported 0x%llx\n", branch_type);
432 return false;
433 }
434
435 /* Ensure at least 1 branch type is enabled */
436 if (!(branch_type & BRBE_ALLOWED_BRANCH_TYPES)) {
437 pr_debug("no branch type enabled 0x%llx\n", branch_type);
438 return false;
439 }
440
441 /*
442 * No branches are recorded in guests nor nVHE hypervisors, so
443 * excluding the host or both kernel and user is invalid.
444 *
445 * Ideally we'd just require exclude_guest and exclude_hv, but setting
446 * event filters with perf for kernel or user don't set exclude_guest.
447 * So effectively, exclude_guest and exclude_hv are ignored.
448 */
449 if (event->attr.exclude_host || (event->attr.exclude_user && event->attr.exclude_kernel)) {
450 pr_debug("branch filter in hypervisor or guest only not supported 0x%llx\n", branch_type);
451 return false;
452 }
453
454 event->hw.branch_reg.config = branch_type_to_brbfcr(event->attr.branch_sample_type);
455 event->hw.extra_reg.config = branch_type_to_brbcr(event->attr.branch_sample_type);
456
457 return true;
458}
459
460unsigned int brbe_num_branch_records(const struct arm_pmu *armpmu)
461{
462 return FIELD_GET(BRBIDR0_EL1_NUMREC_MASK, armpmu->reg_brbidr);
463}
464
465void brbe_probe(struct arm_pmu *armpmu)
466{
467 u64 brbidr, aa64dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1);
468 u32 brbe;
469
470 brbe = cpuid_feature_extract_unsigned_field(aa64dfr0, ID_AA64DFR0_EL1_BRBE_SHIFT);
471 if (!valid_brbe_version(brbe))
472 return;
473
474 brbidr = read_sysreg_s(SYS_BRBIDR0_EL1);
475 if (!valid_brbidr(brbidr))
476 return;
477
478 armpmu->reg_brbidr = brbidr;
479}
480
481/*
482 * BRBE is assumed to be disabled/paused on entry
483 */
484void brbe_enable(const struct arm_pmu *arm_pmu)
485{
486 struct pmu_hw_events *cpuc = this_cpu_ptr(arm_pmu->hw_events);
487 u64 brbfcr = 0, brbcr = 0;
488
489 /*
490 * Discard existing records to avoid a discontinuity, e.g. records
491 * missed during handling an overflow.
492 */
493 brbe_invalidate();
494
495 /*
496 * Merge the permitted branch filters of all events.
497 */
498 for (int i = 0; i < ARMPMU_MAX_HWEVENTS; i++) {
499 struct perf_event *event = cpuc->events[i];
500
501 if (event && has_branch_stack(event)) {
502 brbfcr |= event->hw.branch_reg.config;
503 brbcr |= event->hw.extra_reg.config;
504 }
505 }
506
507 /*
508 * In VHE mode with MDCR_EL2.HPMN equal to PMCR_EL0.N, BRBCR_EL1.FZP
509 * controls freezing the branch records on counter overflow rather than
510 * BRBCR_EL2.FZP (which writes to BRBCR_EL1 are redirected to).
511 * The exception levels are enabled/disabled in BRBCR_EL2, so keep EL1
512 * and EL0 recording disabled for guests.
513 *
514 * As BRBCR_EL1 CC and MPRED bits also need to match, use the same
515 * value for both registers just masking the exception levels.
516 */
517 if (is_kernel_in_hyp_mode())
518 write_sysreg_s(brbcr & ~(BRBCR_ELx_ExBRE | BRBCR_ELx_E0BRE), SYS_BRBCR_EL12);
519 write_sysreg_s(brbcr, SYS_BRBCR_EL1);
520 /* Ensure BRBCR_ELx settings take effect before unpausing */
521 isb();
522
523 /* Finally write SYS_BRBFCR_EL to unpause BRBE */
524 write_sysreg_s(brbfcr, SYS_BRBFCR_EL1);
525 /* Synchronization in PMCR write ensures ordering WRT PMU enabling */
526}
527
528void brbe_disable(void)
529{
530 /*
531 * No need for synchronization here as synchronization in PMCR write
532 * ensures ordering and in the interrupt handler this is a NOP as
533 * we're already paused.
534 */
535 write_sysreg_s(BRBFCR_EL1_PAUSED, SYS_BRBFCR_EL1);
536 write_sysreg_s(0, SYS_BRBCR_EL1);
537}
538
539static const int brbe_type_to_perf_type_map[BRBINFx_EL1_TYPE_DEBUG_EXIT + 1][2] = {
540 [BRBINFx_EL1_TYPE_DIRECT_UNCOND] = { PERF_BR_UNCOND, 0 },
541 [BRBINFx_EL1_TYPE_INDIRECT] = { PERF_BR_IND, 0 },
542 [BRBINFx_EL1_TYPE_DIRECT_LINK] = { PERF_BR_CALL, 0 },
543 [BRBINFx_EL1_TYPE_INDIRECT_LINK] = { PERF_BR_IND_CALL, 0 },
544 [BRBINFx_EL1_TYPE_RET] = { PERF_BR_RET, 0 },
545 [BRBINFx_EL1_TYPE_DIRECT_COND] = { PERF_BR_COND, 0 },
546 [BRBINFx_EL1_TYPE_CALL] = { PERF_BR_SYSCALL, 0 },
547 [BRBINFx_EL1_TYPE_ERET] = { PERF_BR_ERET, 0 },
548 [BRBINFx_EL1_TYPE_IRQ] = { PERF_BR_IRQ, 0 },
549 [BRBINFx_EL1_TYPE_TRAP] = { PERF_BR_IRQ, 0 },
550 [BRBINFx_EL1_TYPE_SERROR] = { PERF_BR_SERROR, 0 },
551 [BRBINFx_EL1_TYPE_ALIGN_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_ALGN },
552 [BRBINFx_EL1_TYPE_INSN_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_INST },
553 [BRBINFx_EL1_TYPE_DATA_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_DATA },
554};
555
556static void brbe_set_perf_entry_type(struct perf_branch_entry *entry, u64 brbinf)
557{
558 int brbe_type = brbinf_get_type(brbinf);
559
560 if (brbe_type <= BRBINFx_EL1_TYPE_DEBUG_EXIT) {
561 const int *br_type = brbe_type_to_perf_type_map[brbe_type];
562
563 entry->type = br_type[0];
564 entry->new_type = br_type[1];
565 }
566}
567
568static int brbinf_get_perf_priv(u64 brbinf)
569{
570 int brbe_el = brbinf_get_el(brbinf);
571
572 switch (brbe_el) {
573 case BRBINFx_EL1_EL_EL0:
574 return PERF_BR_PRIV_USER;
575 case BRBINFx_EL1_EL_EL1:
576 return PERF_BR_PRIV_KERNEL;
577 case BRBINFx_EL1_EL_EL2:
578 if (is_kernel_in_hyp_mode())
579 return PERF_BR_PRIV_KERNEL;
580 return PERF_BR_PRIV_HV;
581 default:
582 pr_warn_once("%d - unknown branch privilege captured\n", brbe_el);
583 return PERF_BR_PRIV_UNKNOWN;
584 }
585}
586
587static bool perf_entry_from_brbe_regset(int index, struct perf_branch_entry *entry,
588 const struct perf_event *event)
589{
590 struct brbe_regset bregs;
591 u64 brbinf;
592
593 if (!__read_brbe_regset(&bregs, index))
594 return false;
595
596 brbinf = bregs.brbinf;
597 *entry = (struct perf_branch_entry){ };
598 if (brbe_record_is_complete(brbinf)) {
599 entry->from = bregs.brbsrc;
600 entry->to = bregs.brbtgt;
601 } else if (brbe_record_is_source_only(brbinf)) {
602 entry->from = bregs.brbsrc;
603 entry->to = 0;
604 } else if (brbe_record_is_target_only(brbinf)) {
605 entry->from = 0;
606 entry->to = bregs.brbtgt;
607 }
608
609 brbe_set_perf_entry_type(entry, brbinf);
610
611 if (!branch_sample_no_cycles(event))
612 entry->cycles = brbinf_get_cycles(brbinf);
613
614 if (!branch_sample_no_flags(event)) {
615 /* Mispredict info is available for source only and complete branch records. */
616 if (!brbe_record_is_target_only(brbinf)) {
617 entry->mispred = brbinf_get_mispredict(brbinf);
618 entry->predicted = !entry->mispred;
619 }
620 }
621
622 /*
623 * Branch privilege level is available for target only and complete
624 * branch records.
625 */
626 if (!brbe_record_is_source_only(brbinf))
627 entry->priv = brbinf_get_perf_priv(brbinf);
628
629 return true;
630}
631
632#define PERF_BR_ARM64_ALL ( \
633 BIT(PERF_BR_COND) | \
634 BIT(PERF_BR_UNCOND) | \
635 BIT(PERF_BR_IND) | \
636 BIT(PERF_BR_CALL) | \
637 BIT(PERF_BR_IND_CALL) | \
638 BIT(PERF_BR_RET))
639
640#define PERF_BR_ARM64_ALL_KERNEL ( \
641 BIT(PERF_BR_SYSCALL) | \
642 BIT(PERF_BR_IRQ) | \
643 BIT(PERF_BR_SERROR) | \
644 BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_ALGN) | \
645 BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_DATA) | \
646 BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_INST))
647
648static void prepare_event_branch_type_mask(u64 branch_sample,
649 unsigned long *event_type_mask)
650{
651 if (branch_sample & PERF_SAMPLE_BRANCH_ANY) {
652 if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL)
653 bitmap_from_u64(event_type_mask,
654 BIT(PERF_BR_ERET) | PERF_BR_ARM64_ALL |
655 PERF_BR_ARM64_ALL_KERNEL);
656 else
657 bitmap_from_u64(event_type_mask, PERF_BR_ARM64_ALL);
658 return;
659 }
660
661 bitmap_zero(event_type_mask, PERF_BR_ARM64_MAX);
662
663 if (branch_sample & PERF_SAMPLE_BRANCH_ANY_CALL) {
664 if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL)
665 bitmap_from_u64(event_type_mask, PERF_BR_ARM64_ALL_KERNEL);
666
667 set_bit(PERF_BR_CALL, event_type_mask);
668 set_bit(PERF_BR_IND_CALL, event_type_mask);
669 }
670
671 if (branch_sample & PERF_SAMPLE_BRANCH_IND_JUMP)
672 set_bit(PERF_BR_IND, event_type_mask);
673
674 if (branch_sample & PERF_SAMPLE_BRANCH_COND)
675 set_bit(PERF_BR_COND, event_type_mask);
676
677 if (branch_sample & PERF_SAMPLE_BRANCH_CALL)
678 set_bit(PERF_BR_CALL, event_type_mask);
679
680 if (branch_sample & PERF_SAMPLE_BRANCH_IND_CALL)
681 set_bit(PERF_BR_IND_CALL, event_type_mask);
682
683 if (branch_sample & PERF_SAMPLE_BRANCH_ANY_RETURN) {
684 set_bit(PERF_BR_RET, event_type_mask);
685
686 if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL)
687 set_bit(PERF_BR_ERET, event_type_mask);
688 }
689}
690
691/*
692 * BRBE is configured with an OR of permissions from all events, so there may
693 * be events which have to be dropped or events where just the source or target
694 * address has to be zeroed.
695 */
696static bool filter_branch_privilege(struct perf_branch_entry *entry, u64 branch_sample_type)
697{
698 bool from_user = access_ok((void __user *)(unsigned long)entry->from, 4);
699 bool to_user = access_ok((void __user *)(unsigned long)entry->to, 4);
700 bool exclude_kernel = !((branch_sample_type & PERF_SAMPLE_BRANCH_KERNEL) ||
701 (is_kernel_in_hyp_mode() && (branch_sample_type & PERF_SAMPLE_BRANCH_HV)));
702
703 /* We can only have a half record if permissions have not been expanded */
704 if (!entry->from || !entry->to)
705 return true;
706
707 /*
708 * If record is within a single exception level, just need to either
709 * drop or keep the entire record.
710 */
711 if (from_user == to_user)
712 return ((entry->priv == PERF_BR_PRIV_KERNEL) && !exclude_kernel) ||
713 ((entry->priv == PERF_BR_PRIV_USER) &&
714 (branch_sample_type & PERF_SAMPLE_BRANCH_USER));
715
716 /*
717 * Record is across exception levels, mask addresses for the exception
718 * level we're not capturing.
719 */
720 if (!(branch_sample_type & PERF_SAMPLE_BRANCH_USER)) {
721 if (from_user)
722 entry->from = 0;
723 if (to_user)
724 entry->to = 0;
725 }
726
727 if (exclude_kernel) {
728 if (!from_user)
729 entry->from = 0;
730 if (!to_user)
731 entry->to = 0;
732 }
733
734 return true;
735}
736
737static bool filter_branch_type(struct perf_branch_entry *entry,
738 const unsigned long *event_type_mask)
739{
740 if (entry->type == PERF_BR_EXTEND_ABI)
741 return test_bit(PERF_BR_MAX + entry->new_type, event_type_mask);
742 else
743 return test_bit(entry->type, event_type_mask);
744}
745
746static bool filter_branch_record(struct perf_branch_entry *entry,
747 u64 branch_sample,
748 const unsigned long *event_type_mask)
749{
750 return filter_branch_type(entry, event_type_mask) &&
751 filter_branch_privilege(entry, branch_sample);
752}
753
754void brbe_read_filtered_entries(struct perf_branch_stack *branch_stack,
755 const struct perf_event *event)
756{
757 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
758 int nr_hw = brbe_num_branch_records(cpu_pmu);
759 int nr_banks = DIV_ROUND_UP(nr_hw, BRBE_BANK_MAX_ENTRIES);
760 int nr_filtered = 0;
761 u64 branch_sample_type = event->attr.branch_sample_type;
762 DECLARE_BITMAP(event_type_mask, PERF_BR_ARM64_MAX);
763
764 prepare_event_branch_type_mask(branch_sample_type, event_type_mask);
765
766 for (int bank = 0; bank < nr_banks; bank++) {
767 int nr_remaining = nr_hw - (bank * BRBE_BANK_MAX_ENTRIES);
768 int nr_this_bank = min(nr_remaining, BRBE_BANK_MAX_ENTRIES);
769
770 select_brbe_bank(bank);
771
772 for (int i = 0; i < nr_this_bank; i++) {
773 struct perf_branch_entry *pbe = &branch_stack->entries[nr_filtered];
774
775 if (!perf_entry_from_brbe_regset(i, pbe, event))
776 goto done;
777
778 if (!filter_branch_record(pbe, branch_sample_type, event_type_mask))
779 continue;
780
781 nr_filtered++;
782 }
783 }
784
785done:
786 branch_stack->nr = nr_filtered;
787}