projects/linux·blob uaccess-cleanups · refs

1986 lines · 50.7 KiB

1// SPDX-License-Identifier: GPL-2.0
2// Copyright (C) 2025 Arm Ltd.
3
4#define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__
5
6#include <linux/arm_mpam.h>
7#include <linux/cacheinfo.h>
8#include <linux/cpu.h>
9#include <linux/cpumask.h>
10#include <linux/errno.h>
11#include <linux/limits.h>
12#include <linux/list.h>
13#include <linux/math.h>
14#include <linux/printk.h>
15#include <linux/rculist.h>
16#include <linux/resctrl.h>
17#include <linux/slab.h>
18#include <linux/types.h>
19#include <linux/wait.h>
20
21#include <asm/mpam.h>
22
23#include "mpam_internal.h"
24
25static DECLARE_WAIT_QUEUE_HEAD(resctrl_mon_ctx_waiters);
26
27/*
28 * The classes we've picked to map to resctrl resources, wrapped
29 * in with their resctrl structure.
30 * Class pointer may be NULL.
31 */
32static struct mpam_resctrl_res mpam_resctrl_controls[RDT_NUM_RESOURCES];
33
34#define for_each_mpam_resctrl_control(res, rid) \
35 for (rid = 0, res = &mpam_resctrl_controls[rid]; \
36 rid < RDT_NUM_RESOURCES; \
37 rid++, res = &mpam_resctrl_controls[rid])
38
39/*
40 * The classes we've picked to map to resctrl events.
41 * Resctrl believes all the worlds a Xeon, and these are all on the L3. This
42 * array lets us find the actual class backing the event counters. e.g.
43 * the only memory bandwidth counters may be on the memory controller, but to
44 * make use of them, we pretend they are on L3. Restrict the events considered
45 * to those supported by MPAM.
46 * Class pointer may be NULL.
47 */
48#define MPAM_MAX_EVENT QOS_L3_MBM_TOTAL_EVENT_ID
49static struct mpam_resctrl_mon mpam_resctrl_counters[MPAM_MAX_EVENT + 1];
50
51#define for_each_mpam_resctrl_mon(mon, eventid) \
52 for (eventid = QOS_FIRST_EVENT, mon = &mpam_resctrl_counters[eventid]; \
53 eventid <= MPAM_MAX_EVENT; \
54 eventid++, mon = &mpam_resctrl_counters[eventid])
55
56/* The lock for modifying resctrl's domain lists from cpuhp callbacks. */
57static DEFINE_MUTEX(domain_list_lock);
58
59/*
60 * MPAM emulates CDP by setting different PARTID in the I/D fields of MPAM0_EL1.
61 * This applies globally to all traffic the CPU generates.
62 */
63static bool cdp_enabled;
64
65/*
66 * We use cacheinfo to discover the size of the caches and their id. cacheinfo
67 * populates this from a device_initcall(). mpam_resctrl_setup() must wait.
68 */
69static bool cacheinfo_ready;
70static DECLARE_WAIT_QUEUE_HEAD(wait_cacheinfo_ready);
71
72/*
73 * If resctrl_init() succeeded, resctrl_exit() can be used to remove support
74 * for the filesystem in the event of an error.
75 */
76static bool resctrl_enabled;
77
78bool resctrl_arch_alloc_capable(void)
79{
80 struct mpam_resctrl_res *res;
81 enum resctrl_res_level rid;
82
83 for_each_mpam_resctrl_control(res, rid) {
84 if (res->resctrl_res.alloc_capable)
85 return true;
86 }
87
88 return false;
89}
90
91bool resctrl_arch_mon_capable(void)
92{
93 struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3];
94 struct rdt_resource *l3 = &res->resctrl_res;
95
96 /* All monitors are presented as being on the L3 cache */
97 return l3->mon_capable;
98}
99
100bool resctrl_arch_is_evt_configurable(enum resctrl_event_id evt)
101{
102 return false;
103}
104
105void resctrl_arch_mon_event_config_read(void *info)
106{
107}
108
109void resctrl_arch_mon_event_config_write(void *info)
110{
111}
112
113void resctrl_arch_reset_rmid_all(struct rdt_resource *r, struct rdt_l3_mon_domain *d)
114{
115}
116
117void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_l3_mon_domain *d,
118 u32 closid, u32 rmid, enum resctrl_event_id eventid)
119{
120}
121
122bool resctrl_arch_mbm_cntr_assign_enabled(struct rdt_resource *r)
123{
124 return (r == &mpam_resctrl_controls[RDT_RESOURCE_L3].resctrl_res);
125}
126
127int resctrl_arch_mbm_cntr_assign_set(struct rdt_resource *r, bool enable)
128{
129 return -EINVAL;
130}
131
132int resctrl_arch_io_alloc_enable(struct rdt_resource *r, bool enable)
133{
134 return -EOPNOTSUPP;
135}
136
137bool resctrl_arch_get_io_alloc_enabled(struct rdt_resource *r)
138{
139 return false;
140}
141
142void resctrl_arch_pre_mount(void)
143{
144}
145
146bool resctrl_arch_get_cdp_enabled(enum resctrl_res_level rid)
147{
148 return mpam_resctrl_controls[rid].cdp_enabled;
149}
150
151/**
152 * resctrl_reset_task_closids() - Reset the PARTID/PMG values for all tasks.
153 *
154 * At boot, all existing tasks use partid zero for D and I.
155 * To enable/disable CDP emulation, all these tasks need relabelling.
156 */
157static void resctrl_reset_task_closids(void)
158{
159 struct task_struct *p, *t;
160
161 read_lock(&tasklist_lock);
162 for_each_process_thread(p, t) {
163 resctrl_arch_set_closid_rmid(t, RESCTRL_RESERVED_CLOSID,
164 RESCTRL_RESERVED_RMID);
165 }
166 read_unlock(&tasklist_lock);
167}
168
169static void mpam_resctrl_monitor_sync_abmc_vals(struct rdt_resource *l3)
170{
171 struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[QOS_L3_MBM_TOTAL_EVENT_ID];
172
173 if (!mon->class)
174 return;
175
176 if (!mon->assigned_counters)
177 return;
178
179 l3->mon.num_mbm_cntrs = mon->class->props.num_mbwu_mon;
180 if (cdp_enabled)
181 l3->mon.num_mbm_cntrs /= 2;
182
183 /*
184 * Continue as normal even if enabling cdp causes there to be
185 * zero counters. This avoids giving resctrl mixed messages.
186 */
187}
188
189int resctrl_arch_set_cdp_enabled(enum resctrl_res_level rid, bool enable)
190{
191 u32 partid_i = RESCTRL_RESERVED_CLOSID, partid_d = RESCTRL_RESERVED_CLOSID;
192 struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3];
193 struct rdt_resource *l3 = &res->resctrl_res;
194 int cpu;
195
196 if (!IS_ENABLED(CONFIG_EXPERT) && enable) {
197 /*
198 * If the resctrl fs is mounted more than once, sequentially,
199 * then CDP can lead to the use of out of range PARTIDs.
200 */
201 pr_warn("CDP not supported\n");
202 return -EOPNOTSUPP;
203 }
204
205 if (enable)
206 pr_warn("CDP is an expert feature and may cause MPAM to malfunction.\n");
207
208 /*
209 * resctrl_arch_set_cdp_enabled() is only called with enable set to
210 * false on error and unmount.
211 */
212 cdp_enabled = enable;
213 mpam_resctrl_controls[rid].cdp_enabled = enable;
214
215 if (enable)
216 l3->mon.num_rmid = resctrl_arch_system_num_rmid_idx() / 2;
217 else
218 l3->mon.num_rmid = resctrl_arch_system_num_rmid_idx();
219
220 /* The mbw_max feature can't hide cdp as it's a per-partid maximum. */
221 if (cdp_enabled && !mpam_resctrl_controls[RDT_RESOURCE_MBA].cdp_enabled)
222 mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = false;
223
224 /*
225 * If resctrl has attempted to enable CDP on MBA, re-enable MBA as two
226 * configurations will be provided so there is no aliasing problem.
227 */
228 if (mpam_resctrl_controls[RDT_RESOURCE_MBA].cdp_enabled &&
229 mpam_resctrl_controls[RDT_RESOURCE_MBA].class)
230 mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = true;
231
232 /* On unmount when CDP is disabled, re-enable MBA */
233 if (!cdp_enabled && mpam_resctrl_controls[RDT_RESOURCE_MBA].class)
234 mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = true;
235
236 if (enable) {
237 if (mpam_partid_max < 1)
238 return -EINVAL;
239
240 partid_d = resctrl_get_config_index(RESCTRL_RESERVED_CLOSID, CDP_DATA);
241 partid_i = resctrl_get_config_index(RESCTRL_RESERVED_CLOSID, CDP_CODE);
242 }
243
244 mpam_set_task_partid_pmg(current, partid_d, partid_i, 0, 0);
245 WRITE_ONCE(arm64_mpam_global_default, mpam_get_regval(current));
246
247 resctrl_reset_task_closids();
248 mpam_resctrl_monitor_sync_abmc_vals(l3);
249
250 for_each_possible_cpu(cpu)
251 mpam_set_cpu_defaults(cpu, partid_d, partid_i, 0, 0);
252 on_each_cpu(resctrl_arch_sync_cpu_closid_rmid, NULL, 1);
253
254 return 0;
255}
256
257static bool mpam_resctrl_hide_cdp(enum resctrl_res_level rid)
258{
259 return cdp_enabled && !resctrl_arch_get_cdp_enabled(rid);
260}
261
262/*
263 * MSC may raise an error interrupt if it sees an out or range partid/pmg,
264 * and go on to truncate the value. Regardless of what the hardware supports,
265 * only the system wide safe value is safe to use.
266 */
267u32 resctrl_arch_get_num_closid(struct rdt_resource *ignored)
268{
269 return mpam_partid_max + 1;
270}
271
272u32 resctrl_arch_system_num_rmid_idx(void)
273{
274 return (mpam_pmg_max + 1) * (mpam_partid_max + 1);
275}
276
277u32 resctrl_arch_rmid_idx_encode(u32 closid, u32 rmid)
278{
279 return closid * (mpam_pmg_max + 1) + rmid;
280}
281
282void resctrl_arch_rmid_idx_decode(u32 idx, u32 *closid, u32 *rmid)
283{
284 *closid = idx / (mpam_pmg_max + 1);
285 *rmid = idx % (mpam_pmg_max + 1);
286}
287
288void resctrl_arch_sched_in(struct task_struct *tsk)
289{
290 lockdep_assert_preemption_disabled();
291
292 mpam_thread_switch(tsk);
293}
294
295void resctrl_arch_set_cpu_default_closid_rmid(int cpu, u32 closid, u32 rmid)
296{
297 WARN_ON_ONCE(closid > U16_MAX);
298 WARN_ON_ONCE(rmid > U8_MAX);
299
300 if (!cdp_enabled) {
301 mpam_set_cpu_defaults(cpu, closid, closid, rmid, rmid);
302 } else {
303 /*
304 * When CDP is enabled, resctrl halves the closid range and we
305 * use odd/even partid for one closid.
306 */
307 u32 partid_d = resctrl_get_config_index(closid, CDP_DATA);
308 u32 partid_i = resctrl_get_config_index(closid, CDP_CODE);
309
310 mpam_set_cpu_defaults(cpu, partid_d, partid_i, rmid, rmid);
311 }
312}
313
314void resctrl_arch_sync_cpu_closid_rmid(void *info)
315{
316 struct resctrl_cpu_defaults *r = info;
317
318 lockdep_assert_preemption_disabled();
319
320 if (r) {
321 resctrl_arch_set_cpu_default_closid_rmid(smp_processor_id(),
322 r->closid, r->rmid);
323 }
324
325 resctrl_arch_sched_in(current);
326}
327
328void resctrl_arch_set_closid_rmid(struct task_struct *tsk, u32 closid, u32 rmid)
329{
330 WARN_ON_ONCE(closid > U16_MAX);
331 WARN_ON_ONCE(rmid > U8_MAX);
332
333 if (!cdp_enabled) {
334 mpam_set_task_partid_pmg(tsk, closid, closid, rmid, rmid);
335 } else {
336 u32 partid_d = resctrl_get_config_index(closid, CDP_DATA);
337 u32 partid_i = resctrl_get_config_index(closid, CDP_CODE);
338
339 mpam_set_task_partid_pmg(tsk, partid_d, partid_i, rmid, rmid);
340 }
341}
342
343bool resctrl_arch_match_closid(struct task_struct *tsk, u32 closid)
344{
345 u64 regval = mpam_get_regval(tsk);
346 u32 tsk_closid = FIELD_GET(MPAM0_EL1_PARTID_D, regval);
347
348 if (cdp_enabled)
349 tsk_closid >>= 1;
350
351 return tsk_closid == closid;
352}
353
354/* The task's pmg is not unique, the partid must be considered too */
355bool resctrl_arch_match_rmid(struct task_struct *tsk, u32 closid, u32 rmid)
356{
357 u64 regval = mpam_get_regval(tsk);
358 u32 tsk_closid = FIELD_GET(MPAM0_EL1_PARTID_D, regval);
359 u32 tsk_rmid = FIELD_GET(MPAM0_EL1_PMG_D, regval);
360
361 if (cdp_enabled)
362 tsk_closid >>= 1;
363
364 return (tsk_closid == closid) && (tsk_rmid == rmid);
365}
366
367struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l)
368{
369 if (l >= RDT_NUM_RESOURCES)
370 return NULL;
371
372 return &mpam_resctrl_controls[l].resctrl_res;
373}
374
375static int resctrl_arch_mon_ctx_alloc_no_wait(enum resctrl_event_id evtid)
376{
377 struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[evtid];
378
379 if (!mpam_is_enabled())
380 return -EINVAL;
381
382 if (!mon->class)
383 return -EINVAL;
384
385 switch (evtid) {
386 case QOS_L3_OCCUP_EVENT_ID:
387 /* With CDP, one monitor gets used for both code/data reads */
388 return mpam_alloc_csu_mon(mon->class);
389 case QOS_L3_MBM_LOCAL_EVENT_ID:
390 case QOS_L3_MBM_TOTAL_EVENT_ID:
391 return USE_PRE_ALLOCATED;
392 default:
393 return -EOPNOTSUPP;
394 }
395}
396
397void *resctrl_arch_mon_ctx_alloc(struct rdt_resource *r,
398 enum resctrl_event_id evtid)
399{
400 DEFINE_WAIT(wait);
401 int *ret;
402
403 ret = kmalloc_obj(*ret);
404 if (!ret)
405 return ERR_PTR(-ENOMEM);
406
407 do {
408 prepare_to_wait(&resctrl_mon_ctx_waiters, &wait,
409 TASK_INTERRUPTIBLE);
410 *ret = resctrl_arch_mon_ctx_alloc_no_wait(evtid);
411 if (*ret == -ENOSPC)
412 schedule();
413 } while (*ret == -ENOSPC && !signal_pending(current));
414 finish_wait(&resctrl_mon_ctx_waiters, &wait);
415
416 return ret;
417}
418
419static void resctrl_arch_mon_ctx_free_no_wait(enum resctrl_event_id evtid,
420 u32 mon_idx)
421{
422 struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[evtid];
423
424 if (!mpam_is_enabled())
425 return;
426
427 if (!mon->class)
428 return;
429
430 if (evtid == QOS_L3_OCCUP_EVENT_ID)
431 mpam_free_csu_mon(mon->class, mon_idx);
432
433 wake_up(&resctrl_mon_ctx_waiters);
434}
435
436void resctrl_arch_mon_ctx_free(struct rdt_resource *r,
437 enum resctrl_event_id evtid, void *arch_mon_ctx)
438{
439 u32 mon_idx = *(u32 *)arch_mon_ctx;
440
441 kfree(arch_mon_ctx);
442
443 resctrl_arch_mon_ctx_free_no_wait(evtid, mon_idx);
444}
445
446static int __read_mon(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp,
447 enum mpam_device_features mon_type,
448 int mon_idx,
449 enum resctrl_conf_type cdp_type, u32 closid, u32 rmid, u64 *val)
450{
451 struct mon_cfg cfg;
452
453 if (!mpam_is_enabled())
454 return -EINVAL;
455
456 /* Shift closid to account for CDP */
457 closid = resctrl_get_config_index(closid, cdp_type);
458
459 if (mon_idx == USE_PRE_ALLOCATED) {
460 int mbwu_idx = resctrl_arch_rmid_idx_encode(closid, rmid);
461
462 mon_idx = mon->mbwu_idx_to_mon[mbwu_idx];
463 if (mon_idx == -1)
464 return -ENOENT;
465 }
466
467 if (irqs_disabled()) {
468 /* Check if we can access this domain without an IPI */
469 return -EIO;
470 }
471
472 cfg = (struct mon_cfg) {
473 .mon = mon_idx,
474 .match_pmg = true,
475 .partid = closid,
476 .pmg = rmid,
477 };
478
479 return mpam_msmon_read(mon_comp, &cfg, mon_type, val);
480}
481
482static int read_mon_cdp_safe(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp,
483 enum mpam_device_features mon_type,
484 int mon_idx, u32 closid, u32 rmid, u64 *val)
485{
486 if (cdp_enabled) {
487 u64 code_val = 0, data_val = 0;
488 int err;
489
490 err = __read_mon(mon, mon_comp, mon_type, mon_idx,
491 CDP_CODE, closid, rmid, &code_val);
492 if (err)
493 return err;
494
495 err = __read_mon(mon, mon_comp, mon_type, mon_idx,
496 CDP_DATA, closid, rmid, &data_val);
497 if (err)
498 return err;
499
500 *val += code_val + data_val;
501 return 0;
502 }
503
504 return __read_mon(mon, mon_comp, mon_type, mon_idx,
505 CDP_NONE, closid, rmid, val);
506}
507
508/* MBWU when not in ABMC mode (not supported), and CSU counters. */
509int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain_hdr *hdr,
510 u32 closid, u32 rmid, enum resctrl_event_id eventid,
511 void *arch_priv, u64 *val, void *arch_mon_ctx)
512{
513 struct mpam_resctrl_dom *l3_dom;
514 struct mpam_component *mon_comp;
515 u32 mon_idx = *(u32 *)arch_mon_ctx;
516 enum mpam_device_features mon_type;
517 struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[eventid];
518
519 resctrl_arch_rmid_read_context_check();
520
521 if (!mpam_is_enabled())
522 return -EINVAL;
523
524 if (eventid >= QOS_NUM_EVENTS || !mon->class)
525 return -EINVAL;
526
527 l3_dom = container_of(hdr, struct mpam_resctrl_dom, resctrl_mon_dom.hdr);
528 mon_comp = l3_dom->mon_comp[eventid];
529
530 if (eventid != QOS_L3_OCCUP_EVENT_ID)
531 return -EINVAL;
532
533 mon_type = mpam_feat_msmon_csu;
534
535 return read_mon_cdp_safe(mon, mon_comp, mon_type, mon_idx,
536 closid, rmid, val);
537}
538
539/* MBWU counters when in ABMC mode */
540int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d,
541 u32 closid, u32 rmid, int mon_idx,
542 enum resctrl_event_id eventid, u64 *val)
543{
544 struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[eventid];
545 struct mpam_resctrl_dom *l3_dom;
546 struct mpam_component *mon_comp;
547
548 if (!mpam_is_enabled())
549 return -EINVAL;
550
551 if (eventid == QOS_L3_OCCUP_EVENT_ID || !mon->class)
552 return -EINVAL;
553
554 l3_dom = container_of(d, struct mpam_resctrl_dom, resctrl_mon_dom);
555 mon_comp = l3_dom->mon_comp[eventid];
556
557 return read_mon_cdp_safe(mon, mon_comp, mpam_feat_msmon_mbwu,
558 USE_PRE_ALLOCATED, closid, rmid, val);
559}
560
561static void __reset_mon(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp,
562 int mon_idx,
563 enum resctrl_conf_type cdp_type, u32 closid, u32 rmid)
564{
565 struct mon_cfg cfg = { };
566
567 if (!mpam_is_enabled())
568 return;
569
570 /* Shift closid to account for CDP */
571 closid = resctrl_get_config_index(closid, cdp_type);
572
573 if (mon_idx == USE_PRE_ALLOCATED) {
574 int mbwu_idx = resctrl_arch_rmid_idx_encode(closid, rmid);
575
576 mon_idx = mon->mbwu_idx_to_mon[mbwu_idx];
577 }
578
579 if (mon_idx == -1)
580 return;
581 cfg.mon = mon_idx;
582 mpam_msmon_reset_mbwu(mon_comp, &cfg);
583}
584
585static void reset_mon_cdp_safe(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp,
586 int mon_idx, u32 closid, u32 rmid)
587{
588 if (cdp_enabled) {
589 __reset_mon(mon, mon_comp, mon_idx, CDP_CODE, closid, rmid);
590 __reset_mon(mon, mon_comp, mon_idx, CDP_DATA, closid, rmid);
591 } else {
592 __reset_mon(mon, mon_comp, mon_idx, CDP_NONE, closid, rmid);
593 }
594}
595
596/* Reset an assigned counter */
597void resctrl_arch_reset_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d,
598 u32 closid, u32 rmid, int cntr_id,
599 enum resctrl_event_id eventid)
600{
601 struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[eventid];
602 struct mpam_resctrl_dom *l3_dom;
603 struct mpam_component *mon_comp;
604
605 if (!mpam_is_enabled())
606 return;
607
608 if (eventid == QOS_L3_OCCUP_EVENT_ID || !mon->class)
609 return;
610
611 l3_dom = container_of(d, struct mpam_resctrl_dom, resctrl_mon_dom);
612 mon_comp = l3_dom->mon_comp[eventid];
613
614 reset_mon_cdp_safe(mon, mon_comp, USE_PRE_ALLOCATED, closid, rmid);
615}
616
617/*
618 * The rmid realloc threshold should be for the smallest cache exposed to
619 * resctrl.
620 */
621static int update_rmid_limits(struct mpam_class *class)
622{
623 u32 num_unique_pmg = resctrl_arch_system_num_rmid_idx();
624 struct mpam_props *cprops = &class->props;
625 struct cacheinfo *ci;
626
627 lockdep_assert_cpus_held();
628
629 if (!mpam_has_feature(mpam_feat_msmon_csu, cprops))
630 return 0;
631
632 /*
633 * Assume cache levels are the same size for all CPUs...
634 * The check just requires any online CPU and it can't go offline as we
635 * hold the cpu lock.
636 */
637 ci = get_cpu_cacheinfo_level(raw_smp_processor_id(), class->level);
638 if (!ci || ci->size == 0) {
639 pr_debug("Could not read cache size for class %u\n",
640 class->level);
641 return -EINVAL;
642 }
643
644 if (!resctrl_rmid_realloc_limit ||
645 ci->size < resctrl_rmid_realloc_limit) {
646 resctrl_rmid_realloc_limit = ci->size;
647 resctrl_rmid_realloc_threshold = ci->size / num_unique_pmg;
648 }
649
650 return 0;
651}
652
653static bool cache_has_usable_cpor(struct mpam_class *class)
654{
655 struct mpam_props *cprops = &class->props;
656
657 if (!mpam_has_feature(mpam_feat_cpor_part, cprops))
658 return false;
659
660 /* resctrl uses u32 for all bitmap configurations */
661 return class->props.cpbm_wd <= 32;
662}
663
664static bool mba_class_use_mbw_max(struct mpam_props *cprops)
665{
666 return (mpam_has_feature(mpam_feat_mbw_max, cprops) &&
667 cprops->bwa_wd);
668}
669
670static bool class_has_usable_mba(struct mpam_props *cprops)
671{
672 return mba_class_use_mbw_max(cprops);
673}
674
675static bool cache_has_usable_csu(struct mpam_class *class)
676{
677 struct mpam_props *cprops;
678
679 if (!class)
680 return false;
681
682 cprops = &class->props;
683
684 if (!mpam_has_feature(mpam_feat_msmon_csu, cprops))
685 return false;
686
687 /*
688 * CSU counters settle on the value, so we can get away with
689 * having only one.
690 */
691 if (!cprops->num_csu_mon)
692 return false;
693
694 return true;
695}
696
697static bool class_has_usable_mbwu(struct mpam_class *class)
698{
699 struct mpam_props *cprops = &class->props;
700
701 if (!mpam_has_feature(mpam_feat_msmon_mbwu, cprops))
702 return false;
703
704 if (!cprops->num_mbwu_mon)
705 return false;
706
707 return true;
708}
709
710/*
711 * Calculate the worst-case percentage change from each implemented step
712 * in the control.
713 */
714static u32 get_mba_granularity(struct mpam_props *cprops)
715{
716 if (!mba_class_use_mbw_max(cprops))
717 return 0;
718
719 /*
720 * bwa_wd is the number of bits implemented in the 0.xxx
721 * fixed point fraction. 1 bit is 50%, 2 is 25% etc.
722 */
723 return DIV_ROUND_UP(MAX_MBA_BW, 1 << cprops->bwa_wd);
724}
725
726/*
727 * Each fixed-point hardware value architecturally represents a range
728 * of values: the full range 0% - 100% is split contiguously into
729 * (1 << cprops->bwa_wd) equal bands.
730 *
731 * Although the bwa_bwd fields have 6 bits the maximum valid value is 16
732 * as it reports the width of fields that are at most 16 bits. When
733 * fewer than 16 bits are valid the least significant bits are
734 * ignored. The implied binary point is kept between bits 15 and 16 and
735 * so the valid bits are leftmost.
736 *
737 * See ARM IHI0099B.a "MPAM system component specification", Section 9.3,
738 * "The fixed-point fractional format" for more information.
739 *
740 * Find the nearest percentage value to the upper bound of the selected band:
741 */
742static u32 mbw_max_to_percent(u16 mbw_max, struct mpam_props *cprops)
743{
744 u32 val = mbw_max;
745
746 val >>= 16 - cprops->bwa_wd;
747 val += 1;
748 val *= MAX_MBA_BW;
749 val = DIV_ROUND_CLOSEST(val, 1 << cprops->bwa_wd);
750
751 return val;
752}
753
754/*
755 * Find the band whose upper bound is closest to the specified percentage.
756 *
757 * A round-to-nearest policy is followed here as a balanced compromise
758 * between unexpected under-commit of the resource (where the total of
759 * a set of resource allocations after conversion is less than the
760 * expected total, due to rounding of the individual converted
761 * percentages) and over-commit (where the total of the converted
762 * allocations is greater than expected).
763 */
764static u16 percent_to_mbw_max(u8 pc, struct mpam_props *cprops)
765{
766 u32 val = pc;
767
768 val <<= cprops->bwa_wd;
769 val = DIV_ROUND_CLOSEST(val, MAX_MBA_BW);
770 val = max(val, 1) - 1;
771 val <<= 16 - cprops->bwa_wd;
772
773 return val;
774}
775
776static u32 get_mba_min(struct mpam_props *cprops)
777{
778 if (!mba_class_use_mbw_max(cprops)) {
779 WARN_ON_ONCE(1);
780 return 0;
781 }
782
783 return mbw_max_to_percent(0, cprops);
784}
785
786/* Find the L3 cache that has affinity with this CPU */
787static int find_l3_equivalent_bitmask(int cpu, cpumask_var_t tmp_cpumask)
788{
789 u32 cache_id = get_cpu_cacheinfo_id(cpu, 3);
790
791 lockdep_assert_cpus_held();
792
793 return mpam_get_cpumask_from_cache_id(cache_id, 3, tmp_cpumask);
794}
795
796/*
797 * topology_matches_l3() - Is the provided class the same shape as L3
798 * @victim: The class we'd like to pretend is L3.
799 *
800 * resctrl expects all the world's a Xeon, and all counters are on the
801 * L3. We allow some mapping counters on other classes. This requires
802 * that the CPU->domain mapping is the same kind of shape.
803 *
804 * Using cacheinfo directly would make this work even if resctrl can't
805 * use the L3 - but cacheinfo can't tell us anything about offline CPUs.
806 * Using the L3 resctrl domain list also depends on CPUs being online.
807 * Using the mpam_class we picked for L3 so we can use its domain list
808 * assumes that there are MPAM controls on the L3.
809 * Instead, this path eventually uses the mpam_get_cpumask_from_cache_id()
810 * helper which can tell us about offline CPUs ... but getting the cache_id
811 * to start with relies on at least one CPU per L3 cache being online at
812 * boot.
813 *
814 * Walk the victim component list and compare the affinity mask with the
815 * corresponding L3. The topology matches if each victim:component's affinity
816 * mask is the same as the CPU's corresponding L3's. These lists/masks are
817 * computed from firmware tables so don't change at runtime.
818 */
819static bool topology_matches_l3(struct mpam_class *victim)
820{
821 int cpu, err;
822 struct mpam_component *victim_iter;
823
824 lockdep_assert_cpus_held();
825
826 cpumask_var_t __free(free_cpumask_var) tmp_cpumask = CPUMASK_VAR_NULL;
827 if (!alloc_cpumask_var(&tmp_cpumask, GFP_KERNEL))
828 return false;
829
830 guard(srcu)(&mpam_srcu);
831 list_for_each_entry_srcu(victim_iter, &victim->components, class_list,
832 srcu_read_lock_held(&mpam_srcu)) {
833 if (cpumask_empty(&victim_iter->affinity)) {
834 pr_debug("class %u has CPU-less component %u - can't match L3!\n",
835 victim->level, victim_iter->comp_id);
836 return false;
837 }
838
839 cpu = cpumask_any_and(&victim_iter->affinity, cpu_online_mask);
840 if (WARN_ON_ONCE(cpu >= nr_cpu_ids))
841 return false;
842
843 cpumask_clear(tmp_cpumask);
844 err = find_l3_equivalent_bitmask(cpu, tmp_cpumask);
845 if (err) {
846 pr_debug("Failed to find L3's equivalent component to class %u component %u\n",
847 victim->level, victim_iter->comp_id);
848 return false;
849 }
850
851 /* Any differing bits in the affinity mask? */
852 if (!cpumask_equal(tmp_cpumask, &victim_iter->affinity)) {
853 pr_debug("class %u component %u has Mismatched CPU mask with L3 equivalent\n"
854 "L3:%*pbl != victim:%*pbl\n",
855 victim->level, victim_iter->comp_id,
856 cpumask_pr_args(tmp_cpumask),
857 cpumask_pr_args(&victim_iter->affinity));
858
859 return false;
860 }
861 }
862
863 return true;
864}
865
866/*
867 * Test if the traffic for a class matches that at egress from the L3. For
868 * MSC at memory controllers this is only possible if there is a single L3
869 * as otherwise the counters at the memory can include bandwidth from the
870 * non-local L3.
871 */
872static bool traffic_matches_l3(struct mpam_class *class)
873{
874 int err, cpu;
875
876 lockdep_assert_cpus_held();
877
878 if (class->type == MPAM_CLASS_CACHE && class->level == 3)
879 return true;
880
881 if (class->type == MPAM_CLASS_CACHE && class->level != 3) {
882 pr_debug("class %u is a different cache from L3\n", class->level);
883 return false;
884 }
885
886 if (class->type != MPAM_CLASS_MEMORY) {
887 pr_debug("class %u is neither of type cache or memory\n", class->level);
888 return false;
889 }
890
891 cpumask_var_t __free(free_cpumask_var) tmp_cpumask = CPUMASK_VAR_NULL;
892 if (!alloc_cpumask_var(&tmp_cpumask, GFP_KERNEL)) {
893 pr_debug("cpumask allocation failed\n");
894 return false;
895 }
896
897 cpu = cpumask_any_and(&class->affinity, cpu_online_mask);
898 err = find_l3_equivalent_bitmask(cpu, tmp_cpumask);
899 if (err) {
900 pr_debug("Failed to find L3 downstream to cpu %d\n", cpu);
901 return false;
902 }
903
904 if (!cpumask_equal(tmp_cpumask, cpu_possible_mask)) {
905 pr_debug("There is more than one L3\n");
906 return false;
907 }
908
909 /* Be strict; the traffic might stop in the intermediate cache. */
910 if (get_cpu_cacheinfo_id(cpu, 4) != -1) {
911 pr_debug("L3 isn't the last level of cache\n");
912 return false;
913 }
914
915 if (num_possible_nodes() > 1) {
916 pr_debug("There is more than one numa node\n");
917 return false;
918 }
919
920#ifdef CONFIG_HMEM_REPORTING
921 if (node_devices[cpu_to_node(cpu)]->cache_dev) {
922 pr_debug("There is a memory side cache\n");
923 return false;
924 }
925#endif
926
927 return true;
928}
929
930/* Test whether we can export MPAM_CLASS_CACHE:{2,3}? */
931static void mpam_resctrl_pick_caches(void)
932{
933 struct mpam_class *class;
934 struct mpam_resctrl_res *res;
935
936 lockdep_assert_cpus_held();
937
938 guard(srcu)(&mpam_srcu);
939 list_for_each_entry_srcu(class, &mpam_classes, classes_list,
940 srcu_read_lock_held(&mpam_srcu)) {
941 if (class->type != MPAM_CLASS_CACHE) {
942 pr_debug("class %u is not a cache\n", class->level);
943 continue;
944 }
945
946 if (class->level != 2 && class->level != 3) {
947 pr_debug("class %u is not L2 or L3\n", class->level);
948 continue;
949 }
950
951 if (!cache_has_usable_cpor(class)) {
952 pr_debug("class %u cache misses CPOR\n", class->level);
953 continue;
954 }
955
956 if (!cpumask_equal(&class->affinity, cpu_possible_mask)) {
957 pr_debug("class %u has missing CPUs, mask %*pb != %*pb\n", class->level,
958 cpumask_pr_args(&class->affinity),
959 cpumask_pr_args(cpu_possible_mask));
960 continue;
961 }
962
963 if (class->level == 2)
964 res = &mpam_resctrl_controls[RDT_RESOURCE_L2];
965 else
966 res = &mpam_resctrl_controls[RDT_RESOURCE_L3];
967 res->class = class;
968 }
969}
970
971static void mpam_resctrl_pick_mba(void)
972{
973 struct mpam_class *class, *candidate_class = NULL;
974 struct mpam_resctrl_res *res;
975
976 lockdep_assert_cpus_held();
977
978 guard(srcu)(&mpam_srcu);
979 list_for_each_entry_srcu(class, &mpam_classes, classes_list,
980 srcu_read_lock_held(&mpam_srcu)) {
981 struct mpam_props *cprops = &class->props;
982
983 if (class->level != 3 && class->type == MPAM_CLASS_CACHE) {
984 pr_debug("class %u is a cache but not the L3\n", class->level);
985 continue;
986 }
987
988 if (!class_has_usable_mba(cprops)) {
989 pr_debug("class %u has no bandwidth control\n",
990 class->level);
991 continue;
992 }
993
994 if (!cpumask_equal(&class->affinity, cpu_possible_mask)) {
995 pr_debug("class %u has missing CPUs\n", class->level);
996 continue;
997 }
998
999 if (!topology_matches_l3(class)) {
1000 pr_debug("class %u topology doesn't match L3\n",
1001 class->level);
1002 continue;
1003 }
1004
1005 if (!traffic_matches_l3(class)) {
1006 pr_debug("class %u traffic doesn't match L3 egress\n",
1007 class->level);
1008 continue;
1009 }
1010
1011 /*
1012 * Pick a resource to be MBA that as close as possible to
1013 * the L3. mbm_total counts the bandwidth leaving the L3
1014 * cache and MBA should correspond as closely as possible
1015 * for proper operation of mba_sc.
1016 */
1017 if (!candidate_class || class->level < candidate_class->level)
1018 candidate_class = class;
1019 }
1020
1021 if (candidate_class) {
1022 pr_debug("selected class %u to back MBA\n",
1023 candidate_class->level);
1024 res = &mpam_resctrl_controls[RDT_RESOURCE_MBA];
1025 res->class = candidate_class;
1026 }
1027}
1028
1029static void __free_mbwu_mon(struct mpam_class *class, int *array,
1030 u16 num_mbwu_mon)
1031{
1032 for (int i = 0; i < num_mbwu_mon; i++) {
1033 if (array[i] < 0)
1034 continue;
1035
1036 mpam_free_mbwu_mon(class, array[i]);
1037 array[i] = -1;
1038 }
1039}
1040
1041static int __alloc_mbwu_mon(struct mpam_class *class, int *array,
1042 u16 num_mbwu_mon)
1043{
1044 for (int i = 0; i < num_mbwu_mon; i++) {
1045 int mbwu_mon = mpam_alloc_mbwu_mon(class);
1046
1047 if (mbwu_mon < 0) {
1048 __free_mbwu_mon(class, array, num_mbwu_mon);
1049 return mbwu_mon;
1050 }
1051 array[i] = mbwu_mon;
1052 }
1053
1054 return 0;
1055}
1056
1057static int *__alloc_mbwu_array(struct mpam_class *class, u16 num_mbwu_mon)
1058{
1059 int err;
1060
1061 int *array __free(kvfree) = kvmalloc_objs(*array, num_mbwu_mon);
1062 if (!array)
1063 return ERR_PTR(-ENOMEM);
1064
1065 memset(array, -1, num_mbwu_mon * sizeof(*array));
1066
1067 err = __alloc_mbwu_mon(class, array, num_mbwu_mon);
1068 if (err)
1069 return ERR_PTR(err);
1070 return_ptr(array);
1071}
1072
1073static void counter_update_class(enum resctrl_event_id evt_id,
1074 struct mpam_class *class)
1075{
1076 struct mpam_class *existing_class = mpam_resctrl_counters[evt_id].class;
1077
1078 if (existing_class) {
1079 if (class->level == 3) {
1080 pr_debug("Existing class is L3 - L3 wins\n");
1081 return;
1082 }
1083
1084 if (existing_class->level < class->level) {
1085 pr_debug("Existing class is closer to L3, %u versus %u - closer is better\n",
1086 existing_class->level, class->level);
1087 return;
1088 }
1089 }
1090
1091 mpam_resctrl_counters[evt_id].class = class;
1092}
1093
1094static void mpam_resctrl_pick_counters(void)
1095{
1096 struct mpam_class *class;
1097
1098 lockdep_assert_cpus_held();
1099
1100 guard(srcu)(&mpam_srcu);
1101 list_for_each_entry_srcu(class, &mpam_classes, classes_list,
1102 srcu_read_lock_held(&mpam_srcu)) {
1103 /* The name of the resource is L3... */
1104 if (class->type == MPAM_CLASS_CACHE && class->level != 3) {
1105 pr_debug("class %u is a cache but not the L3", class->level);
1106 continue;
1107 }
1108
1109 if (!cpumask_equal(&class->affinity, cpu_possible_mask)) {
1110 pr_debug("class %u does not cover all CPUs",
1111 class->level);
1112 continue;
1113 }
1114
1115 if (cache_has_usable_csu(class)) {
1116 pr_debug("class %u has usable CSU",
1117 class->level);
1118
1119 /* CSU counters only make sense on a cache. */
1120 switch (class->type) {
1121 case MPAM_CLASS_CACHE:
1122 if (update_rmid_limits(class))
1123 break;
1124
1125 counter_update_class(QOS_L3_OCCUP_EVENT_ID, class);
1126 break;
1127 default:
1128 break;
1129 }
1130 }
1131
1132 if (class_has_usable_mbwu(class) &&
1133 topology_matches_l3(class) &&
1134 traffic_matches_l3(class)) {
1135 pr_debug("class %u has usable MBWU, and matches L3 topology and traffic\n",
1136 class->level);
1137
1138 /*
1139 * An MSC measures bandwidth for a path determined by
1140 * its location in hardware. We can't distinguish
1141 * traffic by destination so we don't know if it's
1142 * staying on the same NUMA node. Hence, we can't
1143 * calculate mbm_local except when we only have one L3
1144 * and it's equivalent to mbm_total and so always use
1145 * mbm_total.
1146 */
1147 counter_update_class(QOS_L3_MBM_TOTAL_EVENT_ID, class);
1148 }
1149 }
1150}
1151
1152static void __config_cntr(struct mpam_resctrl_mon *mon, u32 cntr_id,
1153 enum resctrl_conf_type cdp_type, u32 closid, u32 rmid,
1154 bool assign)
1155{
1156 /* Same CDP index remap as closid; maps cntr_id to assigned_counters[]. */
1157 u32 mbwu_idx, mon_idx = resctrl_get_config_index(cntr_id, cdp_type);
1158
1159 closid = resctrl_get_config_index(closid, cdp_type);
1160 mbwu_idx = resctrl_arch_rmid_idx_encode(closid, rmid);
1161
1162 if (assign)
1163 mon->mbwu_idx_to_mon[mbwu_idx] = mon->assigned_counters[mon_idx];
1164 else
1165 mon->mbwu_idx_to_mon[mbwu_idx] = -1;
1166}
1167
1168void resctrl_arch_config_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d,
1169 enum resctrl_event_id evtid, u32 rmid, u32 closid,
1170 u32 cntr_id, bool assign)
1171{
1172 struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[evtid];
1173
1174 if (evtid != QOS_L3_MBM_TOTAL_EVENT_ID) {
1175 pr_debug("unexpected event id\n");
1176 return;
1177 }
1178
1179 if (!mon->mbwu_idx_to_mon || !mon->assigned_counters) {
1180 pr_debug("monitor arrays not allocated\n");
1181 return;
1182 }
1183
1184 if (cdp_enabled) {
1185 __config_cntr(mon, cntr_id, CDP_CODE, closid, rmid, assign);
1186 __config_cntr(mon, cntr_id, CDP_DATA, closid, rmid, assign);
1187 } else {
1188 __config_cntr(mon, cntr_id, CDP_NONE, closid, rmid, assign);
1189 }
1190
1191 resctrl_arch_reset_cntr(r, d, closid, rmid, cntr_id, QOS_L3_MBM_TOTAL_EVENT_ID);
1192}
1193
1194static int mpam_resctrl_control_init(struct mpam_resctrl_res *res)
1195{
1196 struct mpam_class *class = res->class;
1197 struct mpam_props *cprops = &class->props;
1198 struct rdt_resource *r = &res->resctrl_res;
1199
1200 switch (r->rid) {
1201 case RDT_RESOURCE_L2:
1202 case RDT_RESOURCE_L3:
1203 r->schema_fmt = RESCTRL_SCHEMA_BITMAP;
1204 r->cache.arch_has_sparse_bitmasks = true;
1205
1206 r->cache.cbm_len = class->props.cpbm_wd;
1207 /* mpam_devices will reject empty bitmaps */
1208 r->cache.min_cbm_bits = 1;
1209
1210 if (r->rid == RDT_RESOURCE_L2) {
1211 r->name = "L2";
1212 r->ctrl_scope = RESCTRL_L2_CACHE;
1213 r->cdp_capable = true;
1214 } else {
1215 r->name = "L3";
1216 r->ctrl_scope = RESCTRL_L3_CACHE;
1217 r->cdp_capable = true;
1218 }
1219
1220 /*
1221 * Which bits are shared with other ...things... Unknown
1222 * devices use partid-0 which uses all the bitmap fields. Until
1223 * we have configured the SMMU and GIC not to do this 'all the
1224 * bits' is the correct answer here.
1225 */
1226 r->cache.shareable_bits = resctrl_get_default_ctrl(r);
1227 r->alloc_capable = true;
1228 break;
1229 case RDT_RESOURCE_MBA:
1230 r->schema_fmt = RESCTRL_SCHEMA_RANGE;
1231 r->ctrl_scope = RESCTRL_L3_CACHE;
1232
1233 r->membw.delay_linear = true;
1234 r->membw.throttle_mode = THREAD_THROTTLE_UNDEFINED;
1235 r->membw.min_bw = get_mba_min(cprops);
1236 r->membw.max_bw = MAX_MBA_BW;
1237 r->membw.bw_gran = get_mba_granularity(cprops);
1238
1239 r->name = "MB";
1240 r->alloc_capable = true;
1241 break;
1242 default:
1243 return -EINVAL;
1244 }
1245
1246 return 0;
1247}
1248
1249static int mpam_resctrl_pick_domain_id(int cpu, struct mpam_component *comp)
1250{
1251 struct mpam_class *class = comp->class;
1252
1253 if (class->type == MPAM_CLASS_CACHE)
1254 return comp->comp_id;
1255
1256 if (topology_matches_l3(class)) {
1257 /* Use the corresponding L3 component ID as the domain ID */
1258 int id = get_cpu_cacheinfo_id(cpu, 3);
1259
1260 /* Implies topology_matches_l3() made a mistake */
1261 if (WARN_ON_ONCE(id == -1))
1262 return comp->comp_id;
1263
1264 return id;
1265 }
1266
1267 /* Otherwise, expose the ID used by the firmware table code. */
1268 return comp->comp_id;
1269}
1270
1271/*
1272 * This must run after all event counters have been picked so that any free
1273 * running counters have already been allocated.
1274 */
1275static int mpam_resctrl_monitor_init_abmc(struct mpam_resctrl_mon *mon)
1276{
1277 struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3];
1278 size_t num_rmid = resctrl_arch_system_num_rmid_idx();
1279 struct rdt_resource *l3 = &res->resctrl_res;
1280 struct mpam_class *class = mon->class;
1281 u16 num_mbwu_mon;
1282 int *cntrs;
1283
1284 int *rmid_array __free(kvfree) = kvmalloc_objs(*rmid_array, num_rmid);
1285 if (!rmid_array) {
1286 pr_debug("Failed to allocate RMID array\n");
1287 return -ENOMEM;
1288 }
1289 memset(rmid_array, -1, num_rmid * sizeof(*rmid_array));
1290
1291 num_mbwu_mon = class->props.num_mbwu_mon;
1292 cntrs = __alloc_mbwu_array(mon->class, num_mbwu_mon);
1293 if (IS_ERR(cntrs))
1294 return PTR_ERR(cntrs);
1295 mon->assigned_counters = cntrs;
1296 mon->mbwu_idx_to_mon = no_free_ptr(rmid_array);
1297
1298 l3->mon.mbm_cntr_assignable = true;
1299 l3->mon.mbm_assign_on_mkdir = true;
1300 l3->mon.mbm_cntr_configurable = false;
1301 l3->mon.mbm_cntr_assign_fixed = true;
1302
1303 mpam_resctrl_monitor_sync_abmc_vals(l3);
1304
1305 return 0;
1306}
1307
1308static int mpam_resctrl_monitor_init(struct mpam_resctrl_mon *mon,
1309 enum resctrl_event_id type)
1310{
1311 struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3];
1312 struct rdt_resource *l3 = &res->resctrl_res;
1313
1314 lockdep_assert_cpus_held();
1315
1316 /*
1317 * There also needs to be an L3 cache present.
1318 * The check just requires any online CPU and it can't go offline as we
1319 * hold the cpu lock.
1320 */
1321 if (get_cpu_cacheinfo_id(raw_smp_processor_id(), 3) == -1)
1322 return 0;
1323
1324 /*
1325 * If there are no MPAM resources on L3, force it into existence.
1326 * topology_matches_l3() already ensures this looks like the L3.
1327 * The domain-ids will be fixed up by mpam_resctrl_domain_hdr_init().
1328 */
1329 if (!res->class) {
1330 pr_warn_once("Faking L3 MSC to enable counters.\n");
1331 res->class = mpam_resctrl_counters[type].class;
1332 }
1333
1334 /*
1335 * Called multiple times!, once per event type that has a
1336 * monitoring class.
1337 * Setting name is necessary on monitor only platforms.
1338 */
1339 l3->name = "L3";
1340 l3->mon_scope = RESCTRL_L3_CACHE;
1341
1342 /*
1343 * num-rmid is the upper bound for the number of monitoring groups that
1344 * can exist simultaneously, including the default monitoring group for
1345 * each control group. Hence, advertise the whole rmid_idx space even
1346 * though each control group has its own pmg/rmid space. Unfortunately,
1347 * this does mean userspace needs to know the architecture to correctly
1348 * interpret this value.
1349 */
1350 l3->mon.num_rmid = resctrl_arch_system_num_rmid_idx();
1351
1352 if (type == QOS_L3_MBM_TOTAL_EVENT_ID) {
1353 int err;
1354
1355 err = mpam_resctrl_monitor_init_abmc(mon);
1356 if (err)
1357 return err;
1358
1359 static_assert(MAX_EVT_CONFIG_BITS == 0x7f);
1360 l3->mon.mbm_cfg_mask = MAX_EVT_CONFIG_BITS;
1361 }
1362
1363 if (!resctrl_enable_mon_event(type, false, 0, NULL))
1364 return -EINVAL;
1365
1366 l3->mon_capable = true;
1367
1368 return 0;
1369}
1370
1371u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d,
1372 u32 closid, enum resctrl_conf_type type)
1373{
1374 u32 partid;
1375 struct mpam_config *cfg;
1376 struct mpam_props *cprops;
1377 struct mpam_resctrl_res *res;
1378 struct mpam_resctrl_dom *dom;
1379 enum mpam_device_features configured_by;
1380
1381 lockdep_assert_cpus_held();
1382
1383 if (!mpam_is_enabled())
1384 return resctrl_get_default_ctrl(r);
1385
1386 res = container_of(r, struct mpam_resctrl_res, resctrl_res);
1387 dom = container_of(d, struct mpam_resctrl_dom, resctrl_ctrl_dom);
1388 cprops = &res->class->props;
1389
1390 /*
1391 * When CDP is enabled, but the resource doesn't support it,
1392 * the control is cloned across both partids.
1393 * Pick one at random to read:
1394 */
1395 if (mpam_resctrl_hide_cdp(r->rid))
1396 type = CDP_DATA;
1397
1398 partid = resctrl_get_config_index(closid, type);
1399 cfg = &dom->ctrl_comp->cfg[partid];
1400
1401 switch (r->rid) {
1402 case RDT_RESOURCE_L2:
1403 case RDT_RESOURCE_L3:
1404 configured_by = mpam_feat_cpor_part;
1405 break;
1406 case RDT_RESOURCE_MBA:
1407 if (mpam_has_feature(mpam_feat_mbw_max, cprops)) {
1408 configured_by = mpam_feat_mbw_max;
1409 break;
1410 }
1411 fallthrough;
1412 default:
1413 return resctrl_get_default_ctrl(r);
1414 }
1415
1416 if (!r->alloc_capable || partid >= resctrl_arch_get_num_closid(r) ||
1417 !mpam_has_feature(configured_by, cfg))
1418 return resctrl_get_default_ctrl(r);
1419
1420 switch (configured_by) {
1421 case mpam_feat_cpor_part:
1422 return cfg->cpbm;
1423 case mpam_feat_mbw_max:
1424 return mbw_max_to_percent(cfg->mbw_max, cprops);
1425 default:
1426 return resctrl_get_default_ctrl(r);
1427 }
1428}
1429
1430int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d,
1431 u32 closid, enum resctrl_conf_type t, u32 cfg_val)
1432{
1433 int err;
1434 u32 partid;
1435 struct mpam_config cfg;
1436 struct mpam_props *cprops;
1437 struct mpam_resctrl_res *res;
1438 struct mpam_resctrl_dom *dom;
1439
1440 lockdep_assert_cpus_held();
1441 lockdep_assert_irqs_enabled();
1442
1443 if (!mpam_is_enabled())
1444 return -EINVAL;
1445
1446 /*
1447 * No need to check the CPU as mpam_apply_config() doesn't care, and
1448 * resctrl_arch_update_domains() relies on this.
1449 */
1450 res = container_of(r, struct mpam_resctrl_res, resctrl_res);
1451 dom = container_of(d, struct mpam_resctrl_dom, resctrl_ctrl_dom);
1452 cprops = &res->class->props;
1453
1454 if (mpam_resctrl_hide_cdp(r->rid))
1455 t = CDP_DATA;
1456
1457 partid = resctrl_get_config_index(closid, t);
1458 if (!r->alloc_capable || partid >= resctrl_arch_get_num_closid(r)) {
1459 pr_debug("Not alloc capable or computed PARTID out of range\n");
1460 return -EINVAL;
1461 }
1462
1463 /*
1464 * Copy the current config to avoid clearing other resources when the
1465 * same component is exposed multiple times through resctrl.
1466 */
1467 cfg = dom->ctrl_comp->cfg[partid];
1468
1469 switch (r->rid) {
1470 case RDT_RESOURCE_L2:
1471 case RDT_RESOURCE_L3:
1472 cfg.cpbm = cfg_val;
1473 mpam_set_feature(mpam_feat_cpor_part, &cfg);
1474 break;
1475 case RDT_RESOURCE_MBA:
1476 if (mpam_has_feature(mpam_feat_mbw_max, cprops)) {
1477 cfg.mbw_max = percent_to_mbw_max(cfg_val, cprops);
1478 mpam_set_feature(mpam_feat_mbw_max, &cfg);
1479 break;
1480 }
1481 fallthrough;
1482 default:
1483 return -EINVAL;
1484 }
1485
1486 /*
1487 * When CDP is enabled, but the resource doesn't support it, we need to
1488 * apply the same configuration to the other partid.
1489 */
1490 if (mpam_resctrl_hide_cdp(r->rid)) {
1491 partid = resctrl_get_config_index(closid, CDP_CODE);
1492 err = mpam_apply_config(dom->ctrl_comp, partid, &cfg);
1493 if (err)
1494 return err;
1495
1496 partid = resctrl_get_config_index(closid, CDP_DATA);
1497 return mpam_apply_config(dom->ctrl_comp, partid, &cfg);
1498 }
1499
1500 return mpam_apply_config(dom->ctrl_comp, partid, &cfg);
1501}
1502
1503int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid)
1504{
1505 int err;
1506 struct rdt_ctrl_domain *d;
1507
1508 lockdep_assert_cpus_held();
1509 lockdep_assert_irqs_enabled();
1510
1511 if (!mpam_is_enabled())
1512 return -EINVAL;
1513
1514 list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list) {
1515 for (enum resctrl_conf_type t = 0; t < CDP_NUM_TYPES; t++) {
1516 struct resctrl_staged_config *cfg = &d->staged_config[t];
1517
1518 if (!cfg->have_new_ctrl)
1519 continue;
1520
1521 err = resctrl_arch_update_one(r, d, closid, t,
1522 cfg->new_ctrl);
1523 if (err)
1524 return err;
1525 }
1526 }
1527
1528 return 0;
1529}
1530
1531void resctrl_arch_reset_all_ctrls(struct rdt_resource *r)
1532{
1533 struct mpam_resctrl_res *res;
1534
1535 lockdep_assert_cpus_held();
1536
1537 if (!mpam_is_enabled())
1538 return;
1539
1540 res = container_of(r, struct mpam_resctrl_res, resctrl_res);
1541 mpam_reset_class_locked(res->class);
1542}
1543
1544static void mpam_resctrl_domain_hdr_init(int cpu, struct mpam_component *comp,
1545 enum resctrl_res_level rid,
1546 struct rdt_domain_hdr *hdr)
1547{
1548 lockdep_assert_cpus_held();
1549
1550 INIT_LIST_HEAD(&hdr->list);
1551 hdr->id = mpam_resctrl_pick_domain_id(cpu, comp);
1552 hdr->rid = rid;
1553 cpumask_set_cpu(cpu, &hdr->cpu_mask);
1554}
1555
1556static void mpam_resctrl_online_domain_hdr(unsigned int cpu,
1557 struct rdt_domain_hdr *hdr)
1558{
1559 lockdep_assert_cpus_held();
1560
1561 cpumask_set_cpu(cpu, &hdr->cpu_mask);
1562}
1563
1564/**
1565 * mpam_resctrl_offline_domain_hdr() - Update the domain header to remove a CPU.
1566 * @cpu: The CPU to remove from the domain.
1567 * @hdr: The domain's header.
1568 *
1569 * Removes @cpu from the header mask. If this was the last CPU in the domain,
1570 * the domain header is removed from its parent list and true is returned,
1571 * indicating the parent structure can be freed.
1572 * If there are other CPUs in the domain, returns false.
1573 */
1574static bool mpam_resctrl_offline_domain_hdr(unsigned int cpu,
1575 struct rdt_domain_hdr *hdr)
1576{
1577 lockdep_assert_held(&domain_list_lock);
1578
1579 cpumask_clear_cpu(cpu, &hdr->cpu_mask);
1580 if (cpumask_empty(&hdr->cpu_mask)) {
1581 list_del_rcu(&hdr->list);
1582 synchronize_rcu();
1583 return true;
1584 }
1585
1586 return false;
1587}
1588
1589static void mpam_resctrl_domain_insert(struct list_head *list,
1590 struct rdt_domain_hdr *new)
1591{
1592 struct rdt_domain_hdr *err;
1593 struct list_head *pos = NULL;
1594
1595 lockdep_assert_held(&domain_list_lock);
1596
1597 err = resctrl_find_domain(list, new->id, &pos);
1598 if (WARN_ON_ONCE(err))
1599 return;
1600
1601 list_add_tail_rcu(&new->list, pos);
1602}
1603
1604static struct mpam_component *find_component(struct mpam_class *class, int cpu)
1605{
1606 struct mpam_component *comp;
1607
1608 guard(srcu)(&mpam_srcu);
1609 list_for_each_entry_srcu(comp, &class->components, class_list,
1610 srcu_read_lock_held(&mpam_srcu)) {
1611 if (cpumask_test_cpu(cpu, &comp->affinity))
1612 return comp;
1613 }
1614
1615 return NULL;
1616}
1617
1618static struct mpam_resctrl_dom *
1619mpam_resctrl_alloc_domain(unsigned int cpu, struct mpam_resctrl_res *res)
1620{
1621 int err;
1622 struct mpam_resctrl_dom *dom;
1623 struct rdt_l3_mon_domain *mon_d;
1624 struct rdt_ctrl_domain *ctrl_d;
1625 struct mpam_class *class = res->class;
1626 struct mpam_component *comp_iter, *ctrl_comp;
1627 struct rdt_resource *r = &res->resctrl_res;
1628
1629 lockdep_assert_held(&domain_list_lock);
1630
1631 ctrl_comp = NULL;
1632 guard(srcu)(&mpam_srcu);
1633 list_for_each_entry_srcu(comp_iter, &class->components, class_list,
1634 srcu_read_lock_held(&mpam_srcu)) {
1635 if (cpumask_test_cpu(cpu, &comp_iter->affinity)) {
1636 ctrl_comp = comp_iter;
1637 break;
1638 }
1639 }
1640
1641 /* class has no component for this CPU */
1642 if (WARN_ON_ONCE(!ctrl_comp))
1643 return ERR_PTR(-EINVAL);
1644
1645 dom = kzalloc_node(sizeof(*dom), GFP_KERNEL, cpu_to_node(cpu));
1646 if (!dom)
1647 return ERR_PTR(-ENOMEM);
1648
1649 if (r->alloc_capable) {
1650 dom->ctrl_comp = ctrl_comp;
1651
1652 ctrl_d = &dom->resctrl_ctrl_dom;
1653 mpam_resctrl_domain_hdr_init(cpu, ctrl_comp, r->rid, &ctrl_d->hdr);
1654 ctrl_d->hdr.type = RESCTRL_CTRL_DOMAIN;
1655 err = resctrl_online_ctrl_domain(r, ctrl_d);
1656 if (err)
1657 goto free_domain;
1658
1659 mpam_resctrl_domain_insert(&r->ctrl_domains, &ctrl_d->hdr);
1660 } else {
1661 pr_debug("Skipped control domain online - no controls\n");
1662 }
1663
1664 if (r->mon_capable) {
1665 struct mpam_component *any_mon_comp = NULL;
1666 struct mpam_resctrl_mon *mon;
1667 enum resctrl_event_id eventid;
1668
1669 /*
1670 * Even if the monitor domain is backed by a different
1671 * component, the L3 component IDs need to be used... only
1672 * there may be no ctrl_comp for the L3.
1673 * Search each event's class list for a component with
1674 * overlapping CPUs and set up the dom->mon_comp array.
1675 */
1676
1677 for_each_mpam_resctrl_mon(mon, eventid) {
1678 struct mpam_component *mon_comp;
1679
1680 if (!mon->class)
1681 continue; // dummy resource
1682
1683 mon_comp = find_component(mon->class, cpu);
1684 dom->mon_comp[eventid] = mon_comp;
1685 if (mon_comp)
1686 any_mon_comp = mon_comp;
1687 }
1688 if (!any_mon_comp) {
1689 WARN_ON_ONCE(0);
1690 err = -EFAULT;
1691 goto offline_ctrl_domain;
1692 }
1693
1694 mon_d = &dom->resctrl_mon_dom;
1695 mpam_resctrl_domain_hdr_init(cpu, any_mon_comp, r->rid, &mon_d->hdr);
1696 mon_d->hdr.type = RESCTRL_MON_DOMAIN;
1697 err = resctrl_online_mon_domain(r, &mon_d->hdr);
1698 if (err)
1699 goto offline_ctrl_domain;
1700
1701 mpam_resctrl_domain_insert(&r->mon_domains, &mon_d->hdr);
1702 } else {
1703 pr_debug("Skipped monitor domain online - no monitors\n");
1704 }
1705
1706 return dom;
1707
1708offline_ctrl_domain:
1709 if (r->alloc_capable) {
1710 mpam_resctrl_offline_domain_hdr(cpu, &ctrl_d->hdr);
1711 resctrl_offline_ctrl_domain(r, ctrl_d);
1712 }
1713free_domain:
1714 kfree(dom);
1715 dom = ERR_PTR(err);
1716
1717 return dom;
1718}
1719
1720/*
1721 * We know all the monitors are associated with the L3, even if there are no
1722 * controls and therefore no control component. Find the cache-id for the CPU
1723 * and use that to search for existing resctrl domains.
1724 * This relies on mpam_resctrl_pick_domain_id() using the L3 cache-id
1725 * for anything that is not a cache.
1726 */
1727static struct mpam_resctrl_dom *mpam_resctrl_get_mon_domain_from_cpu(int cpu)
1728{
1729 int cache_id;
1730 struct mpam_resctrl_dom *dom;
1731 struct mpam_resctrl_res *l3 = &mpam_resctrl_controls[RDT_RESOURCE_L3];
1732
1733 lockdep_assert_cpus_held();
1734
1735 if (!l3->class)
1736 return NULL;
1737 cache_id = get_cpu_cacheinfo_id(cpu, 3);
1738 if (cache_id < 0)
1739 return NULL;
1740
1741 list_for_each_entry_rcu(dom, &l3->resctrl_res.mon_domains, resctrl_mon_dom.hdr.list) {
1742 if (dom->resctrl_mon_dom.hdr.id == cache_id)
1743 return dom;
1744 }
1745
1746 return NULL;
1747}
1748
1749static struct mpam_resctrl_dom *
1750mpam_resctrl_get_domain_from_cpu(int cpu, struct mpam_resctrl_res *res)
1751{
1752 struct mpam_resctrl_dom *dom;
1753 struct rdt_resource *r = &res->resctrl_res;
1754
1755 lockdep_assert_cpus_held();
1756
1757 list_for_each_entry_rcu(dom, &r->ctrl_domains, resctrl_ctrl_dom.hdr.list) {
1758 if (cpumask_test_cpu(cpu, &dom->ctrl_comp->affinity))
1759 return dom;
1760 }
1761
1762 if (r->rid != RDT_RESOURCE_L3)
1763 return NULL;
1764
1765 /* Search the mon domain list too - needed on monitor only platforms. */
1766 return mpam_resctrl_get_mon_domain_from_cpu(cpu);
1767}
1768
1769int mpam_resctrl_online_cpu(unsigned int cpu)
1770{
1771 struct mpam_resctrl_res *res;
1772 enum resctrl_res_level rid;
1773
1774 guard(mutex)(&domain_list_lock);
1775 for_each_mpam_resctrl_control(res, rid) {
1776 struct mpam_resctrl_dom *dom;
1777 struct rdt_resource *r = &res->resctrl_res;
1778
1779 if (!res->class)
1780 continue; // dummy_resource;
1781
1782 dom = mpam_resctrl_get_domain_from_cpu(cpu, res);
1783 if (!dom) {
1784 dom = mpam_resctrl_alloc_domain(cpu, res);
1785 if (IS_ERR(dom))
1786 return PTR_ERR(dom);
1787 } else {
1788 if (r->alloc_capable) {
1789 struct rdt_ctrl_domain *ctrl_d = &dom->resctrl_ctrl_dom;
1790
1791 mpam_resctrl_online_domain_hdr(cpu, &ctrl_d->hdr);
1792 }
1793 if (r->mon_capable) {
1794 struct rdt_l3_mon_domain *mon_d = &dom->resctrl_mon_dom;
1795
1796 mpam_resctrl_online_domain_hdr(cpu, &mon_d->hdr);
1797 }
1798 }
1799 }
1800
1801 resctrl_online_cpu(cpu);
1802
1803 return 0;
1804}
1805
1806void mpam_resctrl_offline_cpu(unsigned int cpu)
1807{
1808 struct mpam_resctrl_res *res;
1809 enum resctrl_res_level rid;
1810
1811 resctrl_offline_cpu(cpu);
1812
1813 guard(mutex)(&domain_list_lock);
1814 for_each_mpam_resctrl_control(res, rid) {
1815 struct mpam_resctrl_dom *dom;
1816 struct rdt_l3_mon_domain *mon_d;
1817 struct rdt_ctrl_domain *ctrl_d;
1818 bool ctrl_dom_empty, mon_dom_empty;
1819 struct rdt_resource *r = &res->resctrl_res;
1820
1821 if (!res->class)
1822 continue; // dummy resource
1823
1824 dom = mpam_resctrl_get_domain_from_cpu(cpu, res);
1825 if (WARN_ON_ONCE(!dom))
1826 continue;
1827
1828 if (r->alloc_capable) {
1829 ctrl_d = &dom->resctrl_ctrl_dom;
1830 ctrl_dom_empty = mpam_resctrl_offline_domain_hdr(cpu, &ctrl_d->hdr);
1831 if (ctrl_dom_empty)
1832 resctrl_offline_ctrl_domain(&res->resctrl_res, ctrl_d);
1833 } else {
1834 ctrl_dom_empty = true;
1835 }
1836
1837 if (r->mon_capable) {
1838 mon_d = &dom->resctrl_mon_dom;
1839 mon_dom_empty = mpam_resctrl_offline_domain_hdr(cpu, &mon_d->hdr);
1840 if (mon_dom_empty)
1841 resctrl_offline_mon_domain(&res->resctrl_res, &mon_d->hdr);
1842 } else {
1843 mon_dom_empty = true;
1844 }
1845
1846 if (ctrl_dom_empty && mon_dom_empty)
1847 kfree(dom);
1848 }
1849}
1850
1851int mpam_resctrl_setup(void)
1852{
1853 int err = 0;
1854 struct mpam_resctrl_res *res;
1855 enum resctrl_res_level rid;
1856 struct mpam_resctrl_mon *mon;
1857 enum resctrl_event_id eventid;
1858
1859 wait_event(wait_cacheinfo_ready, cacheinfo_ready);
1860
1861 cpus_read_lock();
1862 for_each_mpam_resctrl_control(res, rid) {
1863 INIT_LIST_HEAD_RCU(&res->resctrl_res.ctrl_domains);
1864 INIT_LIST_HEAD_RCU(&res->resctrl_res.mon_domains);
1865 res->resctrl_res.rid = rid;
1866 }
1867
1868 /* Find some classes to use for controls */
1869 mpam_resctrl_pick_caches();
1870 mpam_resctrl_pick_mba();
1871
1872 /* Initialise the resctrl structures from the classes */
1873 for_each_mpam_resctrl_control(res, rid) {
1874 if (!res->class)
1875 continue; // dummy resource
1876
1877 err = mpam_resctrl_control_init(res);
1878 if (err) {
1879 pr_debug("Failed to initialise rid %u\n", rid);
1880 goto internal_error;
1881 }
1882 }
1883
1884 /* Find some classes to use for monitors */
1885 mpam_resctrl_pick_counters();
1886
1887 for_each_mpam_resctrl_mon(mon, eventid) {
1888 if (!mon->class)
1889 continue; // dummy resource
1890
1891 err = mpam_resctrl_monitor_init(mon, eventid);
1892 if (err) {
1893 pr_debug("Failed to initialise event %u\n", eventid);
1894 goto internal_error;
1895 }
1896 }
1897
1898 cpus_read_unlock();
1899
1900 if (!resctrl_arch_alloc_capable() && !resctrl_arch_mon_capable()) {
1901 pr_debug("No alloc(%u) or monitor(%u) found - resctrl not supported\n",
1902 resctrl_arch_alloc_capable(), resctrl_arch_mon_capable());
1903 return -EOPNOTSUPP;
1904 }
1905
1906 err = resctrl_init();
1907 if (err)
1908 return err;
1909
1910 WRITE_ONCE(resctrl_enabled, true);
1911
1912 return 0;
1913
1914internal_error:
1915 cpus_read_unlock();
1916 pr_debug("Internal error %d - resctrl not supported\n", err);
1917 return err;
1918}
1919
1920void mpam_resctrl_exit(void)
1921{
1922 if (!READ_ONCE(resctrl_enabled))
1923 return;
1924
1925 WRITE_ONCE(resctrl_enabled, false);
1926 resctrl_exit();
1927}
1928
1929static void mpam_resctrl_teardown_mon(struct mpam_resctrl_mon *mon, struct mpam_class *class)
1930{
1931 u32 num_mbwu_mon = class->props.num_mbwu_mon;
1932
1933 if (!mon->mbwu_idx_to_mon)
1934 return;
1935
1936 if (mon->assigned_counters) {
1937 __free_mbwu_mon(class, mon->assigned_counters, num_mbwu_mon);
1938 kvfree(mon->assigned_counters);
1939 mon->assigned_counters = NULL;
1940 }
1941
1942 kvfree(mon->mbwu_idx_to_mon);
1943 mon->mbwu_idx_to_mon = NULL;
1944}
1945
1946/*
1947 * The driver is detaching an MSC from this class, if resctrl was using it,
1948 * pull on resctrl_exit().
1949 */
1950void mpam_resctrl_teardown_class(struct mpam_class *class)
1951{
1952 struct mpam_resctrl_res *res;
1953 enum resctrl_res_level rid;
1954 struct mpam_resctrl_mon *mon;
1955 enum resctrl_event_id eventid;
1956
1957 might_sleep();
1958
1959 for_each_mpam_resctrl_control(res, rid) {
1960 if (res->class == class) {
1961 res->class = NULL;
1962 break;
1963 }
1964 }
1965 for_each_mpam_resctrl_mon(mon, eventid) {
1966 if (mon->class == class) {
1967 mon->class = NULL;
1968
1969 mpam_resctrl_teardown_mon(mon, class);
1970 break;
1971 }
1972 }
1973}
1974
1975static int __init __cacheinfo_ready(void)
1976{
1977 cacheinfo_ready = true;
1978 wake_up(&wait_cacheinfo_ready);
1979
1980 return 0;
1981}
1982device_initcall_sync(__cacheinfo_ready);
1983
1984#ifdef CONFIG_MPAM_KUNIT_TEST
1985#include "test_mpam_resctrl.c"
1986#endif