projects/linux·blob master · refs

1984 lines · 50.7 KiB

1// SPDX-License-Identifier: GPL-2.0
2// Copyright (C) 2025 Arm Ltd.
3
4#define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__
5
6#include <linux/arm_mpam.h>
7#include <linux/cacheinfo.h>
8#include <linux/cpu.h>
9#include <linux/cpumask.h>
10#include <linux/errno.h>
11#include <linux/limits.h>
12#include <linux/list.h>
13#include <linux/math.h>
14#include <linux/printk.h>
15#include <linux/rculist.h>
16#include <linux/resctrl.h>
17#include <linux/slab.h>
18#include <linux/types.h>
19#include <linux/wait.h>
20
21#include <asm/mpam.h>
22
23#include "mpam_internal.h"
24
25static DECLARE_WAIT_QUEUE_HEAD(resctrl_mon_ctx_waiters);
26
27/*
28 * The classes we've picked to map to resctrl resources, wrapped
29 * in with their resctrl structure.
30 * Class pointer may be NULL.
31 */
32static struct mpam_resctrl_res mpam_resctrl_controls[RDT_NUM_RESOURCES];
33
34#define for_each_mpam_resctrl_control(res, rid) \
35 for (rid = 0, res = &mpam_resctrl_controls[rid]; \
36 rid < RDT_NUM_RESOURCES; \
37 rid++, res = &mpam_resctrl_controls[rid])
38
39/*
40 * The classes we've picked to map to resctrl events.
41 * Resctrl believes all the worlds a Xeon, and these are all on the L3. This
42 * array lets us find the actual class backing the event counters. e.g.
43 * the only memory bandwidth counters may be on the memory controller, but to
44 * make use of them, we pretend they are on L3. Restrict the events considered
45 * to those supported by MPAM.
46 * Class pointer may be NULL.
47 */
48#define MPAM_MAX_EVENT QOS_L3_MBM_TOTAL_EVENT_ID
49static struct mpam_resctrl_mon mpam_resctrl_counters[MPAM_MAX_EVENT + 1];
50
51#define for_each_mpam_resctrl_mon(mon, eventid) \
52 for (eventid = QOS_FIRST_EVENT, mon = &mpam_resctrl_counters[eventid]; \
53 eventid <= MPAM_MAX_EVENT; \
54 eventid++, mon = &mpam_resctrl_counters[eventid])
55
56/* The lock for modifying resctrl's domain lists from cpuhp callbacks. */
57static DEFINE_MUTEX(domain_list_lock);
58
59/*
60 * MPAM emulates CDP by setting different PARTID in the I/D fields of MPAM0_EL1.
61 * This applies globally to all traffic the CPU generates.
62 */
63static bool cdp_enabled;
64
65/*
66 * We use cacheinfo to discover the size of the caches and their id. cacheinfo
67 * populates this from a device_initcall(). mpam_resctrl_setup() must wait.
68 */
69static bool cacheinfo_ready;
70static DECLARE_WAIT_QUEUE_HEAD(wait_cacheinfo_ready);
71
72/*
73 * If resctrl_init() succeeded, resctrl_exit() can be used to remove support
74 * for the filesystem in the event of an error.
75 */
76static bool resctrl_enabled;
77
78bool resctrl_arch_alloc_capable(void)
79{
80 struct mpam_resctrl_res *res;
81 enum resctrl_res_level rid;
82
83 for_each_mpam_resctrl_control(res, rid) {
84 if (res->resctrl_res.alloc_capable)
85 return true;
86 }
87
88 return false;
89}
90
91bool resctrl_arch_mon_capable(void)
92{
93 struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3];
94 struct rdt_resource *l3 = &res->resctrl_res;
95
96 /* All monitors are presented as being on the L3 cache */
97 return l3->mon_capable;
98}
99
100bool resctrl_arch_is_evt_configurable(enum resctrl_event_id evt)
101{
102 return false;
103}
104
105void resctrl_arch_mon_event_config_read(void *info)
106{
107}
108
109void resctrl_arch_mon_event_config_write(void *info)
110{
111}
112
113void resctrl_arch_reset_rmid_all(struct rdt_resource *r, struct rdt_l3_mon_domain *d)
114{
115}
116
117void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_l3_mon_domain *d,
118 u32 closid, u32 rmid, enum resctrl_event_id eventid)
119{
120}
121
122bool resctrl_arch_mbm_cntr_assign_enabled(struct rdt_resource *r)
123{
124 return (r == &mpam_resctrl_controls[RDT_RESOURCE_L3].resctrl_res);
125}
126
127int resctrl_arch_mbm_cntr_assign_set(struct rdt_resource *r, bool enable)
128{
129 return -EINVAL;
130}
131
132int resctrl_arch_io_alloc_enable(struct rdt_resource *r, bool enable)
133{
134 return -EOPNOTSUPP;
135}
136
137bool resctrl_arch_get_io_alloc_enabled(struct rdt_resource *r)
138{
139 return false;
140}
141
142void resctrl_arch_pre_mount(void)
143{
144}
145
146bool resctrl_arch_get_cdp_enabled(enum resctrl_res_level rid)
147{
148 return mpam_resctrl_controls[rid].cdp_enabled;
149}
150
151u32 resctrl_arch_preconvert_bw(const struct rdt_resource *r, u32 val)
152{
153 return val;
154}
155
156/**
157 * resctrl_reset_task_closids() - Reset the PARTID/PMG values for all tasks.
158 *
159 * At boot, all existing tasks use partid zero for D and I.
160 * To enable/disable CDP emulation, all these tasks need relabelling.
161 */
162static void resctrl_reset_task_closids(void)
163{
164 struct task_struct *p, *t;
165
166 read_lock(&tasklist_lock);
167 for_each_process_thread(p, t) {
168 resctrl_arch_set_closid_rmid(t, RESCTRL_RESERVED_CLOSID,
169 RESCTRL_RESERVED_RMID);
170 }
171 read_unlock(&tasklist_lock);
172}
173
174static void mpam_resctrl_monitor_sync_abmc_vals(struct rdt_resource *l3)
175{
176 struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[QOS_L3_MBM_TOTAL_EVENT_ID];
177
178 if (!mon->class)
179 return;
180
181 if (!mon->assigned_counters)
182 return;
183
184 l3->mon.num_mbm_cntrs = mon->class->props.num_mbwu_mon;
185 if (cdp_enabled)
186 l3->mon.num_mbm_cntrs /= 2;
187
188 /*
189 * Continue as normal even if enabling cdp causes there to be
190 * zero counters. This avoids giving resctrl mixed messages.
191 */
192}
193
194int resctrl_arch_set_cdp_enabled(enum resctrl_res_level rid, bool enable)
195{
196 u32 partid_i = RESCTRL_RESERVED_CLOSID, partid_d = RESCTRL_RESERVED_CLOSID;
197 struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3];
198 struct rdt_resource *l3 = &res->resctrl_res;
199 int cpu;
200
201 if (!IS_ENABLED(CONFIG_EXPERT) && enable) {
202 /*
203 * If the resctrl fs is mounted more than once, sequentially,
204 * then CDP can lead to the use of out of range PARTIDs.
205 */
206 pr_warn("CDP not supported\n");
207 return -EOPNOTSUPP;
208 }
209
210 if (enable)
211 pr_warn("CDP is an expert feature and may cause MPAM to malfunction.\n");
212
213 /*
214 * resctrl_arch_set_cdp_enabled() is only called with enable set to
215 * false on error and unmount.
216 */
217 cdp_enabled = enable;
218 mpam_resctrl_controls[rid].cdp_enabled = enable;
219
220 if (enable)
221 l3->mon.num_rmid = resctrl_arch_system_num_rmid_idx() / 2;
222 else
223 l3->mon.num_rmid = resctrl_arch_system_num_rmid_idx();
224
225 /* The mbw_max feature can't hide cdp as it's a per-partid maximum. */
226 if (cdp_enabled && !mpam_resctrl_controls[RDT_RESOURCE_MBA].cdp_enabled)
227 mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = false;
228
229 /*
230 * If resctrl has attempted to enable CDP on MBA, re-enable MBA as two
231 * configurations will be provided so there is no aliasing problem.
232 */
233 if (mpam_resctrl_controls[RDT_RESOURCE_MBA].cdp_enabled &&
234 mpam_resctrl_controls[RDT_RESOURCE_MBA].class)
235 mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = true;
236
237 /* On unmount when CDP is disabled, re-enable MBA */
238 if (!cdp_enabled && mpam_resctrl_controls[RDT_RESOURCE_MBA].class)
239 mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = true;
240
241 if (enable) {
242 if (mpam_partid_max < 1)
243 return -EINVAL;
244
245 partid_d = resctrl_get_config_index(RESCTRL_RESERVED_CLOSID, CDP_DATA);
246 partid_i = resctrl_get_config_index(RESCTRL_RESERVED_CLOSID, CDP_CODE);
247 }
248
249 mpam_set_task_partid_pmg(current, partid_d, partid_i, 0, 0);
250 WRITE_ONCE(arm64_mpam_global_default, mpam_get_regval(current));
251
252 resctrl_reset_task_closids();
253 mpam_resctrl_monitor_sync_abmc_vals(l3);
254
255 for_each_possible_cpu(cpu)
256 mpam_set_cpu_defaults(cpu, partid_d, partid_i, 0, 0);
257 on_each_cpu(resctrl_arch_sync_cpu_closid_rmid, NULL, 1);
258
259 return 0;
260}
261
262static bool mpam_resctrl_hide_cdp(enum resctrl_res_level rid)
263{
264 return cdp_enabled && !resctrl_arch_get_cdp_enabled(rid);
265}
266
267/*
268 * MSC may raise an error interrupt if it sees an out or range partid/pmg,
269 * and go on to truncate the value. Regardless of what the hardware supports,
270 * only the system wide safe value is safe to use.
271 */
272u32 resctrl_arch_get_num_closid(struct rdt_resource *ignored)
273{
274 return mpam_partid_max + 1;
275}
276
277u32 resctrl_arch_system_num_rmid_idx(void)
278{
279 return (mpam_pmg_max + 1) * (mpam_partid_max + 1);
280}
281
282u32 resctrl_arch_rmid_idx_encode(u32 closid, u32 rmid)
283{
284 return closid * (mpam_pmg_max + 1) + rmid;
285}
286
287void resctrl_arch_rmid_idx_decode(u32 idx, u32 *closid, u32 *rmid)
288{
289 *closid = idx / (mpam_pmg_max + 1);
290 *rmid = idx % (mpam_pmg_max + 1);
291}
292
293void resctrl_arch_sched_in(struct task_struct *tsk)
294{
295 lockdep_assert_preemption_disabled();
296
297 mpam_thread_switch(tsk);
298}
299
300void resctrl_arch_set_cpu_default_closid_rmid(int cpu, u32 closid, u32 rmid)
301{
302 WARN_ON_ONCE(closid > U16_MAX);
303 WARN_ON_ONCE(rmid > U8_MAX);
304
305 if (!cdp_enabled) {
306 mpam_set_cpu_defaults(cpu, closid, closid, rmid, rmid);
307 } else {
308 /*
309 * When CDP is enabled, resctrl halves the closid range and we
310 * use odd/even partid for one closid.
311 */
312 u32 partid_d = resctrl_get_config_index(closid, CDP_DATA);
313 u32 partid_i = resctrl_get_config_index(closid, CDP_CODE);
314
315 mpam_set_cpu_defaults(cpu, partid_d, partid_i, rmid, rmid);
316 }
317}
318
319void resctrl_arch_sync_cpu_closid_rmid(void *info)
320{
321 struct resctrl_cpu_defaults *r = info;
322
323 lockdep_assert_preemption_disabled();
324
325 if (r) {
326 resctrl_arch_set_cpu_default_closid_rmid(smp_processor_id(),
327 r->closid, r->rmid);
328 }
329
330 resctrl_arch_sched_in(current);
331}
332
333void resctrl_arch_set_closid_rmid(struct task_struct *tsk, u32 closid, u32 rmid)
334{
335 WARN_ON_ONCE(closid > U16_MAX);
336 WARN_ON_ONCE(rmid > U8_MAX);
337
338 if (!cdp_enabled) {
339 mpam_set_task_partid_pmg(tsk, closid, closid, rmid, rmid);
340 } else {
341 u32 partid_d = resctrl_get_config_index(closid, CDP_DATA);
342 u32 partid_i = resctrl_get_config_index(closid, CDP_CODE);
343
344 mpam_set_task_partid_pmg(tsk, partid_d, partid_i, rmid, rmid);
345 }
346}
347
348bool resctrl_arch_match_closid(struct task_struct *tsk, u32 closid)
349{
350 u64 regval = mpam_get_regval(tsk);
351 u32 tsk_closid = FIELD_GET(MPAM0_EL1_PARTID_D, regval);
352
353 if (cdp_enabled)
354 tsk_closid >>= 1;
355
356 return tsk_closid == closid;
357}
358
359/* The task's pmg is not unique, the partid must be considered too */
360bool resctrl_arch_match_rmid(struct task_struct *tsk, u32 closid, u32 rmid)
361{
362 u64 regval = mpam_get_regval(tsk);
363 u32 tsk_closid = FIELD_GET(MPAM0_EL1_PARTID_D, regval);
364 u32 tsk_rmid = FIELD_GET(MPAM0_EL1_PMG_D, regval);
365
366 if (cdp_enabled)
367 tsk_closid >>= 1;
368
369 return (tsk_closid == closid) && (tsk_rmid == rmid);
370}
371
372struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l)
373{
374 if (l >= RDT_NUM_RESOURCES)
375 return NULL;
376
377 return &mpam_resctrl_controls[l].resctrl_res;
378}
379
380static int resctrl_arch_mon_ctx_alloc_no_wait(enum resctrl_event_id evtid)
381{
382 struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[evtid];
383
384 if (!mpam_is_enabled())
385 return -EINVAL;
386
387 if (!mon->class)
388 return -EINVAL;
389
390 switch (evtid) {
391 case QOS_L3_OCCUP_EVENT_ID:
392 /* With CDP, one monitor gets used for both code/data reads */
393 return mpam_alloc_csu_mon(mon->class);
394 case QOS_L3_MBM_LOCAL_EVENT_ID:
395 case QOS_L3_MBM_TOTAL_EVENT_ID:
396 return USE_PRE_ALLOCATED;
397 default:
398 return -EOPNOTSUPP;
399 }
400}
401
402void *resctrl_arch_mon_ctx_alloc(struct rdt_resource *r,
403 enum resctrl_event_id evtid)
404{
405 DEFINE_WAIT(wait);
406 int *ret;
407
408 ret = kmalloc_obj(*ret);
409 if (!ret)
410 return ERR_PTR(-ENOMEM);
411
412 do {
413 prepare_to_wait(&resctrl_mon_ctx_waiters, &wait,
414 TASK_INTERRUPTIBLE);
415 *ret = resctrl_arch_mon_ctx_alloc_no_wait(evtid);
416 if (*ret == -ENOSPC)
417 schedule();
418 } while (*ret == -ENOSPC && !signal_pending(current));
419 finish_wait(&resctrl_mon_ctx_waiters, &wait);
420
421 return ret;
422}
423
424static void resctrl_arch_mon_ctx_free_no_wait(enum resctrl_event_id evtid,
425 u32 mon_idx)
426{
427 struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[evtid];
428
429 if (!mpam_is_enabled())
430 return;
431
432 if (!mon->class)
433 return;
434
435 if (evtid == QOS_L3_OCCUP_EVENT_ID)
436 mpam_free_csu_mon(mon->class, mon_idx);
437
438 wake_up(&resctrl_mon_ctx_waiters);
439}
440
441void resctrl_arch_mon_ctx_free(struct rdt_resource *r,
442 enum resctrl_event_id evtid, void *arch_mon_ctx)
443{
444 u32 mon_idx = *(u32 *)arch_mon_ctx;
445
446 kfree(arch_mon_ctx);
447
448 resctrl_arch_mon_ctx_free_no_wait(evtid, mon_idx);
449}
450
451static int __read_mon(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp,
452 enum mpam_device_features mon_type,
453 int mon_idx,
454 enum resctrl_conf_type cdp_type, u32 closid, u32 rmid, u64 *val)
455{
456 struct mon_cfg cfg;
457
458 if (!mpam_is_enabled())
459 return -EINVAL;
460
461 /* Shift closid to account for CDP */
462 closid = resctrl_get_config_index(closid, cdp_type);
463
464 if (mon_idx == USE_PRE_ALLOCATED) {
465 int mbwu_idx = resctrl_arch_rmid_idx_encode(closid, rmid);
466
467 mon_idx = mon->mbwu_idx_to_mon[mbwu_idx];
468 if (mon_idx == -1)
469 return -ENOENT;
470 }
471
472 if (irqs_disabled()) {
473 /* Check if we can access this domain without an IPI */
474 return -EIO;
475 }
476
477 cfg = (struct mon_cfg) {
478 .mon = mon_idx,
479 .match_pmg = true,
480 .partid = closid,
481 .pmg = rmid,
482 };
483
484 return mpam_msmon_read(mon_comp, &cfg, mon_type, val);
485}
486
487static int read_mon_cdp_safe(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp,
488 enum mpam_device_features mon_type,
489 int mon_idx, u32 closid, u32 rmid, u64 *val)
490{
491 if (cdp_enabled) {
492 u64 code_val = 0, data_val = 0;
493 int err;
494
495 err = __read_mon(mon, mon_comp, mon_type, mon_idx,
496 CDP_CODE, closid, rmid, &code_val);
497 if (err)
498 return err;
499
500 err = __read_mon(mon, mon_comp, mon_type, mon_idx,
501 CDP_DATA, closid, rmid, &data_val);
502 if (err)
503 return err;
504
505 *val = code_val + data_val;
506 return 0;
507 }
508
509 return __read_mon(mon, mon_comp, mon_type, mon_idx,
510 CDP_NONE, closid, rmid, val);
511}
512
513/* MBWU when not in ABMC mode (not supported), and CSU counters. */
514int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain_hdr *hdr,
515 u32 closid, u32 rmid, enum resctrl_event_id eventid,
516 void *arch_priv, u64 *val, void *arch_mon_ctx)
517{
518 struct mpam_resctrl_dom *l3_dom;
519 struct mpam_component *mon_comp;
520 u32 mon_idx = *(u32 *)arch_mon_ctx;
521 enum mpam_device_features mon_type;
522 struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[eventid];
523
524 resctrl_arch_rmid_read_context_check();
525
526 if (!mpam_is_enabled())
527 return -EINVAL;
528
529 if (eventid >= QOS_NUM_EVENTS || !mon->class)
530 return -EINVAL;
531
532 l3_dom = container_of(hdr, struct mpam_resctrl_dom, resctrl_mon_dom.hdr);
533 mon_comp = l3_dom->mon_comp[eventid];
534
535 if (eventid != QOS_L3_OCCUP_EVENT_ID)
536 return -EINVAL;
537
538 mon_type = mpam_feat_msmon_csu;
539
540 return read_mon_cdp_safe(mon, mon_comp, mon_type, mon_idx,
541 closid, rmid, val);
542}
543
544/* MBWU counters when in ABMC mode */
545int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d,
546 u32 closid, u32 rmid, int mon_idx,
547 enum resctrl_event_id eventid, u64 *val)
548{
549 struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[eventid];
550 struct mpam_resctrl_dom *l3_dom;
551 struct mpam_component *mon_comp;
552
553 if (!mpam_is_enabled())
554 return -EINVAL;
555
556 if (eventid == QOS_L3_OCCUP_EVENT_ID || !mon->class)
557 return -EINVAL;
558
559 l3_dom = container_of(d, struct mpam_resctrl_dom, resctrl_mon_dom);
560 mon_comp = l3_dom->mon_comp[eventid];
561
562 return read_mon_cdp_safe(mon, mon_comp, mpam_feat_msmon_mbwu,
563 USE_PRE_ALLOCATED, closid, rmid, val);
564}
565
566static void __reset_mon(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp,
567 int mon_idx,
568 enum resctrl_conf_type cdp_type, u32 closid, u32 rmid)
569{
570 struct mon_cfg cfg = { };
571
572 if (!mpam_is_enabled())
573 return;
574
575 /* Shift closid to account for CDP */
576 closid = resctrl_get_config_index(closid, cdp_type);
577
578 if (mon_idx == USE_PRE_ALLOCATED) {
579 int mbwu_idx = resctrl_arch_rmid_idx_encode(closid, rmid);
580
581 mon_idx = mon->mbwu_idx_to_mon[mbwu_idx];
582 }
583
584 if (mon_idx == -1)
585 return;
586 cfg.mon = mon_idx;
587 mpam_msmon_reset_mbwu(mon_comp, &cfg);
588}
589
590static void reset_mon_cdp_safe(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp,
591 int mon_idx, u32 closid, u32 rmid)
592{
593 if (cdp_enabled) {
594 __reset_mon(mon, mon_comp, mon_idx, CDP_CODE, closid, rmid);
595 __reset_mon(mon, mon_comp, mon_idx, CDP_DATA, closid, rmid);
596 } else {
597 __reset_mon(mon, mon_comp, mon_idx, CDP_NONE, closid, rmid);
598 }
599}
600
601/* Reset an assigned counter */
602void resctrl_arch_reset_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d,
603 u32 closid, u32 rmid, int cntr_id,
604 enum resctrl_event_id eventid)
605{
606 struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[eventid];
607 struct mpam_resctrl_dom *l3_dom;
608 struct mpam_component *mon_comp;
609
610 if (!mpam_is_enabled())
611 return;
612
613 if (eventid == QOS_L3_OCCUP_EVENT_ID || !mon->class)
614 return;
615
616 l3_dom = container_of(d, struct mpam_resctrl_dom, resctrl_mon_dom);
617 mon_comp = l3_dom->mon_comp[eventid];
618
619 reset_mon_cdp_safe(mon, mon_comp, USE_PRE_ALLOCATED, closid, rmid);
620}
621
622/*
623 * The rmid realloc threshold should be for the smallest cache exposed to
624 * resctrl.
625 */
626static int update_rmid_limits(struct mpam_class *class)
627{
628 u32 num_unique_pmg = resctrl_arch_system_num_rmid_idx();
629 struct mpam_props *cprops = &class->props;
630 struct cacheinfo *ci;
631
632 lockdep_assert_cpus_held();
633
634 if (!mpam_has_feature(mpam_feat_msmon_csu, cprops))
635 return 0;
636
637 /*
638 * Assume cache levels are the same size for all CPUs...
639 * The check just requires any online CPU and it can't go offline as we
640 * hold the cpu lock.
641 */
642 ci = get_cpu_cacheinfo_level(raw_smp_processor_id(), class->level);
643 if (!ci || ci->size == 0) {
644 pr_debug("Could not read cache size for class %u\n",
645 class->level);
646 return -EINVAL;
647 }
648
649 if (!resctrl_rmid_realloc_limit ||
650 ci->size < resctrl_rmid_realloc_limit) {
651 resctrl_rmid_realloc_limit = ci->size;
652 resctrl_rmid_realloc_threshold = ci->size / num_unique_pmg;
653 }
654
655 return 0;
656}
657
658static bool cache_has_usable_cpor(struct mpam_class *class)
659{
660 struct mpam_props *cprops = &class->props;
661
662 if (!mpam_has_feature(mpam_feat_cpor_part, cprops))
663 return false;
664
665 /* resctrl uses u32 for all bitmap configurations */
666 return class->props.cpbm_wd <= 32;
667}
668
669static bool mba_class_use_mbw_max(struct mpam_props *cprops)
670{
671 return (mpam_has_feature(mpam_feat_mbw_max, cprops) &&
672 cprops->bwa_wd);
673}
674
675static bool class_has_usable_mba(struct mpam_props *cprops)
676{
677 return mba_class_use_mbw_max(cprops);
678}
679
680static bool cache_has_usable_csu(struct mpam_class *class)
681{
682 struct mpam_props *cprops;
683
684 if (!class)
685 return false;
686
687 cprops = &class->props;
688
689 if (!mpam_has_feature(mpam_feat_msmon_csu, cprops))
690 return false;
691
692 /*
693 * CSU counters settle on the value, so we can get away with
694 * having only one.
695 */
696 if (!cprops->num_csu_mon)
697 return false;
698
699 return true;
700}
701
702static bool class_has_usable_mbwu(struct mpam_class *class)
703{
704 struct mpam_props *cprops = &class->props;
705
706 if (!mpam_has_feature(mpam_feat_msmon_mbwu, cprops))
707 return false;
708
709 if (!cprops->num_mbwu_mon)
710 return false;
711
712 return true;
713}
714
715/*
716 * Calculate the worst-case percentage change from each implemented step
717 * in the control.
718 */
719static u32 get_mba_granularity(struct mpam_props *cprops)
720{
721 if (!mba_class_use_mbw_max(cprops))
722 return 0;
723
724 /*
725 * bwa_wd is the number of bits implemented in the 0.xxx
726 * fixed point fraction. 1 bit is 50%, 2 is 25% etc.
727 */
728 return DIV_ROUND_UP(MAX_MBA_BW, 1 << cprops->bwa_wd);
729}
730
731/*
732 * Each fixed-point hardware value architecturally represents a range
733 * of values: the full range 0% - 100% is split contiguously into
734 * (1 << cprops->bwa_wd) equal bands.
735 *
736 * Although the bwa_bwd fields have 6 bits the maximum valid value is 16
737 * as it reports the width of fields that are at most 16 bits. When
738 * fewer than 16 bits are valid the least significant bits are
739 * ignored. The implied binary point is kept between bits 15 and 16 and
740 * so the valid bits are leftmost.
741 *
742 * See ARM IHI0099B.a "MPAM system component specification", Section 9.3,
743 * "The fixed-point fractional format" for more information.
744 *
745 * Find the nearest percentage value to the upper bound of the selected band:
746 */
747static u32 mbw_max_to_percent(u16 mbw_max, struct mpam_props *cprops)
748{
749 u32 val = mbw_max;
750
751 val >>= 16 - cprops->bwa_wd;
752 val += 1;
753 val *= MAX_MBA_BW;
754 val = DIV_ROUND_CLOSEST(val, 1 << cprops->bwa_wd);
755
756 return val;
757}
758
759/*
760 * Find the band whose upper bound is closest to the specified percentage.
761 *
762 * A round-to-nearest policy is followed here as a balanced compromise
763 * between unexpected under-commit of the resource (where the total of
764 * a set of resource allocations after conversion is less than the
765 * expected total, due to rounding of the individual converted
766 * percentages) and over-commit (where the total of the converted
767 * allocations is greater than expected).
768 */
769static u16 percent_to_mbw_max(u8 pc, struct mpam_props *cprops)
770{
771 u32 val = pc;
772
773 val <<= cprops->bwa_wd;
774 val = DIV_ROUND_CLOSEST(val, MAX_MBA_BW);
775 val = max(val, 1) - 1;
776 val <<= 16 - cprops->bwa_wd;
777
778 return val;
779}
780
781static u32 get_mba_min(struct mpam_props *cprops)
782{
783 if (!mba_class_use_mbw_max(cprops)) {
784 WARN_ON_ONCE(1);
785 return 0;
786 }
787
788 return mbw_max_to_percent(0, cprops);
789}
790
791/* Find the L3 cache that has affinity with this CPU */
792static int find_l3_equivalent_bitmask(int cpu, cpumask_var_t tmp_cpumask)
793{
794 u32 cache_id = get_cpu_cacheinfo_id(cpu, 3);
795
796 lockdep_assert_cpus_held();
797
798 return mpam_get_cpumask_from_cache_id(cache_id, 3, tmp_cpumask);
799}
800
801/*
802 * topology_matches_l3() - Is the provided class the same shape as L3
803 * @victim: The class we'd like to pretend is L3.
804 *
805 * resctrl expects all the world's a Xeon, and all counters are on the
806 * L3. We allow some mapping counters on other classes. This requires
807 * that the CPU->domain mapping is the same kind of shape.
808 *
809 * Using cacheinfo directly would make this work even if resctrl can't
810 * use the L3 - but cacheinfo can't tell us anything about offline CPUs.
811 * Using the L3 resctrl domain list also depends on CPUs being online.
812 * Using the mpam_class we picked for L3 so we can use its domain list
813 * assumes that there are MPAM controls on the L3.
814 * Instead, this path eventually uses the mpam_get_cpumask_from_cache_id()
815 * helper which can tell us about offline CPUs ... but getting the cache_id
816 * to start with relies on at least one CPU per L3 cache being online at
817 * boot.
818 *
819 * Walk the victim component list and compare the affinity mask with the
820 * corresponding L3. The topology matches if each victim:component's affinity
821 * mask is the same as the CPU's corresponding L3's. These lists/masks are
822 * computed from firmware tables so don't change at runtime.
823 */
824static bool topology_matches_l3(struct mpam_class *victim)
825{
826 int cpu, err;
827 struct mpam_component *victim_iter;
828
829 lockdep_assert_cpus_held();
830
831 cpumask_var_t __free(free_cpumask_var) tmp_cpumask = CPUMASK_VAR_NULL;
832 if (!alloc_cpumask_var(&tmp_cpumask, GFP_KERNEL))
833 return false;
834
835 guard(srcu)(&mpam_srcu);
836 list_for_each_entry_srcu(victim_iter, &victim->components, class_list,
837 srcu_read_lock_held(&mpam_srcu)) {
838 if (cpumask_empty(&victim_iter->affinity)) {
839 pr_debug("class %u has CPU-less component %u - can't match L3!\n",
840 victim->level, victim_iter->comp_id);
841 return false;
842 }
843
844 cpu = cpumask_any_and(&victim_iter->affinity, cpu_online_mask);
845 if (WARN_ON_ONCE(cpu >= nr_cpu_ids))
846 return false;
847
848 cpumask_clear(tmp_cpumask);
849 err = find_l3_equivalent_bitmask(cpu, tmp_cpumask);
850 if (err) {
851 pr_debug("Failed to find L3's equivalent component to class %u component %u\n",
852 victim->level, victim_iter->comp_id);
853 return false;
854 }
855
856 /* Any differing bits in the affinity mask? */
857 if (!cpumask_equal(tmp_cpumask, &victim_iter->affinity)) {
858 pr_debug("class %u component %u has Mismatched CPU mask with L3 equivalent\n"
859 "L3:%*pbl != victim:%*pbl\n",
860 victim->level, victim_iter->comp_id,
861 cpumask_pr_args(tmp_cpumask),
862 cpumask_pr_args(&victim_iter->affinity));
863
864 return false;
865 }
866 }
867
868 return true;
869}
870
871/*
872 * Test if the traffic for a class matches that at egress from the L3. For
873 * MSC at memory controllers this is only possible if there is a single L3
874 * as otherwise the counters at the memory can include bandwidth from the
875 * non-local L3.
876 */
877static bool traffic_matches_l3(struct mpam_class *class)
878{
879 int err, cpu;
880
881 lockdep_assert_cpus_held();
882
883 if (class->type == MPAM_CLASS_CACHE && class->level == 3)
884 return true;
885
886 if (class->type == MPAM_CLASS_CACHE && class->level != 3) {
887 pr_debug("class %u is a different cache from L3\n", class->level);
888 return false;
889 }
890
891 if (class->type != MPAM_CLASS_MEMORY) {
892 pr_debug("class %u is neither of type cache or memory\n", class->level);
893 return false;
894 }
895
896 cpumask_var_t __free(free_cpumask_var) tmp_cpumask = CPUMASK_VAR_NULL;
897 if (!alloc_cpumask_var(&tmp_cpumask, GFP_KERNEL)) {
898 pr_debug("cpumask allocation failed\n");
899 return false;
900 }
901
902 cpu = cpumask_any_and(&class->affinity, cpu_online_mask);
903 err = find_l3_equivalent_bitmask(cpu, tmp_cpumask);
904 if (err) {
905 pr_debug("Failed to find L3 downstream to cpu %d\n", cpu);
906 return false;
907 }
908
909 if (!cpumask_equal(tmp_cpumask, cpu_possible_mask)) {
910 pr_debug("There is more than one L3\n");
911 return false;
912 }
913
914 /* Be strict; the traffic might stop in the intermediate cache. */
915 if (get_cpu_cacheinfo_id(cpu, 4) != -1) {
916 pr_debug("L3 isn't the last level of cache\n");
917 return false;
918 }
919
920 if (num_possible_nodes() > 1) {
921 pr_debug("There is more than one numa node\n");
922 return false;
923 }
924
925#ifdef CONFIG_HMEM_REPORTING
926 if (node_devices[cpu_to_node(cpu)]->cache_dev) {
927 pr_debug("There is a memory side cache\n");
928 return false;
929 }
930#endif
931
932 return true;
933}
934
935/* Test whether we can export MPAM_CLASS_CACHE:{2,3}? */
936static void mpam_resctrl_pick_caches(void)
937{
938 struct mpam_class *class;
939 struct mpam_resctrl_res *res;
940
941 lockdep_assert_cpus_held();
942
943 guard(srcu)(&mpam_srcu);
944 list_for_each_entry_srcu(class, &mpam_classes, classes_list,
945 srcu_read_lock_held(&mpam_srcu)) {
946 if (class->type != MPAM_CLASS_CACHE) {
947 pr_debug("class %u is not a cache\n", class->level);
948 continue;
949 }
950
951 if (class->level != 2 && class->level != 3) {
952 pr_debug("class %u is not L2 or L3\n", class->level);
953 continue;
954 }
955
956 if (!cache_has_usable_cpor(class)) {
957 pr_debug("class %u cache misses CPOR\n", class->level);
958 continue;
959 }
960
961 if (!cpumask_equal(&class->affinity, cpu_possible_mask)) {
962 pr_debug("class %u has missing CPUs, mask %*pb != %*pb\n", class->level,
963 cpumask_pr_args(&class->affinity),
964 cpumask_pr_args(cpu_possible_mask));
965 continue;
966 }
967
968 if (class->level == 2)
969 res = &mpam_resctrl_controls[RDT_RESOURCE_L2];
970 else
971 res = &mpam_resctrl_controls[RDT_RESOURCE_L3];
972 res->class = class;
973 }
974}
975
976static void mpam_resctrl_pick_mba(void)
977{
978 struct mpam_class *class, *candidate_class = NULL;
979 struct mpam_resctrl_res *res;
980
981 lockdep_assert_cpus_held();
982
983 guard(srcu)(&mpam_srcu);
984 list_for_each_entry_srcu(class, &mpam_classes, classes_list,
985 srcu_read_lock_held(&mpam_srcu)) {
986 struct mpam_props *cprops = &class->props;
987
988 if (class->level != 3 && class->type == MPAM_CLASS_CACHE) {
989 pr_debug("class %u is a cache but not the L3\n", class->level);
990 continue;
991 }
992
993 if (!class_has_usable_mba(cprops)) {
994 pr_debug("class %u has no bandwidth control\n",
995 class->level);
996 continue;
997 }
998
999 if (!cpumask_equal(&class->affinity, cpu_possible_mask)) {
1000 pr_debug("class %u has missing CPUs\n", class->level);
1001 continue;
1002 }
1003
1004 if (!topology_matches_l3(class)) {
1005 pr_debug("class %u topology doesn't match L3\n",
1006 class->level);
1007 continue;
1008 }
1009
1010 if (!traffic_matches_l3(class)) {
1011 pr_debug("class %u traffic doesn't match L3 egress\n",
1012 class->level);
1013 continue;
1014 }
1015
1016 /*
1017 * Pick a resource to be MBA that as close as possible to
1018 * the L3. mbm_total counts the bandwidth leaving the L3
1019 * cache and MBA should correspond as closely as possible
1020 * for proper operation of mba_sc.
1021 */
1022 if (!candidate_class || class->level < candidate_class->level)
1023 candidate_class = class;
1024 }
1025
1026 if (candidate_class) {
1027 pr_debug("selected class %u to back MBA\n",
1028 candidate_class->level);
1029 res = &mpam_resctrl_controls[RDT_RESOURCE_MBA];
1030 res->class = candidate_class;
1031 }
1032}
1033
1034static void __free_mbwu_mon(struct mpam_class *class, int *array,
1035 u16 num_mbwu_mon)
1036{
1037 for (int i = 0; i < num_mbwu_mon; i++) {
1038 if (array[i] < 0)
1039 continue;
1040
1041 mpam_free_mbwu_mon(class, array[i]);
1042 array[i] = -1;
1043 }
1044}
1045
1046static int __alloc_mbwu_mon(struct mpam_class *class, int *array,
1047 u16 num_mbwu_mon)
1048{
1049 for (int i = 0; i < num_mbwu_mon; i++) {
1050 int mbwu_mon = mpam_alloc_mbwu_mon(class);
1051
1052 if (mbwu_mon < 0) {
1053 __free_mbwu_mon(class, array, num_mbwu_mon);
1054 return mbwu_mon;
1055 }
1056 array[i] = mbwu_mon;
1057 }
1058
1059 return 0;
1060}
1061
1062static int *__alloc_mbwu_array(struct mpam_class *class, u16 num_mbwu_mon)
1063{
1064 int err;
1065
1066 int *array __free(kvfree) = kvmalloc_objs(*array, num_mbwu_mon);
1067 if (!array)
1068 return ERR_PTR(-ENOMEM);
1069
1070 memset(array, -1, num_mbwu_mon * sizeof(*array));
1071
1072 err = __alloc_mbwu_mon(class, array, num_mbwu_mon);
1073 if (err)
1074 return ERR_PTR(err);
1075 return_ptr(array);
1076}
1077
1078static void counter_update_class(enum resctrl_event_id evt_id,
1079 struct mpam_class *class)
1080{
1081 struct mpam_class *existing_class = mpam_resctrl_counters[evt_id].class;
1082
1083 if (existing_class) {
1084 if (existing_class->level == 3) {
1085 pr_debug("Existing class is L3 - L3 wins\n");
1086 return;
1087 }
1088
1089 if (existing_class->level < class->level) {
1090 pr_debug("Existing class is closer to L3, %u versus %u - closer is better\n",
1091 existing_class->level, class->level);
1092 return;
1093 }
1094 }
1095
1096 mpam_resctrl_counters[evt_id].class = class;
1097}
1098
1099static void mpam_resctrl_pick_counters(void)
1100{
1101 struct mpam_class *class;
1102
1103 lockdep_assert_cpus_held();
1104
1105 guard(srcu)(&mpam_srcu);
1106 list_for_each_entry_srcu(class, &mpam_classes, classes_list,
1107 srcu_read_lock_held(&mpam_srcu)) {
1108 /* The name of the resource is L3... */
1109 if (class->type == MPAM_CLASS_CACHE && class->level != 3) {
1110 pr_debug("class %u is a cache but not the L3", class->level);
1111 continue;
1112 }
1113
1114 if (!cpumask_equal(&class->affinity, cpu_possible_mask)) {
1115 pr_debug("class %u does not cover all CPUs",
1116 class->level);
1117 continue;
1118 }
1119
1120 if (cache_has_usable_csu(class)) {
1121 pr_debug("class %u has usable CSU",
1122 class->level);
1123
1124 /* CSU counters only make sense on a cache. */
1125 switch (class->type) {
1126 case MPAM_CLASS_CACHE:
1127 if (update_rmid_limits(class))
1128 break;
1129
1130 counter_update_class(QOS_L3_OCCUP_EVENT_ID, class);
1131 break;
1132 default:
1133 break;
1134 }
1135 }
1136
1137 if (class_has_usable_mbwu(class) &&
1138 topology_matches_l3(class) &&
1139 traffic_matches_l3(class)) {
1140 pr_debug("class %u has usable MBWU, and matches L3 topology and traffic\n",
1141 class->level);
1142
1143 /*
1144 * An MSC measures bandwidth for a path determined by
1145 * its location in hardware. We can't distinguish
1146 * traffic by destination so we don't know if it's
1147 * staying on the same NUMA node. Hence, we can't
1148 * calculate mbm_local except when we only have one L3
1149 * and it's equivalent to mbm_total and so always use
1150 * mbm_total.
1151 */
1152 counter_update_class(QOS_L3_MBM_TOTAL_EVENT_ID, class);
1153 }
1154 }
1155}
1156
1157static void __config_cntr(struct mpam_resctrl_mon *mon, u32 cntr_id,
1158 enum resctrl_conf_type cdp_type, u32 closid, u32 rmid,
1159 bool assign)
1160{
1161 /* Same CDP index remap as closid; maps cntr_id to assigned_counters[]. */
1162 u32 mbwu_idx, mon_idx = resctrl_get_config_index(cntr_id, cdp_type);
1163
1164 closid = resctrl_get_config_index(closid, cdp_type);
1165 mbwu_idx = resctrl_arch_rmid_idx_encode(closid, rmid);
1166
1167 if (assign)
1168 mon->mbwu_idx_to_mon[mbwu_idx] = mon->assigned_counters[mon_idx];
1169 else
1170 mon->mbwu_idx_to_mon[mbwu_idx] = -1;
1171}
1172
1173void resctrl_arch_config_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d,
1174 enum resctrl_event_id evtid, u32 rmid, u32 closid,
1175 u32 cntr_id, bool assign)
1176{
1177 struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[evtid];
1178
1179 if (evtid != QOS_L3_MBM_TOTAL_EVENT_ID) {
1180 pr_debug("unexpected event id\n");
1181 return;
1182 }
1183
1184 if (!mon->mbwu_idx_to_mon || !mon->assigned_counters) {
1185 pr_debug("monitor arrays not allocated\n");
1186 return;
1187 }
1188
1189 if (cdp_enabled) {
1190 __config_cntr(mon, cntr_id, CDP_CODE, closid, rmid, assign);
1191 __config_cntr(mon, cntr_id, CDP_DATA, closid, rmid, assign);
1192 } else {
1193 __config_cntr(mon, cntr_id, CDP_NONE, closid, rmid, assign);
1194 }
1195
1196 resctrl_arch_reset_cntr(r, d, closid, rmid, cntr_id, QOS_L3_MBM_TOTAL_EVENT_ID);
1197}
1198
1199static int mpam_resctrl_control_init(struct mpam_resctrl_res *res)
1200{
1201 struct mpam_class *class = res->class;
1202 struct mpam_props *cprops = &class->props;
1203 struct rdt_resource *r = &res->resctrl_res;
1204
1205 switch (r->rid) {
1206 case RDT_RESOURCE_L2:
1207 case RDT_RESOURCE_L3:
1208 r->schema_fmt = RESCTRL_SCHEMA_BITMAP;
1209 r->cache.arch_has_sparse_bitmasks = true;
1210
1211 r->cache.cbm_len = class->props.cpbm_wd;
1212 /* mpam_devices will reject empty bitmaps */
1213 r->cache.min_cbm_bits = 1;
1214
1215 if (r->rid == RDT_RESOURCE_L2) {
1216 r->name = "L2";
1217 r->ctrl_scope = RESCTRL_L2_CACHE;
1218 r->cdp_capable = true;
1219 } else {
1220 r->name = "L3";
1221 r->ctrl_scope = RESCTRL_L3_CACHE;
1222 r->cdp_capable = true;
1223 }
1224
1225 /*
1226 * Which bits are shared with other ...things... Unknown
1227 * devices use partid-0 which uses all the bitmap fields. Until
1228 * we have configured the SMMU and GIC not to do this 'all the
1229 * bits' is the correct answer here.
1230 */
1231 r->cache.shareable_bits = resctrl_get_default_ctrl(r);
1232 r->alloc_capable = true;
1233 break;
1234 case RDT_RESOURCE_MBA:
1235 r->schema_fmt = RESCTRL_SCHEMA_RANGE;
1236 r->ctrl_scope = RESCTRL_L3_CACHE;
1237
1238 r->membw.delay_linear = true;
1239 r->membw.throttle_mode = THREAD_THROTTLE_UNDEFINED;
1240 r->membw.min_bw = get_mba_min(cprops);
1241 r->membw.max_bw = MAX_MBA_BW;
1242 r->membw.bw_gran = get_mba_granularity(cprops);
1243
1244 r->name = "MB";
1245 r->alloc_capable = true;
1246 break;
1247 default:
1248 return -EINVAL;
1249 }
1250
1251 return 0;
1252}
1253
1254static int mpam_resctrl_pick_domain_id(int cpu, struct mpam_component *comp)
1255{
1256 struct mpam_class *class = comp->class;
1257
1258 if (class->type == MPAM_CLASS_CACHE)
1259 return comp->comp_id;
1260
1261 if (topology_matches_l3(class)) {
1262 /* Use the corresponding L3 component ID as the domain ID */
1263 int id = get_cpu_cacheinfo_id(cpu, 3);
1264
1265 /* Implies topology_matches_l3() made a mistake */
1266 if (WARN_ON_ONCE(id == -1))
1267 return comp->comp_id;
1268
1269 return id;
1270 }
1271
1272 /* Otherwise, expose the ID used by the firmware table code. */
1273 return comp->comp_id;
1274}
1275
1276/*
1277 * This must run after all event counters have been picked so that any free
1278 * running counters have already been allocated.
1279 */
1280static int mpam_resctrl_monitor_init_abmc(struct mpam_resctrl_mon *mon)
1281{
1282 struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3];
1283 size_t num_rmid = resctrl_arch_system_num_rmid_idx();
1284 struct rdt_resource *l3 = &res->resctrl_res;
1285 struct mpam_class *class = mon->class;
1286 u16 num_mbwu_mon;
1287 int *cntrs;
1288
1289 int *rmid_array __free(kvfree) = kvmalloc_objs(*rmid_array, num_rmid);
1290 if (!rmid_array) {
1291 pr_debug("Failed to allocate RMID array\n");
1292 return -ENOMEM;
1293 }
1294 memset(rmid_array, -1, num_rmid * sizeof(*rmid_array));
1295
1296 num_mbwu_mon = class->props.num_mbwu_mon;
1297 cntrs = __alloc_mbwu_array(mon->class, num_mbwu_mon);
1298 if (IS_ERR(cntrs))
1299 return PTR_ERR(cntrs);
1300 mon->assigned_counters = cntrs;
1301 mon->mbwu_idx_to_mon = no_free_ptr(rmid_array);
1302
1303 l3->mon.mbm_cntr_assignable = true;
1304 l3->mon.mbm_assign_on_mkdir = true;
1305 l3->mon.mbm_cntr_configurable = false;
1306 l3->mon.mbm_cntr_assign_fixed = true;
1307
1308 mpam_resctrl_monitor_sync_abmc_vals(l3);
1309
1310 return 0;
1311}
1312
1313static int mpam_resctrl_monitor_init(struct mpam_resctrl_mon *mon,
1314 enum resctrl_event_id type)
1315{
1316 struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3];
1317 struct rdt_resource *l3 = &res->resctrl_res;
1318
1319 lockdep_assert_cpus_held();
1320
1321 /*
1322 * There also needs to be an L3 cache present.
1323 * The check just requires any online CPU and it can't go offline as we
1324 * hold the cpu lock.
1325 */
1326 if (get_cpu_cacheinfo_id(raw_smp_processor_id(), 3) == -1)
1327 return 0;
1328
1329 /*
1330 * If there are no MPAM resources on L3, force it into existence.
1331 * topology_matches_l3() already ensures this looks like the L3.
1332 * The domain-ids will be fixed up by mpam_resctrl_domain_hdr_init().
1333 */
1334 if (!res->class) {
1335 pr_warn_once("Faking L3 MSC to enable counters.\n");
1336 res->class = mpam_resctrl_counters[type].class;
1337 }
1338
1339 /*
1340 * Called multiple times!, once per event type that has a
1341 * monitoring class.
1342 * Setting name is necessary on monitor only platforms.
1343 */
1344 l3->name = "L3";
1345 l3->mon_scope = RESCTRL_L3_CACHE;
1346
1347 /*
1348 * num-rmid is the upper bound for the number of monitoring groups that
1349 * can exist simultaneously, including the default monitoring group for
1350 * each control group. Hence, advertise the whole rmid_idx space even
1351 * though each control group has its own pmg/rmid space. Unfortunately,
1352 * this does mean userspace needs to know the architecture to correctly
1353 * interpret this value.
1354 */
1355 l3->mon.num_rmid = resctrl_arch_system_num_rmid_idx();
1356
1357 if (type == QOS_L3_MBM_TOTAL_EVENT_ID) {
1358 int err;
1359
1360 err = mpam_resctrl_monitor_init_abmc(mon);
1361 if (err)
1362 return err;
1363
1364 static_assert(MAX_EVT_CONFIG_BITS == 0x7f);
1365 l3->mon.mbm_cfg_mask = MAX_EVT_CONFIG_BITS;
1366 }
1367
1368 if (!resctrl_enable_mon_event(type, false, 0, NULL))
1369 return -EINVAL;
1370
1371 l3->mon_capable = true;
1372
1373 return 0;
1374}
1375
1376u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d,
1377 u32 closid, enum resctrl_conf_type type)
1378{
1379 u32 partid;
1380 struct mpam_config *cfg;
1381 struct mpam_props *cprops;
1382 struct mpam_resctrl_res *res;
1383 struct mpam_resctrl_dom *dom;
1384 enum mpam_device_features configured_by;
1385
1386 lockdep_assert_cpus_held();
1387
1388 if (!mpam_is_enabled())
1389 return resctrl_get_default_ctrl(r);
1390
1391 res = container_of(r, struct mpam_resctrl_res, resctrl_res);
1392 dom = container_of(d, struct mpam_resctrl_dom, resctrl_ctrl_dom);
1393 cprops = &res->class->props;
1394
1395 /*
1396 * When CDP is enabled, but the resource doesn't support it,
1397 * the control is cloned across both partids.
1398 * Pick one at random to read:
1399 */
1400 if (mpam_resctrl_hide_cdp(r->rid))
1401 type = CDP_DATA;
1402
1403 partid = resctrl_get_config_index(closid, type);
1404 cfg = &dom->ctrl_comp->cfg[partid];
1405
1406 switch (r->rid) {
1407 case RDT_RESOURCE_L2:
1408 case RDT_RESOURCE_L3:
1409 configured_by = mpam_feat_cpor_part;
1410 break;
1411 case RDT_RESOURCE_MBA:
1412 if (mpam_has_feature(mpam_feat_mbw_max, cprops)) {
1413 configured_by = mpam_feat_mbw_max;
1414 break;
1415 }
1416 fallthrough;
1417 default:
1418 return resctrl_get_default_ctrl(r);
1419 }
1420
1421 if (!r->alloc_capable || partid >= resctrl_arch_get_num_closid(r) ||
1422 !mpam_has_feature(configured_by, cfg))
1423 return resctrl_get_default_ctrl(r);
1424
1425 switch (configured_by) {
1426 case mpam_feat_cpor_part:
1427 return cfg->cpbm;
1428 case mpam_feat_mbw_max:
1429 return mbw_max_to_percent(cfg->mbw_max, cprops);
1430 default:
1431 return resctrl_get_default_ctrl(r);
1432 }
1433}
1434
1435int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d,
1436 u32 closid, enum resctrl_conf_type t, u32 cfg_val)
1437{
1438 int err;
1439 u32 partid;
1440 struct mpam_config cfg;
1441 struct mpam_props *cprops;
1442 struct mpam_resctrl_res *res;
1443 struct mpam_resctrl_dom *dom;
1444
1445 lockdep_assert_cpus_held();
1446 lockdep_assert_irqs_enabled();
1447
1448 if (!mpam_is_enabled())
1449 return -EINVAL;
1450
1451 /*
1452 * No need to check the CPU as mpam_apply_config() doesn't care, and
1453 * resctrl_arch_update_domains() relies on this.
1454 */
1455 res = container_of(r, struct mpam_resctrl_res, resctrl_res);
1456 dom = container_of(d, struct mpam_resctrl_dom, resctrl_ctrl_dom);
1457 cprops = &res->class->props;
1458
1459 if (mpam_resctrl_hide_cdp(r->rid))
1460 t = CDP_DATA;
1461
1462 partid = resctrl_get_config_index(closid, t);
1463 if (!r->alloc_capable || partid >= resctrl_arch_get_num_closid(r)) {
1464 pr_debug("Not alloc capable or computed PARTID out of range\n");
1465 return -EINVAL;
1466 }
1467
1468 /*
1469 * Copy the current config to avoid clearing other resources when the
1470 * same component is exposed multiple times through resctrl.
1471 */
1472 cfg = dom->ctrl_comp->cfg[partid];
1473
1474 switch (r->rid) {
1475 case RDT_RESOURCE_L2:
1476 case RDT_RESOURCE_L3:
1477 cfg.cpbm = cfg_val;
1478 mpam_set_feature(mpam_feat_cpor_part, &cfg);
1479 break;
1480 case RDT_RESOURCE_MBA:
1481 if (mpam_has_feature(mpam_feat_mbw_max, cprops)) {
1482 cfg.mbw_max = percent_to_mbw_max(cfg_val, cprops);
1483 mpam_set_feature(mpam_feat_mbw_max, &cfg);
1484 break;
1485 }
1486 fallthrough;
1487 default:
1488 return -EINVAL;
1489 }
1490
1491 /*
1492 * When CDP is enabled, but the resource doesn't support it, we need to
1493 * apply the same configuration to the other partid.
1494 */
1495 if (mpam_resctrl_hide_cdp(r->rid)) {
1496 partid = resctrl_get_config_index(closid, CDP_CODE);
1497 err = mpam_apply_config(dom->ctrl_comp, partid, &cfg);
1498 if (err)
1499 return err;
1500
1501 partid = resctrl_get_config_index(closid, CDP_DATA);
1502 return mpam_apply_config(dom->ctrl_comp, partid, &cfg);
1503 }
1504
1505 return mpam_apply_config(dom->ctrl_comp, partid, &cfg);
1506}
1507
1508int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid)
1509{
1510 int err;
1511 struct rdt_ctrl_domain *d;
1512
1513 lockdep_assert_cpus_held();
1514 lockdep_assert_irqs_enabled();
1515
1516 if (!mpam_is_enabled())
1517 return -EINVAL;
1518
1519 list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list) {
1520 for (enum resctrl_conf_type t = 0; t < CDP_NUM_TYPES; t++) {
1521 struct resctrl_staged_config *cfg = &d->staged_config[t];
1522
1523 if (!cfg->have_new_ctrl)
1524 continue;
1525
1526 err = resctrl_arch_update_one(r, d, closid, t,
1527 cfg->new_ctrl);
1528 if (err)
1529 return err;
1530 }
1531 }
1532
1533 return 0;
1534}
1535
1536void resctrl_arch_reset_all_ctrls(struct rdt_resource *r)
1537{
1538 struct mpam_resctrl_res *res;
1539
1540 lockdep_assert_cpus_held();
1541
1542 if (!mpam_is_enabled())
1543 return;
1544
1545 res = container_of(r, struct mpam_resctrl_res, resctrl_res);
1546 mpam_reset_class_locked(res->class);
1547}
1548
1549static void mpam_resctrl_domain_hdr_init(int cpu, struct mpam_component *comp,
1550 enum resctrl_res_level rid,
1551 struct rdt_domain_hdr *hdr)
1552{
1553 lockdep_assert_cpus_held();
1554
1555 INIT_LIST_HEAD(&hdr->list);
1556 hdr->id = mpam_resctrl_pick_domain_id(cpu, comp);
1557 hdr->rid = rid;
1558 cpumask_set_cpu(cpu, &hdr->cpu_mask);
1559}
1560
1561static void mpam_resctrl_online_domain_hdr(unsigned int cpu,
1562 struct rdt_domain_hdr *hdr)
1563{
1564 lockdep_assert_cpus_held();
1565
1566 cpumask_set_cpu(cpu, &hdr->cpu_mask);
1567}
1568
1569/**
1570 * mpam_resctrl_offline_domain_hdr() - Update the domain header to remove a CPU.
1571 * @cpu: The CPU to remove from the domain.
1572 * @hdr: The domain's header.
1573 *
1574 * Removes @cpu from the header mask. If this was the last CPU in the domain,
1575 * the domain header is removed from its parent list and true is returned,
1576 * indicating the parent structure can be freed.
1577 * If there are other CPUs in the domain, returns false.
1578 */
1579static bool mpam_resctrl_offline_domain_hdr(unsigned int cpu,
1580 struct rdt_domain_hdr *hdr)
1581{
1582 lockdep_assert_held(&domain_list_lock);
1583
1584 cpumask_clear_cpu(cpu, &hdr->cpu_mask);
1585 if (cpumask_empty(&hdr->cpu_mask)) {
1586 list_del_rcu(&hdr->list);
1587 synchronize_rcu();
1588 return true;
1589 }
1590
1591 return false;
1592}
1593
1594static void mpam_resctrl_domain_insert(struct list_head *list,
1595 struct rdt_domain_hdr *new)
1596{
1597 struct rdt_domain_hdr *err;
1598 struct list_head *pos = NULL;
1599
1600 lockdep_assert_held(&domain_list_lock);
1601
1602 err = resctrl_find_domain(list, new->id, &pos);
1603 if (WARN_ON_ONCE(err))
1604 return;
1605
1606 list_add_tail_rcu(&new->list, pos);
1607}
1608
1609static struct mpam_component *find_component(struct mpam_class *class, int cpu)
1610{
1611 struct mpam_component *comp;
1612
1613 guard(srcu)(&mpam_srcu);
1614 list_for_each_entry_srcu(comp, &class->components, class_list,
1615 srcu_read_lock_held(&mpam_srcu)) {
1616 if (cpumask_test_cpu(cpu, &comp->affinity))
1617 return comp;
1618 }
1619
1620 return NULL;
1621}
1622
1623static struct mpam_resctrl_dom *
1624mpam_resctrl_alloc_domain(unsigned int cpu, struct mpam_resctrl_res *res)
1625{
1626 int err;
1627 struct mpam_resctrl_dom *dom;
1628 struct rdt_l3_mon_domain *mon_d;
1629 struct rdt_ctrl_domain *ctrl_d;
1630 struct mpam_class *class = res->class;
1631 struct mpam_component *comp_iter, *ctrl_comp;
1632 struct rdt_resource *r = &res->resctrl_res;
1633
1634 lockdep_assert_held(&domain_list_lock);
1635
1636 ctrl_comp = NULL;
1637 guard(srcu)(&mpam_srcu);
1638 list_for_each_entry_srcu(comp_iter, &class->components, class_list,
1639 srcu_read_lock_held(&mpam_srcu)) {
1640 if (cpumask_test_cpu(cpu, &comp_iter->affinity)) {
1641 ctrl_comp = comp_iter;
1642 break;
1643 }
1644 }
1645
1646 /* class has no component for this CPU */
1647 if (WARN_ON_ONCE(!ctrl_comp))
1648 return ERR_PTR(-EINVAL);
1649
1650 dom = kzalloc_node(sizeof(*dom), GFP_KERNEL, cpu_to_node(cpu));
1651 if (!dom)
1652 return ERR_PTR(-ENOMEM);
1653
1654 if (r->alloc_capable) {
1655 dom->ctrl_comp = ctrl_comp;
1656
1657 ctrl_d = &dom->resctrl_ctrl_dom;
1658 mpam_resctrl_domain_hdr_init(cpu, ctrl_comp, r->rid, &ctrl_d->hdr);
1659 ctrl_d->hdr.type = RESCTRL_CTRL_DOMAIN;
1660 err = resctrl_online_ctrl_domain(r, ctrl_d);
1661 if (err)
1662 goto free_domain;
1663
1664 mpam_resctrl_domain_insert(&r->ctrl_domains, &ctrl_d->hdr);
1665 } else {
1666 pr_debug("Skipped control domain online - no controls\n");
1667 }
1668
1669 if (r->mon_capable) {
1670 struct mpam_component *any_mon_comp = NULL;
1671 struct mpam_resctrl_mon *mon;
1672 enum resctrl_event_id eventid;
1673
1674 /*
1675 * Even if the monitor domain is backed by a different
1676 * component, the L3 component IDs need to be used... only
1677 * there may be no ctrl_comp for the L3.
1678 * Search each event's class list for a component with
1679 * overlapping CPUs and set up the dom->mon_comp array.
1680 */
1681
1682 for_each_mpam_resctrl_mon(mon, eventid) {
1683 struct mpam_component *mon_comp;
1684
1685 if (!mon->class)
1686 continue; // dummy resource
1687
1688 mon_comp = find_component(mon->class, cpu);
1689 dom->mon_comp[eventid] = mon_comp;
1690 if (mon_comp)
1691 any_mon_comp = mon_comp;
1692 }
1693 if (!any_mon_comp) {
1694 WARN_ON_ONCE(0);
1695 err = -EFAULT;
1696 goto offline_ctrl_domain;
1697 }
1698
1699 mon_d = &dom->resctrl_mon_dom;
1700 mpam_resctrl_domain_hdr_init(cpu, any_mon_comp, r->rid, &mon_d->hdr);
1701 mon_d->hdr.type = RESCTRL_MON_DOMAIN;
1702 err = resctrl_online_mon_domain(r, &mon_d->hdr);
1703 if (err)
1704 goto offline_ctrl_domain;
1705
1706 mpam_resctrl_domain_insert(&r->mon_domains, &mon_d->hdr);
1707 } else {
1708 pr_debug("Skipped monitor domain online - no monitors\n");
1709 }
1710
1711 return dom;
1712
1713offline_ctrl_domain:
1714 if (r->alloc_capable) {
1715 mpam_resctrl_offline_domain_hdr(cpu, &ctrl_d->hdr);
1716 resctrl_offline_ctrl_domain(r, ctrl_d);
1717 }
1718free_domain:
1719 kfree(dom);
1720 dom = ERR_PTR(err);
1721
1722 return dom;
1723}
1724
1725/*
1726 * We know all the monitors are associated with the L3, even if there are no
1727 * controls and therefore no control component. Find the cache-id for the CPU
1728 * and use that to search for existing resctrl domains.
1729 * This relies on mpam_resctrl_pick_domain_id() using the L3 cache-id
1730 * for anything that is not a cache.
1731 */
1732static struct mpam_resctrl_dom *mpam_resctrl_get_mon_domain_from_cpu(int cpu)
1733{
1734 int cache_id;
1735 struct mpam_resctrl_dom *dom;
1736 struct mpam_resctrl_res *l3 = &mpam_resctrl_controls[RDT_RESOURCE_L3];
1737
1738 lockdep_assert_cpus_held();
1739
1740 if (!l3->class)
1741 return NULL;
1742 cache_id = get_cpu_cacheinfo_id(cpu, 3);
1743 if (cache_id < 0)
1744 return NULL;
1745
1746 list_for_each_entry_rcu(dom, &l3->resctrl_res.mon_domains, resctrl_mon_dom.hdr.list) {
1747 if (dom->resctrl_mon_dom.hdr.id == cache_id)
1748 return dom;
1749 }
1750
1751 return NULL;
1752}
1753
1754static struct mpam_resctrl_dom *
1755mpam_resctrl_get_domain_from_cpu(int cpu, struct mpam_resctrl_res *res)
1756{
1757 struct mpam_resctrl_dom *dom;
1758 struct rdt_resource *r = &res->resctrl_res;
1759
1760 lockdep_assert_cpus_held();
1761
1762 list_for_each_entry_rcu(dom, &r->ctrl_domains, resctrl_ctrl_dom.hdr.list) {
1763 if (cpumask_test_cpu(cpu, &dom->ctrl_comp->affinity))
1764 return dom;
1765 }
1766
1767 if (r->rid != RDT_RESOURCE_L3)
1768 return NULL;
1769
1770 /* Search the mon domain list too - needed on monitor only platforms. */
1771 return mpam_resctrl_get_mon_domain_from_cpu(cpu);
1772}
1773
1774int mpam_resctrl_online_cpu(unsigned int cpu)
1775{
1776 struct mpam_resctrl_res *res;
1777 enum resctrl_res_level rid;
1778
1779 guard(mutex)(&domain_list_lock);
1780 for_each_mpam_resctrl_control(res, rid) {
1781 struct mpam_resctrl_dom *dom;
1782 struct rdt_resource *r = &res->resctrl_res;
1783
1784 if (!res->class)
1785 continue; // dummy_resource;
1786
1787 dom = mpam_resctrl_get_domain_from_cpu(cpu, res);
1788 if (!dom) {
1789 dom = mpam_resctrl_alloc_domain(cpu, res);
1790 if (IS_ERR(dom))
1791 return PTR_ERR(dom);
1792 } else {
1793 if (r->alloc_capable) {
1794 struct rdt_ctrl_domain *ctrl_d = &dom->resctrl_ctrl_dom;
1795
1796 mpam_resctrl_online_domain_hdr(cpu, &ctrl_d->hdr);
1797 }
1798 if (r->mon_capable) {
1799 struct rdt_l3_mon_domain *mon_d = &dom->resctrl_mon_dom;
1800
1801 mpam_resctrl_online_domain_hdr(cpu, &mon_d->hdr);
1802 }
1803 }
1804 }
1805
1806 resctrl_online_cpu(cpu);
1807
1808 return 0;
1809}
1810
1811void mpam_resctrl_offline_cpu(unsigned int cpu)
1812{
1813 struct mpam_resctrl_res *res;
1814 enum resctrl_res_level rid;
1815
1816 resctrl_offline_cpu(cpu);
1817
1818 guard(mutex)(&domain_list_lock);
1819 for_each_mpam_resctrl_control(res, rid) {
1820 struct mpam_resctrl_dom *dom;
1821 struct rdt_l3_mon_domain *mon_d;
1822 struct rdt_ctrl_domain *ctrl_d;
1823 bool ctrl_dom_empty, mon_dom_empty;
1824 struct rdt_resource *r = &res->resctrl_res;
1825
1826 if (!res->class)
1827 continue; // dummy resource
1828
1829 dom = mpam_resctrl_get_domain_from_cpu(cpu, res);
1830 if (WARN_ON_ONCE(!dom))
1831 continue;
1832
1833 if (r->alloc_capable) {
1834 ctrl_d = &dom->resctrl_ctrl_dom;
1835 ctrl_dom_empty = mpam_resctrl_offline_domain_hdr(cpu, &ctrl_d->hdr);
1836 if (ctrl_dom_empty)
1837 resctrl_offline_ctrl_domain(&res->resctrl_res, ctrl_d);
1838 } else {
1839 ctrl_dom_empty = true;
1840 }
1841
1842 if (r->mon_capable) {
1843 mon_d = &dom->resctrl_mon_dom;
1844 mon_dom_empty = mpam_resctrl_offline_domain_hdr(cpu, &mon_d->hdr);
1845 if (mon_dom_empty)
1846 resctrl_offline_mon_domain(&res->resctrl_res, &mon_d->hdr);
1847 } else {
1848 mon_dom_empty = true;
1849 }
1850
1851 if (ctrl_dom_empty && mon_dom_empty)
1852 kfree(dom);
1853 }
1854}
1855
1856int mpam_resctrl_setup(void)
1857{
1858 int err = 0;
1859 struct mpam_resctrl_res *res;
1860 enum resctrl_res_level rid;
1861 struct mpam_resctrl_mon *mon;
1862 enum resctrl_event_id eventid;
1863
1864 wait_event(wait_cacheinfo_ready, cacheinfo_ready);
1865
1866 cpus_read_lock();
1867 for_each_mpam_resctrl_control(res, rid) {
1868 INIT_LIST_HEAD_RCU(&res->resctrl_res.ctrl_domains);
1869 INIT_LIST_HEAD_RCU(&res->resctrl_res.mon_domains);
1870 res->resctrl_res.rid = rid;
1871 }
1872
1873 /* Find some classes to use for controls */
1874 mpam_resctrl_pick_caches();
1875 mpam_resctrl_pick_mba();
1876
1877 /* Initialise the resctrl structures from the classes */
1878 for_each_mpam_resctrl_control(res, rid) {
1879 if (!res->class)
1880 continue; // dummy resource
1881
1882 err = mpam_resctrl_control_init(res);
1883 if (err) {
1884 pr_debug("Failed to initialise rid %u\n", rid);
1885 goto internal_error;
1886 }
1887 }
1888
1889 /* Find some classes to use for monitors */
1890 mpam_resctrl_pick_counters();
1891
1892 for_each_mpam_resctrl_mon(mon, eventid) {
1893 if (!mon->class)
1894 continue; // dummy resource
1895
1896 err = mpam_resctrl_monitor_init(mon, eventid);
1897 if (err) {
1898 pr_debug("Failed to initialise event %u\n", eventid);
1899 goto internal_error;
1900 }
1901 }
1902
1903 cpus_read_unlock();
1904
1905 if (!resctrl_arch_alloc_capable() && !resctrl_arch_mon_capable()) {
1906 pr_debug("No alloc(%u) or monitor(%u) found - resctrl not supported\n",
1907 resctrl_arch_alloc_capable(), resctrl_arch_mon_capable());
1908 return -EOPNOTSUPP;
1909 }
1910
1911 err = resctrl_init();
1912 if (err)
1913 return err;
1914
1915 WRITE_ONCE(resctrl_enabled, true);
1916
1917 return 0;
1918
1919internal_error:
1920 cpus_read_unlock();
1921 pr_debug("Internal error %d - resctrl not supported\n", err);
1922 return err;
1923}
1924
1925void mpam_resctrl_exit(void)
1926{
1927 if (!READ_ONCE(resctrl_enabled))
1928 return;
1929
1930 WRITE_ONCE(resctrl_enabled, false);
1931 resctrl_exit();
1932}
1933
1934static void mpam_resctrl_teardown_mon(struct mpam_resctrl_mon *mon, struct mpam_class *class)
1935{
1936 u32 num_mbwu_mon = class->props.num_mbwu_mon;
1937
1938 if (!mon->mbwu_idx_to_mon)
1939 return;
1940
1941 if (mon->assigned_counters) {
1942 __free_mbwu_mon(class, mon->assigned_counters, num_mbwu_mon);
1943 kvfree(mon->assigned_counters);
1944 mon->assigned_counters = NULL;
1945 }
1946
1947 kvfree(mon->mbwu_idx_to_mon);
1948 mon->mbwu_idx_to_mon = NULL;
1949}
1950
1951void mpam_resctrl_teardown_class(struct mpam_class *class)
1952{
1953 struct mpam_resctrl_res *res;
1954 enum resctrl_res_level rid;
1955 struct mpam_resctrl_mon *mon;
1956 enum resctrl_event_id eventid;
1957
1958 might_sleep();
1959
1960 for_each_mpam_resctrl_control(res, rid) {
1961 if (res->class == class)
1962 res->class = NULL;
1963 }
1964 for_each_mpam_resctrl_mon(mon, eventid) {
1965 if (mon->class == class) {
1966 mon->class = NULL;
1967
1968 mpam_resctrl_teardown_mon(mon, class);
1969 }
1970 }
1971}
1972
1973static int __init __cacheinfo_ready(void)
1974{
1975 cacheinfo_ready = true;
1976 wake_up(&wait_cacheinfo_ready);
1977
1978 return 0;
1979}
1980device_initcall_sync(__cacheinfo_ready);
1981
1982#ifdef CONFIG_MPAM_KUNIT_TEST
1983#include "test_mpam_resctrl.c"
1984#endif