linux - drivers/resctrl/mpam

3474 lines · 87.7 KiB

1	// SPDX-License-Identifier: GPL-2.0
2	// Copyright (C) 2025 Arm Ltd.
3
4	#define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__
5
6	#include <linux/acpi.h>
7	#include <linux/atomic.h>
8	#include <linux/arm_mpam.h>
9	#include <linux/bitfield.h>
10	#include <linux/bitmap.h>
11	#include <linux/cacheinfo.h>
12	#include <linux/cpu.h>
13	#include <linux/cpumask.h>
14	#include <linux/device.h>
15	#include <linux/errname.h>
16	#include <linux/errno.h>
17	#include <linux/gfp.h>
18	#include <linux/interrupt.h>
19	#include <linux/irq.h>
20	#include <linux/irqdesc.h>
21	#include <linux/list.h>
22	#include <linux/lockdep.h>
23	#include <linux/mailbox_client.h>
24	#include <linux/mutex.h>
25	#include <linux/platform_device.h>
26	#include <linux/printk.h>
27	#include <linux/property.h>
28	#include <linux/srcu.h>
29	#include <linux/spinlock.h>
30	#include <linux/types.h>
31	#include <linux/workqueue.h>
32
33	#include <acpi/pcc.h>
34	#include <acpi/acpi_io.h>
35
36	#include "mpam_internal.h"
37
38	/* Values for the T241 errata workaround */
39	#define T241_CHIPS_MAX 4
40	#define T241_CHIP_NSLICES 12
41	#define T241_SPARE_REG0_OFF 0x1b0000
42	#define T241_SPARE_REG1_OFF 0x1c0000
43	#define T241_CHIP_ID(phys) FIELD_GET(GENMASK_ULL(44, 43), phys)
44	#define T241_SHADOW_REG_OFF(sidx, pid) (0x360048 + (sidx) * 0x10000 + (pid) * 8)
45	#define SMCCC_SOC_ID_T241 0x036b0241
46	static void __iomem *t241_scratch_regs[T241_CHIPS_MAX];
47
48	/*
49	* mpam_list_lock protects the SRCU lists when writing. Once the
50	* mpam_enabled key is enabled these lists are read-only,
51	* unless the error interrupt disables the driver.
52	*/
53	static DEFINE_MUTEX(mpam_list_lock);
54	static LIST_HEAD(mpam_all_msc);
55
56	struct srcu_struct mpam_srcu;
57
58	/* PCC channels might be serving multiple MSCs, so keep a refcounted list. */
59	static DEFINE_MUTEX(pcc_chan_list_lock);
60	static LIST_HEAD(pcc_chan_list);
61
62	static void mpam_pcc_chan_release(struct kref *ref)
63	{
64	struct mpam_pcc_chan *cur = container_of(ref, struct mpam_pcc_chan,
65	refcount);
66
67	pcc_mbox_free_channel(cur->pcc_chan);
68	list_del(&cur->pcc_chans);
69	mutex_destroy(&cur->pcc_chan_lock);
70	kfree(cur);
71	}
72
73	static struct mpam_pcc_chan mpam_pcc_chan_get(struct device dev,
74	int subspace_id)
75	{
76	struct mpam_pcc_chan *cur;
77
78	guard(mutex)(&pcc_chan_list_lock);
79
80	list_for_each_entry(cur, &pcc_chan_list, pcc_chans) {
81	if (cur->subspace_id == subspace_id) {
82	kref_get(&cur->refcount);
83
84	return cur;
85	}
86	}
87
88	cur = kzalloc_obj(*cur);
89	if (!cur)
90	return ERR_PTR(-ENOMEM);
91
92	cur->pcc_cl.tx_block = true;
93
94	cur->pcc_chan = pcc_mbox_request_channel(&cur->pcc_cl, subspace_id);
95	if (IS_ERR(cur->pcc_chan)) {
96	long err = PTR_ERR(cur->pcc_chan);
97
98	kfree(cur);
99	return ERR_PTR(err);
100	}
101
102	/*
103	* Timeout based on the "nominal latency" in us, from the
104	* PCC ACPI table. tx_tout is in ms.
105	* Add some margin here to be on the safe side.
106	*/
107	cur->pcc_cl.tx_tout = DIV_ROUND_UP(cur->pcc_chan->latency * 5, 1000);
108
109	mutex_init(&cur->pcc_chan_lock);
110
111	cur->subspace_id = subspace_id;
112	kref_init(&cur->refcount);
113
114	list_add_tail(&cur->pcc_chans, &pcc_chan_list);
115
116	return cur;
117	}
118
119	static int mpam_pcc_chan_put(struct mpam_pcc_chan *pcc_chan)
120	{
121	struct mpam_pcc_chan cur, tmp;
122
123	if (!pcc_chan)
124	return 0;
125
126	guard(mutex)(&pcc_chan_list_lock);
127
128	list_for_each_entry_safe(cur, tmp, &pcc_chan_list, pcc_chans) {
129	if (cur == pcc_chan) {
130	kref_put(&cur->refcount, mpam_pcc_chan_release);
131
132	return 0;
133	}
134	}
135
136	return -ENOENT;
137	}
138
139	/*
140	* Number of MSCs that have been probed. Once all MSCs have been probed MPAM
141	* can be enabled.
142	*/
143	static atomic_t mpam_num_msc;
144
145	static int mpam_cpuhp_state;
146	static DEFINE_MUTEX(mpam_cpuhp_state_lock);
147
148	/*
149	* The smallest common values for any CPU or MSC in the system.
150	* Generating traffic outside this range will result in screaming interrupts.
151	*/
152	u16 mpam_partid_max;
153	u8 mpam_pmg_max;
154	static bool partid_max_init, partid_max_published;
155	static DEFINE_SPINLOCK(partid_max_lock);
156
157	/*
158	* mpam is enabled once all devices have been probed from CPU online callbacks,
159	* scheduled via this work_struct. If access to an MSC depends on a CPU that
160	* was not brought online at boot, this can happen surprisingly late.
161	*/
162	static DECLARE_WORK(mpam_enable_work, &mpam_enable);
163
164	/*
165	* All mpam error interrupts indicate a software bug. On receipt, disable the
166	* driver.
167	*/
168	static DECLARE_WORK(mpam_broken_work, &mpam_disable);
169
170	/* When mpam is disabled, the printed reason to aid debugging */
171	static char *mpam_disable_reason;
172	static int mpam_disable_errno;
173	static int mpam_disable_mpam_fb_err;
174
175	void mpam_fb_disable_mpam(int err, int mpam_fb_err)
176	{
177	/* Prevent repeated calls when mpam_disable() does MSC accesses. */
178	if (!mpam_is_enabled())
179	return;
180
181	mpam_disable_errno = err;
182	mpam_disable_mpam_fb_err = mpam_fb_err;
183
184	mpam_disable_reason = "MPAM-Fb error";
185	schedule_work(&mpam_broken_work);
186	}
187
188	/*
189	* Whether resctrl has been setup. Used by cpuhp in preference to
190	* mpam_is_enabled(). The disable call after an error interrupt makes
191	* mpam_is_enabled() false before the cpuhp callbacks are made.
192	* Reads/writes should hold mpam_cpuhp_state_lock, (or be cpuhp callbacks).
193	*/
194	static bool mpam_resctrl_enabled;
195
196	/*
197	* An MSC is a physical container for controls and monitors, each identified by
198	* their RIS index. These share a base-address, interrupts and some MMIO
199	* registers. A vMSC is a virtual container for RIS in an MSC that control or
200	* monitor the same thing. Members of a vMSC are all RIS in the same MSC, but
201	* not all RIS in an MSC share a vMSC.
202	*
203	* Components are a group of vMSC that control or monitor the same thing but
204	* are from different MSC, so have different base-address, interrupts etc.
205	* Classes are the set components of the same type.
206	*
207	* The features of a vMSC is the union of the RIS it contains.
208	* The features of a Class and Component are the common subset of the vMSC
209	* they contain.
210	*
211	* e.g. The system cache may have bandwidth controls on multiple interfaces,
212	* for regulating traffic from devices independently of traffic from CPUs.
213	* If these are two RIS in one MSC, they will be treated as controlling
214	* different things, and will not share a vMSC/component/class.
215	*
216	* e.g. The L2 may have one MSC and two RIS, one for cache-controls another
217	* for bandwidth. These two RIS are members of the same vMSC.
218	*
219	* e.g. The set of RIS that make up the L2 are grouped as a component. These
220	* are sometimes termed slices. They should be configured the same, as if there
221	* were only one.
222	*
223	* e.g. The SoC probably has more than one L2, each attached to a distinct set
224	* of CPUs. All the L2 components are grouped as a class.
225	*
226	* When creating an MSC, struct mpam_msc is added to the all mpam_all_msc list,
227	* then linked via struct mpam_ris to a vmsc, component and class.
228	* The same MSC may exist under different class->component->vmsc paths, but the
229	* RIS index will be unique.
230	*/
231	LIST_HEAD(mpam_classes);
232
233	/* List of all objects that can be free()d after synchronise_srcu() */
234	static LLIST_HEAD(mpam_garbage);
235
236	static inline void init_garbage(struct mpam_garbage *garbage)
237	{
238	init_llist_node(&garbage->llist);
239	}
240
241	#define add_to_garbage(x) \
242	do { \
243	__typeof__(x) _x = (x); \
244	_x->garbage.to_free = _x; \
245	llist_add(&_x->garbage.llist, &mpam_garbage); \
246	} while (0)
247
248	static void mpam_free_garbage(void)
249	{
250	struct mpam_garbage iter, tmp;
251	struct llist_node *to_free = llist_del_all(&mpam_garbage);
252
253	if (!to_free)
254	return;
255
256	synchronize_srcu(&mpam_srcu);
257
258	llist_for_each_entry_safe(iter, tmp, to_free, llist) {
259	if (iter->pdev)
260	devm_kfree(&iter->pdev->dev, iter->to_free);
261	else
262	kvfree(iter->to_free);
263	}
264	}
265
266	/*
267	* Once mpam is enabled, new requestors cannot further reduce the available
268	* partid. Assert that the size is fixed, and new requestors will be turned
269	* away. This is needed when walking over structures sized by PARTID.
270	*
271	* During mpam_disable() these structures are not fixed, but the MSC state
272	* is still reset using whatever sizes have been discovered so far. As only
273	* PARTID 0 will be used after mpam_disable(), any race would be benign.
274	* Skip the check if a mpam_disable_reason has been set.
275	*/
276	static void mpam_assert_partid_sizes_fixed(void)
277	{
278	if (!mpam_disable_reason)
279	WARN_ON_ONCE(!partid_max_published);
280	}
281
282	static int __mpam_read_reg(struct mpam_msc msc, u16 reg, u32 res)
283	{
284
285	if (msc->iface == MPAM_IFACE_PCC)
286	return mpam_fb_send_read_request(msc, reg, res);
287
288	WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility));
289	*res = readl_relaxed(msc->mapped_hwpage + reg);
290
291	return 0;
292	}
293
294	static inline int _mpam_read_partsel_reg(struct mpam_msc *msc, u16 reg,
295	u32 *res)
296	{
297	lockdep_assert_held_once(&msc->part_sel_lock);
298	return __mpam_read_reg(msc, reg, res);
299	}
300
301	#define mpam_read_partsel_reg(msc, reg, res) _mpam_read_partsel_reg(msc, MPAMF_##reg, res)
302
303	static int __mpam_write_reg(struct mpam_msc *msc, u16 reg, u32 val)
304	{
305
306	if (msc->iface == MPAM_IFACE_PCC)
307	return mpam_fb_send_write_request(msc, reg, val);
308
309	WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility));
310	WARN_ON_ONCE(reg + sizeof(u32) > msc->mapped_hwpage_sz);
311	writel_relaxed(val, msc->mapped_hwpage + reg);
312
313	return 0;
314	}
315
316	static inline int _mpam_write_partsel_reg(struct mpam_msc *msc, u16 reg, u32 val)
317	{
318	lockdep_assert_held_once(&msc->part_sel_lock);
319	return __mpam_write_reg(msc, reg, val);
320	}
321
322	#define mpam_write_partsel_reg(msc, reg, val) _mpam_write_partsel_reg(msc, MPAMCFG_##reg, val)
323
324	static inline int _mpam_read_monsel_reg(struct mpam_msc *msc, u16 reg,
325	u32 *res)
326	{
327	mpam_mon_sel_lock_held(msc);
328	return __mpam_read_reg(msc, reg, res);
329	}
330
331	#define mpam_read_monsel_reg(msc, reg, res) _mpam_read_monsel_reg(msc, MSMON_##reg, res)
332
333	static inline int _mpam_write_monsel_reg(struct mpam_msc *msc, u16 reg, u32 val)
334	{
335	mpam_mon_sel_lock_held(msc);
336	return __mpam_write_reg(msc, reg, val);
337	}
338
339	#define mpam_write_monsel_reg(msc, reg, val) _mpam_write_monsel_reg(msc, MSMON_##reg, val)
340
341	static bool mpam_msc_check_aidr(struct mpam_msc *msc)
342	{
343	u32 aidr, major, minor;
344
345	if (__mpam_read_reg(msc, MPAMF_AIDR, &aidr))
346	return false;
347
348	major = FIELD_GET(MPAMF_AIDR_ARCH_MAJOR_REV, aidr);
349	minor = FIELD_GET(MPAMF_AIDR_ARCH_MINOR_REV, aidr);
350	/*
351	* v0.0 and >v2.x aren't supported, but anything else should be backward
352	* compatible to v0.1 or v1.0.
353	*/
354	if (!major && !minor)
355	return false;
356	if (major > 1)
357	return false;
358
359	return true;
360	}
361
362	static int mpam_msc_read_idr(struct mpam_msc msc, u64 res)
363	{
364	u32 idr_high, idr_low;
365	int ret;
366
367	lockdep_assert_held(&msc->part_sel_lock);
368
369	ret = mpam_read_partsel_reg(msc, IDR, &idr_low);
370	if (ret)
371	return ret;
372
373	if (FIELD_GET(MPAMF_IDR_EXT, idr_low)) {
374	ret = mpam_read_partsel_reg(msc, IDR + 4, &idr_high);
375	if (ret)
376	return ret;
377	} else {
378	idr_high = 0;
379	}
380
381	*res = ((u64)idr_high << 32) \| idr_low;
382
383	return 0;
384	}
385
386	static int mpam_msc_clear_esr(struct mpam_msc *msc)
387	{
388	u32 esr_low;
389	int ret;
390
391	ret = __mpam_read_reg(msc, MPAMF_ESR, &esr_low);
392	if (ret)
393	return ret;
394
395	if (!esr_low)
396	return 0;
397
398	/*
399	* Clearing the high/low bits of MPAMF_ESR can not be atomic.
400	* Clear the top half first, so that the pending error bits in the
401	* lower half prevent hardware from updating either half of the
402	* register.
403	*/
404	if (msc->has_extd_esr) {
405	ret = __mpam_write_reg(msc, MPAMF_ESR + 4, 0);
406	if (ret)
407	return ret;
408	}
409
410	return __mpam_write_reg(msc, MPAMF_ESR, 0);
411	}
412
413	static int mpam_msc_read_esr(struct mpam_msc msc, u64 res)
414	{
415	u32 esr_high, esr_low;
416	int ret;
417
418	ret = __mpam_read_reg(msc, MPAMF_ESR, &esr_low);
419	if (ret)
420	return ret;
421
422	if (msc->has_extd_esr) {
423	ret = __mpam_read_reg(msc, MPAMF_ESR + 4, &esr_high);
424	if (ret)
425	return ret;
426	} else {
427	esr_high = 0;
428	}
429
430	*res = ((u64)esr_high << 32) \| esr_low;
431
432	return 0;
433	}
434
435	static int __mpam_part_sel_raw(u32 partsel, struct mpam_msc *msc)
436	{
437	lockdep_assert_held(&msc->part_sel_lock);
438
439	return mpam_write_partsel_reg(msc, PART_SEL, partsel);
440	}
441
442	static int __mpam_part_sel(u8 ris_idx, u16 partid, struct mpam_msc *msc)
443	{
444	u32 partsel = FIELD_PREP(MPAMCFG_PART_SEL_RIS, ris_idx) \|
445	FIELD_PREP(MPAMCFG_PART_SEL_PARTID_SEL, partid);
446
447	return __mpam_part_sel_raw(partsel, msc);
448	}
449
450	static int __mpam_intpart_sel(u8 ris_idx, u16 intpartid, struct mpam_msc *msc)
451	{
452	u32 partsel = FIELD_PREP(MPAMCFG_PART_SEL_RIS, ris_idx) \|
453	FIELD_PREP(MPAMCFG_PART_SEL_PARTID_SEL, intpartid) \|
454	MPAMCFG_PART_SEL_INTERNAL;
455
456	return __mpam_part_sel_raw(partsel, msc);
457	}
458
459	int mpam_register_requestor(u16 partid_max, u8 pmg_max)
460	{
461	guard(spinlock)(&partid_max_lock);
462	if (!partid_max_init) {
463	mpam_partid_max = partid_max;
464	mpam_pmg_max = pmg_max;
465	partid_max_init = true;
466	} else if (!partid_max_published) {
467	mpam_partid_max = min(mpam_partid_max, partid_max);
468	mpam_pmg_max = min(mpam_pmg_max, pmg_max);
469	} else {
470	/* New requestors can't lower the values */
471	if (partid_max < mpam_partid_max \|\| pmg_max < mpam_pmg_max)
472	return -EBUSY;
473	}
474
475	return 0;
476	}
477	EXPORT_SYMBOL(mpam_register_requestor);
478
479	static struct mpam_class *
480	mpam_class_alloc(u8 level_idx, enum mpam_class_types type)
481	{
482	struct mpam_class *class;
483
484	lockdep_assert_held(&mpam_list_lock);
485
486	class = kzalloc_obj(*class);
487	if (!class)
488	return ERR_PTR(-ENOMEM);
489	init_garbage(&class->garbage);
490
491	INIT_LIST_HEAD_RCU(&class->components);
492	/* Affinity is updated when ris are added */
493	class->level = level_idx;
494	class->type = type;
495	INIT_LIST_HEAD_RCU(&class->classes_list);
496	ida_init(&class->ida_csu_mon);
497	ida_init(&class->ida_mbwu_mon);
498
499	list_add_rcu(&class->classes_list, &mpam_classes);
500
501	return class;
502	}
503
504	static void mpam_class_destroy(struct mpam_class *class)
505	{
506	lockdep_assert_held(&mpam_list_lock);
507
508	list_del_rcu(&class->classes_list);
509	add_to_garbage(class);
510	}
511
512	static struct mpam_class *
513	mpam_class_find(u8 level_idx, enum mpam_class_types type)
514	{
515	struct mpam_class *class;
516
517	lockdep_assert_held(&mpam_list_lock);
518
519	list_for_each_entry(class, &mpam_classes, classes_list) {
520	if (class->type == type && class->level == level_idx)
521	return class;
522	}
523
524	return mpam_class_alloc(level_idx, type);
525	}
526
527	static struct mpam_component *
528	mpam_component_alloc(struct mpam_class *class, int id)
529	{
530	struct mpam_component *comp;
531
532	lockdep_assert_held(&mpam_list_lock);
533
534	comp = kzalloc_obj(*comp);
535	if (!comp)
536	return ERR_PTR(-ENOMEM);
537	init_garbage(&comp->garbage);
538
539	comp->comp_id = id;
540	INIT_LIST_HEAD_RCU(&comp->vmsc);
541	/* Affinity is updated when RIS are added */
542	INIT_LIST_HEAD_RCU(&comp->class_list);
543	comp->class = class;
544
545	list_add_rcu(&comp->class_list, &class->components);
546
547	return comp;
548	}
549
550	static void __destroy_component_cfg(struct mpam_component *comp);
551
552	static void mpam_component_destroy(struct mpam_component *comp)
553	{
554	struct mpam_class *class = comp->class;
555
556	lockdep_assert_held(&mpam_list_lock);
557
558	__destroy_component_cfg(comp);
559
560	list_del_rcu(&comp->class_list);
561	add_to_garbage(comp);
562
563	if (list_empty(&class->components))
564	mpam_class_destroy(class);
565	}
566
567	static struct mpam_component *
568	mpam_component_find(struct mpam_class *class, int id)
569	{
570	struct mpam_component *comp;
571
572	lockdep_assert_held(&mpam_list_lock);
573
574	list_for_each_entry(comp, &class->components, class_list) {
575	if (comp->comp_id == id)
576	return comp;
577	}
578
579	return mpam_component_alloc(class, id);
580	}
581
582	static struct mpam_vmsc *
583	mpam_vmsc_alloc(struct mpam_component comp, struct mpam_msc msc)
584	{
585	struct mpam_vmsc *vmsc;
586
587	lockdep_assert_held(&mpam_list_lock);
588
589	vmsc = kzalloc_obj(*vmsc);
590	if (!vmsc)
591	return ERR_PTR(-ENOMEM);
592	init_garbage(&vmsc->garbage);
593
594	INIT_LIST_HEAD_RCU(&vmsc->ris);
595	INIT_LIST_HEAD_RCU(&vmsc->comp_list);
596	vmsc->comp = comp;
597	vmsc->msc = msc;
598
599	list_add_rcu(&vmsc->comp_list, &comp->vmsc);
600
601	return vmsc;
602	}
603
604	static void mpam_vmsc_destroy(struct mpam_vmsc *vmsc)
605	{
606	struct mpam_component *comp = vmsc->comp;
607
608	lockdep_assert_held(&mpam_list_lock);
609
610	list_del_rcu(&vmsc->comp_list);
611	add_to_garbage(vmsc);
612
613	if (list_empty(&comp->vmsc))
614	mpam_component_destroy(comp);
615	}
616
617	static struct mpam_vmsc *
618	mpam_vmsc_find(struct mpam_component comp, struct mpam_msc msc)
619	{
620	struct mpam_vmsc *vmsc;
621
622	lockdep_assert_held(&mpam_list_lock);
623
624	list_for_each_entry(vmsc, &comp->vmsc, comp_list) {
625	if (vmsc->msc->id == msc->id)
626	return vmsc;
627	}
628
629	return mpam_vmsc_alloc(comp, msc);
630	}
631
632	/*
633	* The cacheinfo structures are only populated when CPUs are online.
634	* This helper walks the acpi tables to include offline CPUs too.
635	*/
636	int mpam_get_cpumask_from_cache_id(unsigned long cache_id, u32 cache_level,
637	cpumask_t *affinity)
638	{
639	return acpi_pptt_get_cpumask_from_cache_id(cache_id, affinity);
640	}
641
642	/*
643	* cpumask_of_node() only knows about online CPUs. This can't tell us whether
644	* a class is represented on all possible CPUs.
645	*/
646	static void get_cpumask_from_node_id(u32 node_id, cpumask_t *affinity)
647	{
648	int cpu;
649
650	for_each_possible_cpu(cpu) {
651	if (node_id == cpu_to_node(cpu))
652	cpumask_set_cpu(cpu, affinity);
653	}
654	}
655
656	static int mpam_ris_get_affinity(struct mpam_msc msc, cpumask_t affinity,
657	enum mpam_class_types type,
658	struct mpam_class *class,
659	struct mpam_component *comp)
660	{
661	int err;
662
663	switch (type) {
664	case MPAM_CLASS_CACHE:
665	err = mpam_get_cpumask_from_cache_id(comp->comp_id, class->level,
666	affinity);
667	if (err) {
668	dev_warn_once(&msc->pdev->dev,
669	"Failed to determine CPU affinity\n");
670	return err;
671	}
672
673	if (cpumask_empty(affinity))
674	dev_warn_once(&msc->pdev->dev, "no CPUs associated with cache node\n");
675
676	break;
677	case MPAM_CLASS_MEMORY:
678	get_cpumask_from_node_id(comp->comp_id, affinity);
679	/* affinity may be empty for CPU-less memory nodes */
680	break;
681	case MPAM_CLASS_UNKNOWN:
682	return 0;
683	}
684
685	cpumask_and(affinity, affinity, &msc->accessibility);
686
687	return 0;
688	}
689
690	static int mpam_ris_create_locked(struct mpam_msc *msc, u8 ris_idx,
691	enum mpam_class_types type, u8 class_id,
692	int component_id)
693	{
694	int err;
695	struct mpam_vmsc *vmsc;
696	struct mpam_msc_ris *ris;
697	struct mpam_class *class;
698	struct mpam_component *comp;
699	struct platform_device *pdev = msc->pdev;
700
701	lockdep_assert_held(&mpam_list_lock);
702
703	if (ris_idx > MPAM_MSC_MAX_NUM_RIS)
704	return -EINVAL;
705
706	if (test_and_set_bit(ris_idx, &msc->ris_idxs))
707	return -EBUSY;
708
709	ris = devm_kzalloc(&msc->pdev->dev, sizeof(*ris), GFP_KERNEL);
710	if (!ris)
711	return -ENOMEM;
712	init_garbage(&ris->garbage);
713	ris->garbage.pdev = pdev;
714
715	class = mpam_class_find(class_id, type);
716	if (IS_ERR(class))
717	return PTR_ERR(class);
718
719	comp = mpam_component_find(class, component_id);
720	if (IS_ERR(comp)) {
721	if (list_empty(&class->components))
722	mpam_class_destroy(class);
723	return PTR_ERR(comp);
724	}
725
726	vmsc = mpam_vmsc_find(comp, msc);
727	if (IS_ERR(vmsc)) {
728	if (list_empty(&comp->vmsc))
729	mpam_component_destroy(comp);
730	return PTR_ERR(vmsc);
731	}
732
733	err = mpam_ris_get_affinity(msc, &ris->affinity, type, class, comp);
734	if (err) {
735	if (list_empty(&vmsc->ris))
736	mpam_vmsc_destroy(vmsc);
737	return err;
738	}
739
740	ris->ris_idx = ris_idx;
741	INIT_LIST_HEAD_RCU(&ris->msc_list);
742	INIT_LIST_HEAD_RCU(&ris->vmsc_list);
743	ris->vmsc = vmsc;
744
745	cpumask_or(&comp->affinity, &comp->affinity, &ris->affinity);
746	cpumask_or(&class->affinity, &class->affinity, &ris->affinity);
747	list_add_rcu(&ris->vmsc_list, &vmsc->ris);
748	list_add_rcu(&ris->msc_list, &msc->ris);
749
750	return 0;
751	}
752
753	static void mpam_ris_destroy(struct mpam_msc_ris *ris)
754	{
755	struct mpam_vmsc *vmsc = ris->vmsc;
756	struct mpam_msc *msc = vmsc->msc;
757	struct mpam_component *comp = vmsc->comp;
758	struct mpam_class *class = comp->class;
759
760	lockdep_assert_held(&mpam_list_lock);
761
762	/*
763	* It is assumed affinities don't overlap. If they do the class becomes
764	* unusable immediately.
765	*/
766	cpumask_andnot(&class->affinity, &class->affinity, &ris->affinity);
767	cpumask_andnot(&comp->affinity, &comp->affinity, &ris->affinity);
768	clear_bit(ris->ris_idx, &msc->ris_idxs);
769	list_del_rcu(&ris->msc_list);
770	list_del_rcu(&ris->vmsc_list);
771	add_to_garbage(ris);
772
773	if (list_empty(&vmsc->ris))
774	mpam_vmsc_destroy(vmsc);
775	}
776
777	int mpam_ris_create(struct mpam_msc *msc, u8 ris_idx,
778	enum mpam_class_types type, u8 class_id, int component_id)
779	{
780	int err;
781
782	mutex_lock(&mpam_list_lock);
783	err = mpam_ris_create_locked(msc, ris_idx, type, class_id,
784	component_id);
785	mutex_unlock(&mpam_list_lock);
786	if (err)
787	mpam_free_garbage();
788
789	return err;
790	}
791
792	static struct mpam_msc_ris mpam_get_or_create_ris(struct mpam_msc msc,
793	u8 ris_idx)
794	{
795	int err;
796	struct mpam_msc_ris *ris;
797
798	lockdep_assert_held(&mpam_list_lock);
799
800	if (!test_bit(ris_idx, &msc->ris_idxs)) {
801	err = mpam_ris_create_locked(msc, ris_idx, MPAM_CLASS_UNKNOWN,
802	0, 0);
803	if (err)
804	return ERR_PTR(err);
805	}
806
807	list_for_each_entry(ris, &msc->ris, msc_list) {
808	if (ris->ris_idx == ris_idx)
809	return ris;
810	}
811
812	return ERR_PTR(-ENOENT);
813	}
814
815	static int mpam_enable_quirk_nvidia_t241_1(struct mpam_msc *msc,
816	const struct mpam_quirk *quirk)
817	{
818	s32 soc_id = arm_smccc_get_soc_id_version();
819	struct resource *r;
820	phys_addr_t phys;
821
822	/*
823	* A mapping to a device other than the MSC is needed, check
824	* SOC_ID is NVIDIA T241 chip (036b:0241)
825	*/
826	if (soc_id < 0 \|\| soc_id != SMCCC_SOC_ID_T241)
827	return -EINVAL;
828
829	r = platform_get_resource(msc->pdev, IORESOURCE_MEM, 0);
830	if (!r)
831	return -EINVAL;
832
833	/* Find the internal registers base addr from the CHIP ID */
834	msc->t241_id = T241_CHIP_ID(r->start);
835	phys = FIELD_PREP(GENMASK_ULL(45, 44), msc->t241_id) \| 0x19000000ULL;
836
837	t241_scratch_regs[msc->t241_id] = ioremap(phys, SZ_8M);
838	if (WARN_ON_ONCE(!t241_scratch_regs[msc->t241_id]))
839	return -EINVAL;
840
841	pr_info_once("Enabled workaround for NVIDIA T241 erratum T241-MPAM-1\n");
842
843	return 0;
844	}
845
846	static const struct mpam_quirk mpam_quirks[] = {
847	{
848	/* NVIDIA t241 erratum T241-MPAM-1 */
849	.init = mpam_enable_quirk_nvidia_t241_1,
850	.iidr = MPAM_IIDR_NVIDIA_T241,
851	.iidr_mask = MPAM_IIDR_MATCH_ONE,
852	.workaround = T241_SCRUB_SHADOW_REGS,
853	},
854	{
855	/* NVIDIA t241 erratum T241-MPAM-4 */
856	.iidr = MPAM_IIDR_NVIDIA_T241,
857	.iidr_mask = MPAM_IIDR_MATCH_ONE,
858	.workaround = T241_FORCE_MBW_MIN_TO_ONE,
859	},
860	{
861	/* NVIDIA t241 erratum T241-MPAM-6 */
862	.iidr = MPAM_IIDR_NVIDIA_T241,
863	.iidr_mask = MPAM_IIDR_MATCH_ONE,
864	.workaround = T241_MBW_COUNTER_SCALE_64,
865	},
866	{
867	/* ARM CMN-650 CSU erratum 3642720 */
868	.iidr = MPAM_IIDR_ARM_CMN_650,
869	.iidr_mask = MPAM_IIDR_MATCH_ONE,
870	.workaround = IGNORE_CSU_NRDY,
871	},
872	{ NULL } /* Sentinel */
873	};
874
875	static void mpam_enable_quirks(struct mpam_msc *msc)
876	{
877	const struct mpam_quirk *quirk;
878
879	for (quirk = &mpam_quirks[0]; quirk->iidr_mask; quirk++) {
880	int err = 0;
881
882	if (quirk->iidr != (msc->iidr & quirk->iidr_mask))
883	continue;
884
885	if (quirk->init)
886	err = quirk->init(msc, quirk);
887
888	if (err)
889	continue;
890
891	mpam_set_quirk(quirk->workaround, msc);
892	}
893	}
894
895	/*
896	* IHI009A.a has this nugget: "If a monitor does not support automatic behaviour
897	* of NRDY, software can use this bit for any purpose" - so hardware might not
898	* implement this - but it isn't RES0.
899	*
900	* Try and see what values stick in this bit. If we can write either value,
901	* its probably not implemented by hardware.
902	*/
903	static int mpam_ris_hw_probe_csu_nrdy(struct mpam_msc_ris *ris)
904	{
905	u32 now, mon_sel, ctl_val;
906	bool can_set, can_clear;
907	struct mpam_msc *msc = ris->vmsc->msc;
908	int ret;
909
910	ACQUIRE(mon_sel_lock, guard)(msc);
911	ret = ACQUIRE_ERR(mon_sel_lock, &guard);
912	if (ret)
913	return ret;
914
915	mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, 0) \|
916	FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx);
917	ret = mpam_write_monsel_reg(msc, CFG_MON_SEL, mon_sel);
918	if (ret)
919	return ret;
920
921	/* Hardware might ignore nrdy if it's not enabled */
922	ctl_val = MSMON_CFG_CSU_CTL_TYPE_CSU;
923	ctl_val \|= MSMON_CFG_x_CTL_MATCH_PARTID;
924	ctl_val \|= MSMON_CFG_x_CTL_MATCH_PMG;
925	ctl_val \|= MSMON_CFG_x_CTL_EN;
926	ret = mpam_write_monsel_reg(msc, CFG_CSU_FLT, 0);
927	if (ret)
928	return ret;
929	ret = mpam_write_monsel_reg(msc, CFG_CSU_CTL, ctl_val);
930	if (ret)
931	return ret;
932
933	ret = _mpam_write_monsel_reg(msc, MSMON_CSU, MSMON___NRDY);
934	if (ret)
935	return ret;
936	ret = _mpam_read_monsel_reg(msc, MSMON_CSU, &now);
937	if (ret)
938	return ret;
939	can_set = now & MSMON___NRDY;
940
941	ret = _mpam_write_monsel_reg(msc, MSMON_CSU, 0);
942	if (ret)
943	return ret;
944	/* Configuration change to try and coax hardware into setting nrdy */
945	ret = mpam_write_monsel_reg(msc, CFG_CSU_FLT, 0x1);
946	if (ret)
947	return ret;
948	ret = _mpam_read_monsel_reg(msc, MSMON_CSU, &now);
949	if (ret)
950	return ret;
951	can_clear = !(now & MSMON___NRDY);
952
953	return (!can_set \|\| !can_clear);
954	}
955
956	static int mpam_ris_hw_probe(struct mpam_msc_ris *ris)
957	{
958	int fw_has_nrdy_err, err;
959	struct mpam_msc *msc = ris->vmsc->msc;
960	struct device *dev = &msc->pdev->dev;
961	struct mpam_props *props = &ris->props;
962	struct mpam_class *class = ris->vmsc->comp->class;
963
964	lockdep_assert_held(&msc->probe_lock);
965	lockdep_assert_held(&msc->part_sel_lock);
966
967	/* Cache Capacity Partitioning */
968	if (FIELD_GET(MPAMF_IDR_HAS_CCAP_PART, ris->idr)) {
969	u32 ccap_features;
970
971	err = mpam_read_partsel_reg(msc, CCAP_IDR, &ccap_features);
972	if (err)
973	return err;
974
975	props->cmax_wd = FIELD_GET(MPAMF_CCAP_IDR_CMAX_WD, ccap_features);
976	if (props->cmax_wd &&
977	FIELD_GET(MPAMF_CCAP_IDR_HAS_CMAX_SOFTLIM, ccap_features))
978	mpam_set_feature(mpam_feat_cmax_softlim, props);
979
980	if (props->cmax_wd &&
981	!FIELD_GET(MPAMF_CCAP_IDR_NO_CMAX, ccap_features))
982	mpam_set_feature(mpam_feat_cmax_cmax, props);
983
984	if (props->cmax_wd &&
985	FIELD_GET(MPAMF_CCAP_IDR_HAS_CMIN, ccap_features))
986	mpam_set_feature(mpam_feat_cmax_cmin, props);
987
988	props->cassoc_wd = FIELD_GET(MPAMF_CCAP_IDR_CASSOC_WD, ccap_features);
989	if (props->cassoc_wd &&
990	FIELD_GET(MPAMF_CCAP_IDR_HAS_CASSOC, ccap_features))
991	mpam_set_feature(mpam_feat_cmax_cassoc, props);
992	}
993
994	/* Cache Portion partitioning */
995	if (FIELD_GET(MPAMF_IDR_HAS_CPOR_PART, ris->idr)) {
996	u32 cpor_features;
997
998	err = mpam_read_partsel_reg(msc, CPOR_IDR, &cpor_features);
999	if (err)
1000	return err;
1001
1002	props->cpbm_wd = FIELD_GET(MPAMF_CPOR_IDR_CPBM_WD, cpor_features);
1003	if (props->cpbm_wd)
1004	mpam_set_feature(mpam_feat_cpor_part, props);
1005	}
1006
1007	/* Memory bandwidth partitioning */
1008	if (FIELD_GET(MPAMF_IDR_HAS_MBW_PART, ris->idr)) {
1009	u32 mbw_features;
1010	err = mpam_read_partsel_reg(msc, MBW_IDR, &mbw_features);
1011	if (err)
1012	return err;
1013
1014	/* portion bitmap resolution */
1015	props->mbw_pbm_bits = FIELD_GET(MPAMF_MBW_IDR_BWPBM_WD, mbw_features);
1016	if (props->mbw_pbm_bits &&
1017	FIELD_GET(MPAMF_MBW_IDR_HAS_PBM, mbw_features))
1018	mpam_set_feature(mpam_feat_mbw_part, props);
1019
1020	props->bwa_wd = FIELD_GET(MPAMF_MBW_IDR_BWA_WD, mbw_features);
1021
1022	/*
1023	* The BWA_WD field can represent 0-63, but the control fields it
1024	* describes have a maximum of 16 bits.
1025	*/
1026	props->bwa_wd = min(props->bwa_wd, 16);
1027
1028	if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_MAX, mbw_features))
1029	mpam_set_feature(mpam_feat_mbw_max, props);
1030
1031	if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_MIN, mbw_features))
1032	mpam_set_feature(mpam_feat_mbw_min, props);
1033
1034	if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_PROP, mbw_features))
1035	mpam_set_feature(mpam_feat_mbw_prop, props);
1036	}
1037
1038	/* Priority partitioning */
1039	if (FIELD_GET(MPAMF_IDR_HAS_PRI_PART, ris->idr)) {
1040	u32 pri_features;
1041	err = mpam_read_partsel_reg(msc, PRI_IDR, &pri_features);
1042	if (err)
1043	return err;
1044
1045	props->intpri_wd = FIELD_GET(MPAMF_PRI_IDR_INTPRI_WD, pri_features);
1046	if (props->intpri_wd && FIELD_GET(MPAMF_PRI_IDR_HAS_INTPRI, pri_features)) {
1047	mpam_set_feature(mpam_feat_intpri_part, props);
1048	if (FIELD_GET(MPAMF_PRI_IDR_INTPRI_0_IS_LOW, pri_features))
1049	mpam_set_feature(mpam_feat_intpri_part_0_low, props);
1050	}
1051
1052	props->dspri_wd = FIELD_GET(MPAMF_PRI_IDR_DSPRI_WD, pri_features);
1053	if (props->dspri_wd && FIELD_GET(MPAMF_PRI_IDR_HAS_DSPRI, pri_features)) {
1054	mpam_set_feature(mpam_feat_dspri_part, props);
1055	if (FIELD_GET(MPAMF_PRI_IDR_DSPRI_0_IS_LOW, pri_features))
1056	mpam_set_feature(mpam_feat_dspri_part_0_low, props);
1057	}
1058	}
1059
1060	/* Performance Monitoring */
1061	if (FIELD_GET(MPAMF_IDR_HAS_MSMON, ris->idr)) {
1062	u32 msmon_features;
1063
1064	err = mpam_read_partsel_reg(msc, MSMON_IDR, &msmon_features);
1065	if (err)
1066	return err;
1067
1068	/*
1069	* If the firmware max-nrdy-us property is missing, the
1070	* CSU counters can't be used. Should we wait forever?
1071	*/
1072	fw_has_nrdy_err = device_property_read_u32(&msc->pdev->dev,
1073	"arm,not-ready-us",
1074	&msc->nrdy_usec);
1075
1076	if (FIELD_GET(MPAMF_MSMON_IDR_MSMON_CSU, msmon_features)) {
1077	u32 csumonidr;
1078
1079	err = mpam_read_partsel_reg(msc, CSUMON_IDR, &csumonidr);
1080	if (err)
1081	return err;
1082
1083	props->num_csu_mon = FIELD_GET(MPAMF_CSUMON_IDR_NUM_MON, csumonidr);
1084	if (props->num_csu_mon) {
1085	bool hw_managed;
1086
1087	mpam_set_feature(mpam_feat_msmon_csu, props);
1088
1089	if (FIELD_GET(MPAMF_CSUMON_IDR_HAS_XCL, csumonidr))
1090	mpam_set_feature(mpam_feat_msmon_csu_xcl, props);
1091
1092	/* Is NRDY hardware managed? */
1093	err = mpam_ris_hw_probe_csu_nrdy(ris);
1094	if (err < 0)
1095	return err;
1096	hw_managed = err;
1097
1098	/*
1099	* Accept the missing firmware property if NRDY appears
1100	* un-implemented.
1101	*/
1102	if (fw_has_nrdy_err && hw_managed)
1103	dev_err_once(dev, "Counters are not usable because not-ready timeout was not provided by firmware.");
1104	}
1105	}
1106	if (FIELD_GET(MPAMF_MSMON_IDR_MSMON_MBWU, msmon_features)) {
1107	bool has_long;
1108	u32 mbwumon_idr;
1109
1110	err = mpam_read_partsel_reg(msc, MBWUMON_IDR, &mbwumon_idr);
1111	if (err)
1112	return err;
1113
1114	props->num_mbwu_mon = FIELD_GET(MPAMF_MBWUMON_IDR_NUM_MON, mbwumon_idr);
1115	if (props->num_mbwu_mon) {
1116	mpam_set_feature(mpam_feat_msmon_mbwu, props);
1117
1118	if (FIELD_GET(MPAMF_MBWUMON_IDR_HAS_RWBW, mbwumon_idr))
1119	mpam_set_feature(mpam_feat_msmon_mbwu_rwbw, props);
1120
1121	has_long = FIELD_GET(MPAMF_MBWUMON_IDR_HAS_LONG, mbwumon_idr);
1122	if (has_long) {
1123	if (FIELD_GET(MPAMF_MBWUMON_IDR_LWD, mbwumon_idr))
1124	mpam_set_feature(mpam_feat_msmon_mbwu_63counter, props);
1125	else
1126	mpam_set_feature(mpam_feat_msmon_mbwu_44counter, props);
1127	}
1128
1129	mpam_set_feature(mpam_feat_msmon_mbwu_31counter, props);
1130	}
1131	}
1132	}
1133
1134	/*
1135	* RIS with PARTID narrowing don't have enough storage for one
1136	* configuration per PARTID. If these are in a class we could use,
1137	* reduce the supported partid_max to match the number of intpartid.
1138	* If the class is unknown, just ignore it.
1139	*/
1140	if (FIELD_GET(MPAMF_IDR_HAS_PARTID_NRW, ris->idr) &&
1141	class->type != MPAM_CLASS_UNKNOWN) {
1142	u16 partid_max;
1143	u32 nrwidr;
1144
1145	err = mpam_read_partsel_reg(msc, PARTID_NRW_IDR, &nrwidr);
1146	if (err)
1147	return err;
1148
1149	partid_max = FIELD_GET(MPAMF_PARTID_NRW_IDR_INTPARTID_MAX, nrwidr);
1150
1151	mpam_set_feature(mpam_feat_partid_nrw, props);
1152	msc->partid_max = min(msc->partid_max, partid_max);
1153	}
1154
1155	return 0;
1156	}
1157
1158	static int mpam_msc_hw_probe(struct mpam_msc *msc)
1159	{
1160	int ret;
1161	u64 idr;
1162	u16 partid_max;
1163	u8 ris_idx, pmg_max;
1164	struct mpam_msc_ris *ris;
1165	struct device *dev = &msc->pdev->dev;
1166
1167	lockdep_assert_held(&msc->probe_lock);
1168
1169	if (!mpam_msc_check_aidr(msc)) {
1170	dev_err_once(dev, "MSC does not match architecture v1.x\n");
1171	return -EIO;
1172	}
1173
1174	/* Grab an IDR value to find out how many RIS there are */
1175	scoped_guard(mutex, &msc->part_sel_lock) {
1176	ret = mpam_msc_read_idr(msc, &idr);
1177	if (ret)
1178	return ret;
1179
1180	ret = mpam_read_partsel_reg(msc, IIDR, &msc->iidr);
1181	if (ret)
1182	return ret;
1183	}
1184
1185	mpam_enable_quirks(msc);
1186
1187	msc->ris_max = FIELD_GET(MPAMF_IDR_RIS_MAX, idr);
1188
1189	/* Use these values so partid/pmg always starts with a valid value */
1190	msc->partid_max = FIELD_GET(MPAMF_IDR_PARTID_MAX, idr);
1191	msc->pmg_max = FIELD_GET(MPAMF_IDR_PMG_MAX, idr);
1192
1193	for (ris_idx = 0; ris_idx <= msc->ris_max; ris_idx++) {
1194	scoped_guard(mutex, &msc->part_sel_lock) {
1195	ret = __mpam_part_sel(ris_idx, 0, msc);
1196	if (ret)
1197	return ret;
1198
1199	ret = mpam_msc_read_idr(msc, &idr);
1200	if (ret)
1201	return ret;
1202	}
1203
1204	partid_max = FIELD_GET(MPAMF_IDR_PARTID_MAX, idr);
1205	pmg_max = FIELD_GET(MPAMF_IDR_PMG_MAX, idr);
1206	msc->partid_max = min(msc->partid_max, partid_max);
1207	msc->pmg_max = min(msc->pmg_max, pmg_max);
1208	msc->has_extd_esr = FIELD_GET(MPAMF_IDR_HAS_EXTD_ESR, idr);
1209
1210	scoped_guard(mutex, &mpam_list_lock) {
1211	ris = mpam_get_or_create_ris(msc, ris_idx);
1212	if (IS_ERR(ris))
1213	return PTR_ERR(ris);
1214	}
1215	ris->idr = idr;
1216
1217	scoped_guard(mutex, &msc->part_sel_lock) {
1218	ret = __mpam_part_sel(ris_idx, 0, msc);
1219	if (ret)
1220	return ret;
1221
1222	ret = mpam_ris_hw_probe(ris);
1223	if (ret)
1224	return ret;
1225	}
1226	}
1227
1228	/* Clear any stale errors */
1229	ret = mpam_msc_clear_esr(msc);
1230	if (ret)
1231	return ret;
1232
1233	spin_lock(&partid_max_lock);
1234	mpam_partid_max = min(mpam_partid_max, msc->partid_max);
1235	mpam_pmg_max = min(mpam_pmg_max, msc->pmg_max);
1236	spin_unlock(&partid_max_lock);
1237
1238	msc->probed = true;
1239
1240	return 0;
1241	}
1242
1243	struct mon_read {
1244	struct mpam_msc_ris *ris;
1245	struct mon_cfg *ctx;
1246	enum mpam_device_features type;
1247	u64 *val;
1248	int err;
1249	bool waited_timeout;
1250	};
1251
1252	static int mpam_msc_read_mbwu_l(struct mpam_msc msc, u64 res)
1253	{
1254	int ret;
1255	int retry = 3;
1256	u32 mbwu_l_low;
1257	u32 mbwu_l_high1, mbwu_l_high2;
1258
1259	mpam_mon_sel_lock_held(msc);
1260
1261	if (msc->iface == MPAM_IFACE_MMIO) {
1262	WARN_ON_ONCE((MSMON_MBWU_L + sizeof(u64)) > msc->mapped_hwpage_sz);
1263	WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(),
1264	&msc->accessibility));
1265	}
1266
1267	ret = __mpam_read_reg(msc, MSMON_MBWU_L + 4, &mbwu_l_high2);
1268	if (ret)
1269	return ret;
1270
1271	do {
1272	mbwu_l_high1 = mbwu_l_high2;
1273	ret = __mpam_read_reg(msc, MSMON_MBWU_L, &mbwu_l_low);
1274	if (ret)
1275	return ret;
1276	ret = __mpam_read_reg(msc, MSMON_MBWU_L + 4, &mbwu_l_high2);
1277	if (ret)
1278	return ret;
1279
1280	retry--;
1281	} while (mbwu_l_high1 != mbwu_l_high2 && retry > 0);
1282
1283	if (mbwu_l_high1 == mbwu_l_high2) {
1284	*res = ((u64)mbwu_l_high1 << 32) \| mbwu_l_low;
1285	} else {
1286	pr_warn("Failed to read a stable value\n");
1287	*res = MSMON___L_NRDY;
1288	}
1289
1290	return 0;
1291	}
1292
1293	static int mpam_msc_zero_mbwu_l(struct mpam_msc *msc)
1294	{
1295	int ret;
1296
1297	mpam_mon_sel_lock_held(msc);
1298
1299	if (msc->iface == MPAM_IFACE_MMIO) {
1300	WARN_ON_ONCE((MSMON_MBWU_L + sizeof(u64)) > msc->mapped_hwpage_sz);
1301	WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(),
1302	&msc->accessibility));
1303	}
1304
1305	ret = __mpam_write_reg(msc, MSMON_MBWU_L, 0);
1306	if (ret)
1307	return ret;
1308
1309	return __mpam_write_reg(msc, MSMON_MBWU_L + 4, 0);
1310	}
1311
1312	static void gen_msmon_ctl_flt_vals(struct mon_read m, u32 ctl_val,
1313	u32 *flt_val)
1314	{
1315	struct mon_cfg *ctx = m->ctx;
1316
1317	/*
1318	* For CSU counters its implementation-defined what happens when not
1319	* filtering by partid.
1320	*/
1321	*ctl_val = MSMON_CFG_x_CTL_MATCH_PARTID;
1322
1323	*flt_val = FIELD_PREP(MSMON_CFG_x_FLT_PARTID, ctx->partid);
1324
1325	if (m->ctx->match_pmg) {
1326	*ctl_val \|= MSMON_CFG_x_CTL_MATCH_PMG;
1327	*flt_val \|= FIELD_PREP(MSMON_CFG_x_FLT_PMG, ctx->pmg);
1328	}
1329
1330	switch (m->type) {
1331	case mpam_feat_msmon_csu:
1332	*ctl_val \|= MSMON_CFG_CSU_CTL_TYPE_CSU;
1333
1334	if (mpam_has_feature(mpam_feat_msmon_csu_xcl, &m->ris->props))
1335	*flt_val \|= FIELD_PREP(MSMON_CFG_CSU_FLT_XCL, ctx->csu_exclude_clean);
1336
1337	break;
1338	case mpam_feat_msmon_mbwu_31counter:
1339	case mpam_feat_msmon_mbwu_44counter:
1340	case mpam_feat_msmon_mbwu_63counter:
1341	*ctl_val \|= MSMON_CFG_MBWU_CTL_TYPE_MBWU;
1342
1343	if (mpam_has_feature(mpam_feat_msmon_mbwu_rwbw, &m->ris->props))
1344	*flt_val \|= FIELD_PREP(MSMON_CFG_MBWU_FLT_RWBW, ctx->opts);
1345
1346	break;
1347	default:
1348	pr_warn("Unexpected monitor type %d\n", m->type);
1349	}
1350	}
1351
1352	static int read_msmon_ctl_flt_vals(struct mon_read m, u32 ctl_val,
1353	u32 *flt_val)
1354	{
1355	struct mpam_msc *msc = m->ris->vmsc->msc;
1356	int ret;
1357
1358	switch (m->type) {
1359	case mpam_feat_msmon_csu:
1360	ret = mpam_read_monsel_reg(msc, CFG_CSU_CTL, ctl_val);
1361	if (ret)
1362	return ret;
1363	return mpam_read_monsel_reg(msc, CFG_CSU_FLT, flt_val);
1364	case mpam_feat_msmon_mbwu_31counter:
1365	case mpam_feat_msmon_mbwu_44counter:
1366	case mpam_feat_msmon_mbwu_63counter:
1367	ret = mpam_read_monsel_reg(msc, CFG_MBWU_CTL, ctl_val);
1368	if (ret)
1369	return ret;
1370	return mpam_read_monsel_reg(msc, CFG_MBWU_FLT, flt_val);
1371	default:
1372	pr_warn("Unexpected monitor type %d\n", m->type);
1373	return -EINVAL;
1374	}
1375	}
1376
1377	/* Remove values set by the hardware to prevent apparent mismatches. */
1378	static inline void clean_msmon_ctl_val(u32 *cur_ctl)
1379	{
1380	*cur_ctl &= ~MSMON_CFG_x_CTL_OFLOW_STATUS;
1381
1382	if (FIELD_GET(MSMON_CFG_x_CTL_TYPE, *cur_ctl) == MSMON_CFG_MBWU_CTL_TYPE_MBWU)
1383	*cur_ctl &= ~MSMON_CFG_MBWU_CTL_OFLOW_STATUS_L;
1384	}
1385
1386	static int write_msmon_ctl_flt_vals(struct mon_read *m, u32 ctl_val,
1387	u32 flt_val)
1388	{
1389	struct mpam_msc *msc = m->ris->vmsc->msc;
1390	int ret;
1391
1392	/*
1393	* Write the ctl_val with the enable bit cleared, reset the counter,
1394	* then enable counter.
1395	*/
1396	switch (m->type) {
1397	case mpam_feat_msmon_csu:
1398	ret = mpam_write_monsel_reg(msc, CFG_CSU_FLT, flt_val);
1399	if (ret)
1400	return ret;
1401	ret = mpam_write_monsel_reg(msc, CFG_CSU_CTL, ctl_val);
1402	if (ret)
1403	return ret;
1404	ret = mpam_write_monsel_reg(msc, CSU, 0);
1405	if (ret)
1406	return ret;
1407	return mpam_write_monsel_reg(msc, CFG_CSU_CTL, ctl_val \| MSMON_CFG_x_CTL_EN);
1408	case mpam_feat_msmon_mbwu_31counter:
1409	case mpam_feat_msmon_mbwu_44counter:
1410	case mpam_feat_msmon_mbwu_63counter:
1411	ret = mpam_write_monsel_reg(msc, CFG_MBWU_FLT, flt_val);
1412	if (ret)
1413	return ret;
1414	ret = mpam_write_monsel_reg(msc, CFG_MBWU_CTL, ctl_val);
1415	if (ret)
1416	return ret;
1417	ret = mpam_write_monsel_reg(msc, CFG_MBWU_CTL,
1418	ctl_val \| MSMON_CFG_x_CTL_EN);
1419	if (ret)
1420	return ret;
1421	/* Counting monitors require NRDY to be reset by software */
1422	if (m->type == mpam_feat_msmon_mbwu_31counter)
1423	return mpam_write_monsel_reg(msc, MBWU, 0);
1424
1425	return mpam_msc_zero_mbwu_l(m->ris->vmsc->msc);
1426	default:
1427	pr_warn("Unexpected monitor type %d\n", m->type);
1428	return -EINVAL;
1429	}
1430	}
1431
1432	static u64 __mpam_msmon_overflow_val(enum mpam_device_features type)
1433	{
1434	/* TODO: implement scaling counters */
1435	switch (type) {
1436	case mpam_feat_msmon_mbwu_63counter:
1437	return BIT_ULL(hweight_long(MSMON___LWD_VALUE));
1438	case mpam_feat_msmon_mbwu_44counter:
1439	return BIT_ULL(hweight_long(MSMON___L_VALUE));
1440	case mpam_feat_msmon_mbwu_31counter:
1441	return BIT_ULL(hweight_long(MSMON___VALUE));
1442	default:
1443	return 0;
1444	}
1445	}
1446
1447	static u64 mpam_msmon_overflow_val(enum mpam_device_features type,
1448	struct mpam_msc *msc)
1449	{
1450	u64 overflow_val = __mpam_msmon_overflow_val(type);
1451
1452	if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc))
1453	overflow_val *= 64;
1454
1455	return overflow_val;
1456	}
1457
1458	/*
1459	* This function might be called via smp_call_function_any(), so propagate
1460	* errors inside the arg struct.
1461	*/
1462	static void __ris_msmon_read_locked(struct mon_read *m)
1463	{
1464	u64 now;
1465	u32 now32;
1466	bool nrdy = false;
1467	bool config_mismatch;
1468	bool overflow = false;
1469	struct mon_cfg *ctx = m->ctx;
1470	bool reset_on_next_read = false;
1471	struct mpam_msc_ris *ris = m->ris;
1472	struct msmon_mbwu_state *mbwu_state;
1473	struct mpam_msc *msc = m->ris->vmsc->msc;
1474	u32 mon_sel, ctl_val, flt_val, cur_ctl, cur_flt;
1475
1476	mpam_mon_sel_lock_held(msc);
1477
1478	mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, ctx->mon) \|
1479	FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx);
1480	m->err = mpam_write_monsel_reg(msc, CFG_MON_SEL, mon_sel);
1481	if (m->err)
1482	return;
1483
1484	switch (m->type) {
1485	case mpam_feat_msmon_mbwu_31counter:
1486	case mpam_feat_msmon_mbwu_44counter:
1487	case mpam_feat_msmon_mbwu_63counter:
1488	mbwu_state = &ris->mbwu_state[ctx->mon];
1489	if (mbwu_state) {
1490	reset_on_next_read = mbwu_state->reset_on_next_read;
1491	mbwu_state->reset_on_next_read = false;
1492	}
1493	break;
1494	default:
1495	break;
1496	}
1497
1498	/*
1499	* Read the existing configuration to avoid re-writing the same values.
1500	* This saves waiting for 'nrdy' on subsequent reads.
1501	*/
1502	m->err = read_msmon_ctl_flt_vals(m, &cur_ctl, &cur_flt);
1503	if (m->err)
1504	return;
1505
1506	if (mpam_feat_msmon_mbwu_31counter == m->type)
1507	overflow = cur_ctl & MSMON_CFG_x_CTL_OFLOW_STATUS;
1508	else if (mpam_feat_msmon_mbwu_44counter == m->type \|\|
1509	mpam_feat_msmon_mbwu_63counter == m->type)
1510	overflow = cur_ctl & MSMON_CFG_MBWU_CTL_OFLOW_STATUS_L;
1511
1512	clean_msmon_ctl_val(&cur_ctl);
1513	gen_msmon_ctl_flt_vals(m, &ctl_val, &flt_val);
1514	config_mismatch = cur_flt != flt_val \|\|
1515	cur_ctl != (ctl_val \| MSMON_CFG_x_CTL_EN);
1516
1517	if (config_mismatch \|\| reset_on_next_read) {
1518	m->err = write_msmon_ctl_flt_vals(m, ctl_val, flt_val);
1519	if (m->err)
1520	return;
1521	overflow = false;
1522	} else if (overflow) {
1523	m->err = mpam_write_monsel_reg(msc, CFG_MBWU_CTL,
1524	cur_ctl &
1525	~(MSMON_CFG_x_CTL_OFLOW_STATUS \|
1526	MSMON_CFG_MBWU_CTL_OFLOW_STATUS_L));
1527	if (m->err)
1528	return;
1529	}
1530
1531	switch (m->type) {
1532	case mpam_feat_msmon_csu:
1533	m->err = mpam_read_monsel_reg(msc, CSU, &now32);
1534	if (m->err)
1535	return;
1536	nrdy = now32 & MSMON___NRDY;
1537	now = FIELD_GET(MSMON___VALUE, now32);
1538
1539	if (mpam_has_quirk(IGNORE_CSU_NRDY, msc) && m->waited_timeout)
1540	nrdy = false;
1541
1542	break;
1543	case mpam_feat_msmon_mbwu_31counter:
1544	case mpam_feat_msmon_mbwu_44counter:
1545	case mpam_feat_msmon_mbwu_63counter:
1546	if (m->type != mpam_feat_msmon_mbwu_31counter) {
1547	m->err = mpam_msc_read_mbwu_l(msc, &now);
1548	if (m->err)
1549	return;
1550	nrdy = now & MSMON___L_NRDY;
1551
1552	if (m->type == mpam_feat_msmon_mbwu_63counter)
1553	now = FIELD_GET(MSMON___LWD_VALUE, now);
1554	else
1555	now = FIELD_GET(MSMON___L_VALUE, now);
1556	} else {
1557	m->err = mpam_read_monsel_reg(msc, MBWU, &now32);
1558	if (m->err)
1559	return;
1560	nrdy = now32 & MSMON___NRDY;
1561	now = FIELD_GET(MSMON___VALUE, now32);
1562	}
1563
1564	if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc))
1565	now *= 64;
1566
1567	if (nrdy)
1568	break;
1569
1570	mbwu_state = &ris->mbwu_state[ctx->mon];
1571
1572	if (overflow)
1573	mbwu_state->correction += mpam_msmon_overflow_val(m->type, msc);
1574
1575	/*
1576	* Include bandwidth consumed before the last hardware reset and
1577	* a counter size increment for each overflow.
1578	*/
1579	now += mbwu_state->correction;
1580	break;
1581	default:
1582	m->err = -EINVAL;
1583	}
1584
1585	if (nrdy)
1586	m->err = -EBUSY;
1587
1588	if (m->err)
1589	return;
1590
1591	*m->val += now;
1592	}
1593
1594	static void __ris_msmon_read(void *arg)
1595	{
1596	struct mon_read *m = arg;
1597	struct mpam_msc *msc = m->ris->vmsc->msc;
1598
1599	if (!mpam_mon_sel_lock(msc)) {
1600	m->err = -EIO;
1601	return;
1602	}
1603
1604	__ris_msmon_read_locked(m);
1605
1606	mpam_mon_sel_unlock(msc);
1607	}
1608
1609	static int _msmon_read(struct mpam_component comp, struct mon_read arg)
1610	{
1611	int err, any_err = 0;
1612	struct mpam_vmsc *vmsc;
1613
1614	guard(srcu)(&mpam_srcu);
1615	list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list,
1616	srcu_read_lock_held(&mpam_srcu)) {
1617	struct mpam_msc *msc = vmsc->msc;
1618	struct mpam_msc_ris *ris;
1619
1620	list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list,
1621	srcu_read_lock_held(&mpam_srcu)) {
1622	arg->ris = ris;
1623
1624	if (msc->iface == MPAM_IFACE_MMIO) {
1625	err = smp_call_function_any(&msc->accessibility,
1626	__ris_msmon_read,
1627	arg, true);
1628	if (!err)
1629	err = arg->err;
1630	} else {
1631	__ris_msmon_read(arg);
1632	err = arg->err;
1633	}
1634
1635	/*
1636	* Save one error to be returned to the caller, but
1637	* keep reading counters so that get reprogrammed. On
1638	* platforms with NRDY this lets us wait once.
1639	*/
1640	if (err)
1641	any_err = err;
1642	}
1643	}
1644
1645	return any_err;
1646	}
1647
1648	static enum mpam_device_features mpam_msmon_choose_counter(struct mpam_class *class)
1649	{
1650	struct mpam_props *cprops = &class->props;
1651
1652	if (mpam_has_feature(mpam_feat_msmon_mbwu_63counter, cprops))
1653	return mpam_feat_msmon_mbwu_63counter;
1654	if (mpam_has_feature(mpam_feat_msmon_mbwu_44counter, cprops))
1655	return mpam_feat_msmon_mbwu_44counter;
1656
1657	return mpam_feat_msmon_mbwu_31counter;
1658	}
1659
1660	int mpam_msmon_read(struct mpam_component comp, struct mon_cfg ctx,
1661	enum mpam_device_features type, u64 *val)
1662	{
1663	int err;
1664	struct mon_read arg;
1665	u64 wait_jiffies = 0;
1666	struct mpam_class *class = comp->class;
1667	struct mpam_props *cprops = &class->props;
1668
1669	might_sleep();
1670
1671	if (!mpam_is_enabled())
1672	return -EIO;
1673
1674	if (!mpam_has_feature(type, cprops))
1675	return -EOPNOTSUPP;
1676
1677	if (type == mpam_feat_msmon_mbwu)
1678	type = mpam_msmon_choose_counter(class);
1679
1680	arg = (struct mon_read) {
1681	.ctx = ctx,
1682	.type = type,
1683	.val = val,
1684	};
1685	*val = 0;
1686
1687	err = _msmon_read(comp, &arg);
1688	if (err == -EBUSY && class->nrdy_usec)
1689	wait_jiffies = usecs_to_jiffies(class->nrdy_usec);
1690
1691	while (wait_jiffies)
1692	wait_jiffies = schedule_timeout_uninterruptible(wait_jiffies);
1693
1694	if (err == -EBUSY) {
1695	arg = (struct mon_read) {
1696	.ctx = ctx,
1697	.type = type,
1698	.val = val,
1699	.waited_timeout = true,
1700	};
1701	*val = 0;
1702
1703	err = _msmon_read(comp, &arg);
1704	}
1705
1706	return err;
1707	}
1708
1709	void mpam_msmon_reset_mbwu(struct mpam_component comp, struct mon_cfg ctx)
1710	{
1711	struct mpam_msc *msc;
1712	struct mpam_vmsc *vmsc;
1713	struct mpam_msc_ris *ris;
1714
1715	if (!mpam_is_enabled())
1716	return;
1717
1718	guard(srcu)(&mpam_srcu);
1719	list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list,
1720	srcu_read_lock_held(&mpam_srcu)) {
1721	if (!mpam_has_feature(mpam_feat_msmon_mbwu, &vmsc->props))
1722	continue;
1723
1724	msc = vmsc->msc;
1725	list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list,
1726	srcu_read_lock_held(&mpam_srcu)) {
1727	if (!mpam_has_feature(mpam_feat_msmon_mbwu, &ris->props))
1728	continue;
1729
1730	if (WARN_ON_ONCE(!mpam_mon_sel_lock(msc)))
1731	continue;
1732
1733	ris->mbwu_state[ctx->mon].correction = 0;
1734	ris->mbwu_state[ctx->mon].reset_on_next_read = true;
1735	mpam_mon_sel_unlock(msc);
1736	}
1737	}
1738	}
1739
1740	static int mpam_reset_msc_bitmap(struct mpam_msc *msc, u16 reg, u16 wd)
1741	{
1742	u32 num_words, msb;
1743	u32 bm = ~0;
1744	int i;
1745
1746	lockdep_assert_held(&msc->part_sel_lock);
1747
1748	if (wd == 0)
1749	return 0;
1750
1751	/*
1752	* Write all ~0 to all but the last 32bit-word, which may
1753	* have fewer bits...
1754	*/
1755	num_words = DIV_ROUND_UP(wd, 32);
1756	for (i = 0; i < num_words - 1; i++, reg += sizeof(bm)) {
1757	int ret;
1758
1759	ret = __mpam_write_reg(msc, reg, bm);
1760	if (ret)
1761	return ret;
1762	}
1763
1764	/*
1765	* ....and then the last (maybe) partial 32bit word. When wd is a
1766	* multiple of 32, msb should be 31 to write a full 32bit word.
1767	*/
1768	msb = (wd - 1) % 32;
1769	bm = GENMASK(msb, 0);
1770	return __mpam_write_reg(msc, reg, bm);
1771	}
1772
1773	static void mpam_apply_t241_erratum(struct mpam_msc_ris *ris, u16 partid)
1774	{
1775	int sidx, i, lcount = 1000;
1776	void __iomem *regs;
1777	u64 val0, val;
1778
1779	regs = t241_scratch_regs[ris->vmsc->msc->t241_id];
1780
1781	for (i = 0; i < lcount; i++) {
1782	/* Read the shadow register at index 0 */
1783	val0 = readq_relaxed(regs + T241_SHADOW_REG_OFF(0, partid));
1784
1785	/* Check if all the shadow registers have the same value */
1786	for (sidx = 1; sidx < T241_CHIP_NSLICES; sidx++) {
1787	val = readq_relaxed(regs +
1788	T241_SHADOW_REG_OFF(sidx, partid));
1789	if (val != val0)
1790	break;
1791	}
1792	if (sidx == T241_CHIP_NSLICES)
1793	break;
1794	}
1795
1796	if (i == lcount)
1797	pr_warn_once("t241: inconsistent values in shadow regs");
1798
1799	/* Write a value zero to spare registers to take effect of MBW conf */
1800	writeq_relaxed(0, regs + T241_SPARE_REG0_OFF);
1801	writeq_relaxed(0, regs + T241_SPARE_REG1_OFF);
1802	}
1803
1804	static void mpam_quirk_post_config_change(struct mpam_msc_ris *ris, u16 partid,
1805	struct mpam_config *cfg)
1806	{
1807	if (mpam_has_quirk(T241_SCRUB_SHADOW_REGS, ris->vmsc->msc))
1808	mpam_apply_t241_erratum(ris, partid);
1809	}
1810
1811	static u16 mpam_wa_t241_force_mbw_min_to_one(struct mpam_props *props)
1812	{
1813	u16 max_hw_value, min_hw_granule, res0_bits;
1814
1815	res0_bits = 16 - props->bwa_wd;
1816	max_hw_value = ((1 << props->bwa_wd) - 1) << res0_bits;
1817	min_hw_granule = ~max_hw_value;
1818
1819	return min_hw_granule + 1;
1820	}
1821
1822	static u16 mpam_wa_t241_calc_min_from_max(struct mpam_props *props,
1823	struct mpam_config *cfg)
1824	{
1825	u16 val = 0;
1826	u16 max;
1827	u16 delta = ((5 * MPAMCFG_MBW_MAX_MAX) / 100) - 1;
1828
1829	if (mpam_has_feature(mpam_feat_mbw_max, cfg)) {
1830	max = cfg->mbw_max;
1831	} else {
1832	/* Resetting. Hence, use the ris specific default. */
1833	max = GENMASK(15, 16 - props->bwa_wd);
1834	}
1835
1836	if (max > delta)
1837	val = max - delta;
1838
1839	return val;
1840	}
1841
1842	/* Called via IPI. Call while holding an SRCU reference */
1843	static int mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid,
1844	struct mpam_config *cfg)
1845	{
1846	u16 cmax = MPAMCFG_CMAX_CMAX;
1847	struct mpam_msc *msc = ris->vmsc->msc;
1848	struct mpam_props *rprops = &ris->props;
1849	int ret;
1850
1851	guard(mutex)(&msc->part_sel_lock);
1852	ret = __mpam_part_sel(ris->ris_idx, partid, msc);
1853	if (ret)
1854	return ret;
1855
1856	if (mpam_has_feature(mpam_feat_partid_nrw, rprops)) {
1857	/* Update the intpartid mapping */
1858	ret = mpam_write_partsel_reg(msc, INTPARTID,
1859	MPAMCFG_INTPARTID_INTERNAL \| partid);
1860	if (ret)
1861	return ret;
1862
1863	/*
1864	* Then switch to the 'internal' partid to update the
1865	* configuration.
1866	*/
1867	ret = __mpam_intpart_sel(ris->ris_idx, partid, msc);
1868	if (ret)
1869	return ret;
1870	}
1871
1872	if (mpam_has_feature(mpam_feat_cpor_part, rprops)) {
1873	if (mpam_has_feature(mpam_feat_cpor_part, cfg)) {
1874	ret = mpam_write_partsel_reg(msc, CPBM, cfg->cpbm);
1875	if (ret)
1876	return ret;
1877	} else {
1878	ret = mpam_reset_msc_bitmap(msc, MPAMCFG_CPBM,
1879	rprops->cpbm_wd);
1880	if (ret)
1881	return ret;
1882	}
1883	}
1884
1885	if (mpam_has_feature(mpam_feat_mbw_part, rprops)) {
1886	if (mpam_has_feature(mpam_feat_mbw_part, cfg)) {
1887	ret = mpam_write_partsel_reg(msc, MBW_PBM, cfg->mbw_pbm);
1888	if (ret)
1889	return ret;
1890	} else {
1891	ret = mpam_reset_msc_bitmap(msc, MPAMCFG_MBW_PBM,
1892	rprops->mbw_pbm_bits);
1893	if (ret)
1894	return ret;
1895	}
1896	}
1897
1898	if (mpam_has_feature(mpam_feat_mbw_min, rprops)) {
1899	u16 val = 0;
1900
1901	if (mpam_has_quirk(T241_FORCE_MBW_MIN_TO_ONE, msc)) {
1902	u16 min = mpam_wa_t241_force_mbw_min_to_one(rprops);
1903
1904	val = mpam_wa_t241_calc_min_from_max(rprops, cfg);
1905	val = max(val, min);
1906	}
1907
1908	ret = mpam_write_partsel_reg(msc, MBW_MIN, val);
1909	if (ret)
1910	return ret;
1911	}
1912
1913	if (mpam_has_feature(mpam_feat_mbw_max, rprops)) {
1914	if (mpam_has_feature(mpam_feat_mbw_max, cfg)) {
1915	ret = mpam_write_partsel_reg(msc, MBW_MAX, cfg->mbw_max);
1916	if (ret)
1917	return ret;
1918	} else {
1919	ret = mpam_write_partsel_reg(msc, MBW_MAX,
1920	MPAMCFG_MBW_MAX_MAX);
1921	if (ret)
1922	return ret;
1923	}
1924	}
1925
1926	if (mpam_has_feature(mpam_feat_mbw_prop, rprops)) {
1927	ret = mpam_write_partsel_reg(msc, MBW_PROP, 0);
1928	if (ret)
1929	return ret;
1930	}
1931
1932	if (mpam_has_feature(mpam_feat_cmax_cmax, rprops)) {
1933	ret = mpam_write_partsel_reg(msc, CMAX, cmax);
1934	if (ret)
1935	return ret;
1936	}
1937
1938	if (mpam_has_feature(mpam_feat_cmax_cmin, rprops)) {
1939	ret = mpam_write_partsel_reg(msc, CMIN, 0);
1940	if (ret)
1941	return ret;
1942	}
1943
1944	if (mpam_has_feature(mpam_feat_cmax_cassoc, rprops)) {
1945	ret = mpam_write_partsel_reg(msc, CASSOC, MPAMCFG_CASSOC_CASSOC);
1946	if (ret)
1947	return ret;
1948	}
1949
1950	if (mpam_has_feature(mpam_feat_intpri_part, rprops) \|\|
1951	mpam_has_feature(mpam_feat_dspri_part, rprops)) {
1952	u32 pri_val = 0;
1953
1954	if (mpam_has_feature(mpam_feat_intpri_part, rprops)) {
1955	u16 intpri = GENMASK(rprops->intpri_wd - 1, 0);
1956
1957	/* aces high? */
1958	if (!mpam_has_feature(mpam_feat_intpri_part_0_low, rprops))
1959	intpri = 0;
1960
1961	pri_val \|= FIELD_PREP(MPAMCFG_PRI_INTPRI, intpri);
1962	}
1963	if (mpam_has_feature(mpam_feat_dspri_part, rprops)) {
1964	u16 dspri = GENMASK(rprops->dspri_wd - 1, 0);
1965
1966	if (!mpam_has_feature(mpam_feat_dspri_part_0_low, rprops))
1967	dspri = 0;
1968
1969	pri_val \|= FIELD_PREP(MPAMCFG_PRI_DSPRI, dspri);
1970	}
1971
1972	ret = mpam_write_partsel_reg(msc, PRI, pri_val);
1973	if (ret)
1974	return ret;
1975	}
1976
1977	mpam_quirk_post_config_change(ris, partid, cfg);
1978
1979	return 0;
1980	}
1981
1982	/* Call with msc cfg_lock held */
1983	static int mpam_restore_mbwu_state(void *_ris)
1984	{
1985	int i;
1986	u64 val;
1987	struct mon_read mbwu_arg;
1988	struct mpam_msc_ris *ris = _ris;
1989	struct msmon_mbwu_state *mbwu_state;
1990	struct mpam_msc *msc = ris->vmsc->msc;
1991	struct mpam_class *class = ris->vmsc->comp->class;
1992
1993	for (i = 0; i < ris->props.num_mbwu_mon; i++) {
1994	if (WARN_ON_ONCE(!mpam_mon_sel_lock(msc)))
1995	return -EIO;
1996
1997	mbwu_state = &ris->mbwu_state[i];
1998
1999	if (!mbwu_state->enabled) {
2000	mpam_mon_sel_unlock(msc);
2001	continue;
2002	}
2003
2004	val = 0;
2005	mbwu_arg = (struct mon_read) {
2006	.ris = ris,
2007	.ctx = &mbwu_state->cfg,
2008	.type = mpam_msmon_choose_counter(class),
2009	.val = &val,
2010	};
2011
2012	mbwu_state->reset_on_next_read = true;
2013
2014	__ris_msmon_read_locked(&mbwu_arg);
2015
2016	mpam_mon_sel_unlock(msc);
2017
2018	if (mbwu_arg.err && mbwu_arg.err != -EBUSY)
2019	return mbwu_arg.err;
2020	}
2021
2022	return 0;
2023	}
2024
2025	/* Call with MSC cfg_lock held */
2026	static int mpam_save_mbwu_state(void *arg)
2027	{
2028	int i;
2029	int ret;
2030	u64 val;
2031	struct mon_cfg *cfg;
2032	struct mon_read mbwu_arg;
2033	u32 cur_flt, cur_ctl, mon_sel;
2034	struct mpam_msc_ris *ris = arg;
2035	struct msmon_mbwu_state *mbwu_state;
2036	struct mpam_msc *msc = ris->vmsc->msc;
2037	struct mpam_class *class = ris->vmsc->comp->class;
2038
2039	for (i = 0; i < ris->props.num_mbwu_mon; i++) {
2040	if (WARN_ON_ONCE(!mpam_mon_sel_lock(msc)))
2041	return -EIO;
2042
2043	mbwu_state = &ris->mbwu_state[i];
2044	cfg = &mbwu_state->cfg;
2045
2046	mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, i) \|
2047	FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx);
2048	ret = mpam_write_monsel_reg(msc, CFG_MON_SEL, mon_sel);
2049	if (ret)
2050	return ret;
2051	ret = mpam_read_monsel_reg(msc, CFG_MBWU_FLT, &cur_flt);
2052	if (ret)
2053	return ret;
2054	ret = mpam_read_monsel_reg(msc, CFG_MBWU_CTL, &cur_ctl);
2055	if (ret)
2056	return ret;
2057
2058	cfg->mon = i;
2059	cfg->pmg = FIELD_GET(MSMON_CFG_x_FLT_PMG, cur_flt);
2060	cfg->match_pmg = FIELD_GET(MSMON_CFG_x_CTL_MATCH_PMG, cur_ctl);
2061	cfg->partid = FIELD_GET(MSMON_CFG_x_FLT_PARTID, cur_flt);
2062	mbwu_state->enabled = FIELD_GET(MSMON_CFG_x_CTL_EN, cur_ctl);
2063
2064	if (!mbwu_state->enabled) {
2065	mpam_mon_sel_unlock(msc);
2066	continue;
2067	}
2068
2069	val = 0;
2070	mbwu_arg = (struct mon_read) {
2071	.ris = ris,
2072	.ctx = cfg,
2073	.type = mpam_msmon_choose_counter(class),
2074	.val = &val,
2075	};
2076
2077	__ris_msmon_read_locked(&mbwu_arg);
2078
2079	mbwu_state->reset_on_next_read = true;
2080	if (!mbwu_arg.err) {
2081	/*
2082	* __ris_msmon_read_locked() already included the
2083	* previous correction value.
2084	*/
2085	mbwu_state->correction = val;
2086	}
2087
2088	ret = mpam_write_monsel_reg(msc, CFG_MBWU_CTL, 0);
2089	if (ret)
2090	return ret;
2091
2092	mpam_mon_sel_unlock(msc);
2093	}
2094
2095	return 0;
2096	}
2097
2098	/*
2099	* Called via smp_call_on_cpu() to prevent migration, while still being
2100	* pre-emptible. Caller must hold mpam_srcu.
2101	*/
2102	static int mpam_reset_ris(void *arg)
2103	{
2104	u16 partid_max;
2105	struct mpam_config reset_cfg = {};
2106	struct mpam_msc_ris *ris = arg;
2107	int ret;
2108
2109	if (ris->in_reset_state)
2110	return 0;
2111
2112	spin_lock(&partid_max_lock);
2113	partid_max = mpam_partid_max;
2114	spin_unlock(&partid_max_lock);
2115	for (u32 partid = 0; partid <= partid_max; partid++) {
2116	ret = mpam_reprogram_ris_partid(ris, partid, &reset_cfg);
2117	if (ret)
2118	return ret;
2119	}
2120
2121	return 0;
2122	}
2123
2124	/*
2125	* Get the preferred CPU for this MSC. If it is accessible from this CPU,
2126	* this CPU is preferred. This can be preempted/migrated, it will only result
2127	* in more work.
2128	*/
2129	static int mpam_get_msc_preferred_cpu(struct mpam_msc *msc)
2130	{
2131	int cpu = raw_smp_processor_id();
2132
2133	if (cpumask_test_cpu(cpu, &msc->accessibility))
2134	return cpu;
2135
2136	return cpumask_first_and(&msc->accessibility, cpu_online_mask);
2137	}
2138
2139	static int mpam_touch_msc(struct mpam_msc msc, int (fn)(void a), void arg)
2140	{
2141	if (msc->iface != MPAM_IFACE_MMIO)
2142	return fn(arg);
2143
2144	lockdep_assert_irqs_enabled();
2145	lockdep_assert_cpus_held();
2146	WARN_ON_ONCE(!srcu_read_lock_held((&mpam_srcu)));
2147
2148	return smp_call_on_cpu(mpam_get_msc_preferred_cpu(msc), fn, arg, true);
2149	}
2150
2151	struct mpam_write_config_arg {
2152	struct mpam_msc_ris *ris;
2153	struct mpam_component *comp;
2154	u16 partid;
2155	};
2156
2157	static int __write_config(void *arg)
2158	{
2159	struct mpam_write_config_arg *c = arg;
2160
2161	return mpam_reprogram_ris_partid(c->ris, c->partid,
2162	&c->comp->cfg[c->partid]);
2163	}
2164
2165	static void mpam_reprogram_msc(struct mpam_msc *msc)
2166	{
2167	bool reset;
2168	struct mpam_config *cfg;
2169	struct mpam_msc_ris *ris;
2170	struct mpam_write_config_arg arg;
2171
2172	/*
2173	* No lock for mpam_partid_max as partid_max_published has been
2174	* set by mpam_enabled(), so the values can no longer change.
2175	*/
2176	mpam_assert_partid_sizes_fixed();
2177
2178	mutex_lock(&msc->cfg_lock);
2179	list_for_each_entry_srcu(ris, &msc->ris, msc_list,
2180	srcu_read_lock_held(&mpam_srcu)) {
2181	if (!mpam_is_enabled() && !ris->in_reset_state) {
2182	mpam_touch_msc(msc, &mpam_reset_ris, ris);
2183	ris->in_reset_state = true;
2184	continue;
2185	}
2186
2187	arg.comp = ris->vmsc->comp;
2188	arg.ris = ris;
2189	reset = true;
2190	for (u32 partid = 0; partid <= mpam_partid_max; partid++) {
2191	cfg = &ris->vmsc->comp->cfg[partid];
2192	if (!bitmap_empty(cfg->features, MPAM_FEATURE_LAST))
2193	reset = false;
2194
2195	arg.partid = partid;
2196	mpam_touch_msc(msc, __write_config, &arg);
2197	}
2198	ris->in_reset_state = reset;
2199
2200	if (mpam_has_feature(mpam_feat_msmon_mbwu, &ris->props))
2201	mpam_touch_msc(msc, &mpam_restore_mbwu_state, ris);
2202	}
2203	mutex_unlock(&msc->cfg_lock);
2204	}
2205
2206	static int mpam_enable_msc_ecr(void *_msc)
2207	{
2208	struct mpam_msc *msc = _msc;
2209
2210	return __mpam_write_reg(msc, MPAMF_ECR, MPAMF_ECR_INTEN);
2211	}
2212
2213	/* This can run in mpam_disable(), and the interrupt handler on the same CPU */
2214	static int mpam_disable_msc_ecr(void *_msc)
2215	{
2216	struct mpam_msc *msc = _msc;
2217
2218	return __mpam_write_reg(msc, MPAMF_ECR, 0);
2219	}
2220
2221	static void _enable_percpu_irq(void *_irq)
2222	{
2223	int *irq = _irq;
2224
2225	enable_percpu_irq(*irq, IRQ_TYPE_NONE);
2226	}
2227
2228	static int mpam_cpu_online(unsigned int cpu)
2229	{
2230	struct mpam_msc *msc;
2231
2232	guard(srcu)(&mpam_srcu);
2233	list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list,
2234	srcu_read_lock_held(&mpam_srcu)) {
2235	if (!cpumask_test_cpu(cpu, &msc->accessibility))
2236	continue;
2237
2238	if (msc->reenable_error_ppi)
2239	_enable_percpu_irq(&msc->reenable_error_ppi);
2240
2241	if (atomic_fetch_inc(&msc->online_refs) == 0) {
2242	mutex_lock(&msc->error_irq_lock);
2243	if (msc->error_irq_hw_enabled)
2244	mpam_touch_msc(msc, mpam_enable_msc_ecr, msc);
2245	mutex_unlock(&msc->error_irq_lock);
2246
2247	mpam_reprogram_msc(msc);
2248	}
2249	}
2250
2251	if (mpam_resctrl_enabled)
2252	return mpam_resctrl_online_cpu(cpu);
2253
2254	return 0;
2255	}
2256
2257	/* Before mpam is enabled, try to probe new MSC */
2258	static int mpam_discovery_cpu_online(unsigned int cpu)
2259	{
2260	int err = 0;
2261	struct mpam_msc *msc;
2262	bool new_device_probed = false;
2263
2264	if (mpam_is_enabled())
2265	return 0;
2266
2267	guard(srcu)(&mpam_srcu);
2268	list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list,
2269	srcu_read_lock_held(&mpam_srcu)) {
2270	if (!cpumask_test_cpu(cpu, &msc->accessibility))
2271	continue;
2272
2273	mutex_lock(&msc->probe_lock);
2274	if (!msc->probed)
2275	err = mpam_msc_hw_probe(msc);
2276	mutex_unlock(&msc->probe_lock);
2277
2278	if (err)
2279	break;
2280	new_device_probed = true;
2281	}
2282
2283	if (new_device_probed && !err)
2284	schedule_work(&mpam_enable_work);
2285	if (err) {
2286	mpam_disable_reason = "error during probing";
2287	schedule_work(&mpam_broken_work);
2288	}
2289
2290	return err;
2291	}
2292
2293	static int mpam_cpu_offline(unsigned int cpu)
2294	{
2295	struct mpam_msc *msc;
2296
2297	if (mpam_resctrl_enabled)
2298	mpam_resctrl_offline_cpu(cpu);
2299
2300	guard(srcu)(&mpam_srcu);
2301	list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list,
2302	srcu_read_lock_held(&mpam_srcu)) {
2303	if (!cpumask_test_cpu(cpu, &msc->accessibility))
2304	continue;
2305
2306	if (msc->reenable_error_ppi)
2307	disable_percpu_irq(msc->reenable_error_ppi);
2308
2309	if (atomic_dec_and_test(&msc->online_refs)) {
2310	struct mpam_msc_ris *ris;
2311
2312	mutex_lock(&msc->cfg_lock);
2313	list_for_each_entry_srcu(ris, &msc->ris, msc_list,
2314	srcu_read_lock_held(&mpam_srcu)) {
2315	mpam_touch_msc(msc, &mpam_reset_ris, ris);
2316
2317	/*
2318	* The reset state for non-zero partid may be
2319	* lost while the CPUs are offline.
2320	*/
2321	ris->in_reset_state = false;
2322
2323	if (mpam_is_enabled())
2324	mpam_touch_msc(msc, &mpam_save_mbwu_state, ris);
2325	}
2326	mutex_unlock(&msc->cfg_lock);
2327	}
2328	}
2329
2330	return 0;
2331	}
2332
2333	static void mpam_register_cpuhp_callbacks(int (*online)(unsigned int online),
2334	int (*offline)(unsigned int offline),
2335	char *name)
2336	{
2337	mutex_lock(&mpam_cpuhp_state_lock);
2338	if (mpam_cpuhp_state) {
2339	cpuhp_remove_state(mpam_cpuhp_state);
2340	mpam_cpuhp_state = 0;
2341	}
2342
2343	mpam_cpuhp_state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, name, online,
2344	offline);
2345	if (mpam_cpuhp_state <= 0) {
2346	pr_err("Failed to register cpuhp callbacks");
2347	mpam_cpuhp_state = 0;
2348	}
2349	mutex_unlock(&mpam_cpuhp_state_lock);
2350	}
2351
2352	static int __setup_ppi(struct mpam_msc *msc)
2353	{
2354	int cpu;
2355
2356	msc->error_dev_id = alloc_percpu(struct mpam_msc *);
2357	if (!msc->error_dev_id)
2358	return -ENOMEM;
2359
2360	for_each_cpu(cpu, &msc->accessibility)
2361	*per_cpu_ptr(msc->error_dev_id, cpu) = msc;
2362
2363	return 0;
2364	}
2365
2366	static int mpam_msc_setup_error_irq(struct mpam_msc *msc)
2367	{
2368	int irq;
2369
2370	irq = platform_get_irq_byname_optional(msc->pdev, "error");
2371	if (irq <= 0)
2372	return 0;
2373
2374	/* Allocate and initialise the percpu device pointer for PPI */
2375	if (irq_is_percpu(irq))
2376	return __setup_ppi(msc);
2377
2378	/* sanity check: shared interrupts can be routed anywhere? */
2379	if (!cpumask_equal(&msc->accessibility, cpu_possible_mask)) {
2380	pr_err_once("msc:%u is a private resource with a shared error interrupt",
2381	msc->id);
2382	return -EINVAL;
2383	}
2384
2385	return 0;
2386	}
2387
2388	/*
2389	* An MSC can control traffic from a set of CPUs, but may only be accessible
2390	* from a (hopefully wider) set of CPUs. The common reason for this is power
2391	* management. If all the CPUs in a cluster are in PSCI:CPU_SUSPEND, the
2392	* corresponding cache may also be powered off. By making accesses from
2393	* one of those CPUs, we ensure we don't access a cache that's powered off.
2394	*/
2395	static void update_msc_accessibility(struct mpam_msc *msc)
2396	{
2397	u32 affinity_id;
2398	int err;
2399
2400	err = device_property_read_u32(&msc->pdev->dev, "cpu_affinity",
2401	&affinity_id);
2402	if (err)
2403	cpumask_copy(&msc->accessibility, cpu_possible_mask);
2404	else
2405	acpi_pptt_get_cpus_from_container(affinity_id, &msc->accessibility);
2406	}
2407
2408	/*
2409	* There are two ways of reaching a struct mpam_msc_ris. Via the
2410	* class->component->vmsc->ris, or via the msc.
2411	* When destroying the msc, the other side needs unlinking and cleaning up too.
2412	*/
2413	static void mpam_msc_destroy(struct mpam_msc *msc)
2414	{
2415	struct platform_device *pdev = msc->pdev;
2416	struct mpam_msc_ris ris, tmp;
2417
2418	lockdep_assert_held(&mpam_list_lock);
2419
2420	list_for_each_entry_safe(ris, tmp, &msc->ris, msc_list)
2421	mpam_ris_destroy(ris);
2422
2423	list_del_rcu(&msc->all_msc_list);
2424	synchronize_srcu(&mpam_srcu);
2425	mpam_pcc_chan_put(msc->pcc_chan);
2426	platform_set_drvdata(pdev, NULL);
2427
2428	add_to_garbage(msc);
2429	}
2430
2431	static void mpam_msc_drv_remove(struct platform_device *pdev)
2432	{
2433	struct mpam_msc *msc = platform_get_drvdata(pdev);
2434
2435	if (!msc)
2436	return;
2437
2438	mutex_lock(&mpam_list_lock);
2439	mpam_msc_destroy(msc);
2440	mutex_unlock(&mpam_list_lock);
2441
2442	mpam_free_garbage();
2443	}
2444
2445	static struct mpam_msc do_mpam_msc_drv_probe(struct platform_device pdev)
2446	{
2447	int err;
2448	u32 pcc_subspace_id;
2449	struct mpam_msc *msc;
2450	struct resource *msc_res;
2451	struct device *dev = &pdev->dev;
2452
2453	lockdep_assert_held(&mpam_list_lock);
2454
2455	msc = devm_kzalloc(&pdev->dev, sizeof(*msc), GFP_KERNEL);
2456	if (!msc)
2457	return ERR_PTR(-ENOMEM);
2458	init_garbage(&msc->garbage);
2459	msc->garbage.pdev = pdev;
2460
2461	err = devm_mutex_init(dev, &msc->probe_lock);
2462	if (err)
2463	return ERR_PTR(err);
2464
2465	err = devm_mutex_init(dev, &msc->part_sel_lock);
2466	if (err)
2467	return ERR_PTR(err);
2468
2469	err = devm_mutex_init(dev, &msc->error_irq_lock);
2470	if (err)
2471	return ERR_PTR(err);
2472
2473	err = devm_mutex_init(dev, &msc->cfg_lock);
2474	if (err)
2475	return ERR_PTR(err);
2476
2477	msc->id = pdev->id;
2478	msc->pdev = pdev;
2479	INIT_LIST_HEAD_RCU(&msc->all_msc_list);
2480	INIT_LIST_HEAD_RCU(&msc->ris);
2481
2482	update_msc_accessibility(msc);
2483	if (cpumask_empty(&msc->accessibility)) {
2484	dev_err_once(dev, "MSC is not accessible from any CPU!");
2485	return ERR_PTR(-EINVAL);
2486	}
2487
2488	err = mpam_msc_setup_error_irq(msc);
2489	if (err)
2490	return ERR_PTR(err);
2491
2492	if (device_property_read_u32(dev, "pcc-channel", &pcc_subspace_id))
2493	msc->iface = MPAM_IFACE_MMIO;
2494	else
2495	msc->iface = MPAM_IFACE_PCC;
2496
2497	/* Lock type depends on MSC interface used */
2498	err = mpam_mon_sel_lock_init(dev, msc);
2499	if (err)
2500	return ERR_PTR(err);
2501
2502	if (msc->iface == MPAM_IFACE_MMIO) {
2503	void __iomem *io;
2504
2505	io = devm_platform_get_and_ioremap_resource(pdev, 0,
2506	&msc_res);
2507	if (IS_ERR(io)) {
2508	dev_err_once(dev, "Failed to map MSC base address\n");
2509	return ERR_CAST(io);
2510	}
2511	msc->mapped_hwpage_sz = msc_res->end - msc_res->start;
2512	msc->mapped_hwpage = io;
2513	} else if (msc->iface == MPAM_IFACE_PCC) {
2514	u32 mpam_fb_msc_id;
2515
2516	msc->pcc_chan = mpam_pcc_chan_get(dev, pcc_subspace_id);
2517	if (IS_ERR(msc->pcc_chan)) {
2518	pr_err("Failed to request MSC PCC channel\n");
2519	return ERR_CAST(msc->pcc_chan);
2520	}
2521
2522	err = mpam_fb_check_shared_buffer_size(msc);
2523	if (err) {
2524	mpam_pcc_chan_put(msc->pcc_chan);
2525
2526	return ERR_PTR(err);
2527	}
2528
2529	err = mpam_fb_check_protocol_version(msc);
2530	if (err) {
2531	mpam_pcc_chan_put(msc->pcc_chan);
2532
2533	return ERR_PTR(err);
2534	}
2535
2536	if (device_property_read_u32(&pdev->dev, "mpam-fb-msc-id",
2537	&mpam_fb_msc_id))
2538	mpam_fb_msc_id = msc->id;
2539	msc->fb_id = mpam_fb_msc_id;
2540	} else {
2541	return ERR_PTR(-EINVAL);
2542	}
2543
2544	list_add_rcu(&msc->all_msc_list, &mpam_all_msc);
2545	platform_set_drvdata(pdev, msc);
2546
2547	return msc;
2548	}
2549
2550	static int fw_num_msc;
2551
2552	static int mpam_msc_drv_probe(struct platform_device *pdev)
2553	{
2554	int err;
2555	struct mpam_msc *msc = NULL;
2556	void *plat_data = pdev->dev.platform_data;
2557
2558	mutex_lock(&mpam_list_lock);
2559	msc = do_mpam_msc_drv_probe(pdev);
2560	mutex_unlock(&mpam_list_lock);
2561
2562	if (IS_ERR(msc))
2563	return PTR_ERR(msc);
2564
2565	/* Create RIS entries described by firmware */
2566	err = acpi_mpam_parse_resources(msc, plat_data);
2567	if (err) {
2568	mpam_msc_drv_remove(pdev);
2569	return err;
2570	}
2571
2572	if (atomic_add_return(1, &mpam_num_msc) == fw_num_msc)
2573	mpam_register_cpuhp_callbacks(mpam_discovery_cpu_online, NULL,
2574	"mpam:drv_probe");
2575
2576	return 0;
2577	}
2578
2579	static struct platform_driver mpam_msc_driver = {
2580	.driver = {
2581	.name = "mpam_msc",
2582	.suppress_bind_attrs = true,
2583	},
2584	.probe = mpam_msc_drv_probe,
2585	.remove = mpam_msc_drv_remove,
2586	};
2587
2588	/* Any of these features mean the BWA_WD field is valid. */
2589	static bool mpam_has_bwa_wd_feature(struct mpam_props *props)
2590	{
2591	if (mpam_has_feature(mpam_feat_mbw_min, props))
2592	return true;
2593	if (mpam_has_feature(mpam_feat_mbw_max, props))
2594	return true;
2595	if (mpam_has_feature(mpam_feat_mbw_prop, props))
2596	return true;
2597	return false;
2598	}
2599
2600	/* Any of these features mean the CMAX_WD field is valid. */
2601	static bool mpam_has_cmax_wd_feature(struct mpam_props *props)
2602	{
2603	if (mpam_has_feature(mpam_feat_cmax_cmax, props))
2604	return true;
2605	if (mpam_has_feature(mpam_feat_cmax_cmin, props))
2606	return true;
2607	return false;
2608	}
2609
2610	#define MISMATCHED_HELPER(parent, child, helper, field, alias) \
2611	helper(parent) && \
2612	((helper(child) && (parent)->field != (child)->field) \|\| \
2613	(!helper(child) && !(alias)))
2614
2615	#define MISMATCHED_FEAT(parent, child, feat, field, alias) \
2616	mpam_has_feature((feat), (parent)) && \
2617	((mpam_has_feature((feat), (child)) && (parent)->field != (child)->field) \|\| \
2618	(!mpam_has_feature((feat), (child)) && !(alias)))
2619
2620	#define CAN_MERGE_FEAT(parent, child, feat, alias) \
2621	(alias) && !mpam_has_feature((feat), (parent)) && \
2622	mpam_has_feature((feat), (child))
2623
2624	/*
2625	* Combine two props fields.
2626	* If this is for controls that alias the same resource, it is safe to just
2627	* copy the values over. If two aliasing controls implement the same scheme
2628	* a safe value must be picked.
2629	* For non-aliasing controls, these control different resources, and the
2630	* resulting safe value must be compatible with both. When merging values in
2631	* the tree, all the aliasing resources must be handled first.
2632	* On mismatch, parent is modified.
2633	* Quirks on an MSC will apply to all MSC in that class.
2634	*/
2635	static void __props_mismatch(struct mpam_props *parent,
2636	struct mpam_props *child, bool alias)
2637	{
2638	if (CAN_MERGE_FEAT(parent, child, mpam_feat_cpor_part, alias)) {
2639	parent->cpbm_wd = child->cpbm_wd;
2640	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_cpor_part,
2641	cpbm_wd, alias)) {
2642	pr_debug("cleared cpor_part\n");
2643	mpam_clear_feature(mpam_feat_cpor_part, parent);
2644	parent->cpbm_wd = 0;
2645	}
2646
2647	if (CAN_MERGE_FEAT(parent, child, mpam_feat_mbw_part, alias)) {
2648	parent->mbw_pbm_bits = child->mbw_pbm_bits;
2649	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_mbw_part,
2650	mbw_pbm_bits, alias)) {
2651	pr_debug("cleared mbw_part\n");
2652	mpam_clear_feature(mpam_feat_mbw_part, parent);
2653	parent->mbw_pbm_bits = 0;
2654	}
2655
2656	/* bwa_wd is a count of bits, fewer bits means less precision */
2657	if (alias && !mpam_has_bwa_wd_feature(parent) &&
2658	mpam_has_bwa_wd_feature(child)) {
2659	parent->bwa_wd = child->bwa_wd;
2660	} else if (MISMATCHED_HELPER(parent, child, mpam_has_bwa_wd_feature,
2661	bwa_wd, alias)) {
2662	pr_debug("took the min bwa_wd\n");
2663	parent->bwa_wd = min(parent->bwa_wd, child->bwa_wd);
2664	}
2665
2666	if (alias && !mpam_has_cmax_wd_feature(parent) && mpam_has_cmax_wd_feature(child)) {
2667	parent->cmax_wd = child->cmax_wd;
2668	} else if (MISMATCHED_HELPER(parent, child, mpam_has_cmax_wd_feature,
2669	cmax_wd, alias)) {
2670	pr_debug("%s took the min cmax_wd\n", __func__);
2671	parent->cmax_wd = min(parent->cmax_wd, child->cmax_wd);
2672	}
2673
2674	if (CAN_MERGE_FEAT(parent, child, mpam_feat_cmax_cassoc, alias)) {
2675	parent->cassoc_wd = child->cassoc_wd;
2676	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_cmax_cassoc,
2677	cassoc_wd, alias)) {
2678	pr_debug("%s cleared cassoc_wd\n", __func__);
2679	mpam_clear_feature(mpam_feat_cmax_cassoc, parent);
2680	parent->cassoc_wd = 0;
2681	}
2682
2683	/* For num properties, take the minimum */
2684	if (CAN_MERGE_FEAT(parent, child, mpam_feat_msmon_csu, alias)) {
2685	parent->num_csu_mon = child->num_csu_mon;
2686	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_msmon_csu,
2687	num_csu_mon, alias)) {
2688	pr_debug("took the min num_csu_mon\n");
2689	parent->num_csu_mon = min(parent->num_csu_mon,
2690	child->num_csu_mon);
2691	}
2692
2693	if (CAN_MERGE_FEAT(parent, child, mpam_feat_msmon_mbwu, alias)) {
2694	parent->num_mbwu_mon = child->num_mbwu_mon;
2695	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_msmon_mbwu,
2696	num_mbwu_mon, alias)) {
2697	pr_debug("took the min num_mbwu_mon\n");
2698	parent->num_mbwu_mon = min(parent->num_mbwu_mon,
2699	child->num_mbwu_mon);
2700	}
2701
2702	if (CAN_MERGE_FEAT(parent, child, mpam_feat_intpri_part, alias)) {
2703	parent->intpri_wd = child->intpri_wd;
2704	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_intpri_part,
2705	intpri_wd, alias)) {
2706	pr_debug("%s took the min intpri_wd\n", __func__);
2707	parent->intpri_wd = min(parent->intpri_wd, child->intpri_wd);
2708	}
2709
2710	if (CAN_MERGE_FEAT(parent, child, mpam_feat_dspri_part, alias)) {
2711	parent->dspri_wd = child->dspri_wd;
2712	} else if (MISMATCHED_FEAT(parent, child, mpam_feat_dspri_part,
2713	dspri_wd, alias)) {
2714	pr_debug("%s took the min dspri_wd\n", __func__);
2715	parent->dspri_wd = min(parent->dspri_wd, child->dspri_wd);
2716	}
2717
2718	/* TODO: alias support for these two */
2719	/* {int,ds}pri may not have differing 0-low behaviour */
2720	if (mpam_has_feature(mpam_feat_intpri_part, parent) &&
2721	(!mpam_has_feature(mpam_feat_intpri_part, child) \|\|
2722	mpam_has_feature(mpam_feat_intpri_part_0_low, parent) !=
2723	mpam_has_feature(mpam_feat_intpri_part_0_low, child))) {
2724	pr_debug("%s cleared intpri_part\n", __func__);
2725	mpam_clear_feature(mpam_feat_intpri_part, parent);
2726	mpam_clear_feature(mpam_feat_intpri_part_0_low, parent);
2727	}
2728	if (mpam_has_feature(mpam_feat_dspri_part, parent) &&
2729	(!mpam_has_feature(mpam_feat_dspri_part, child) \|\|
2730	mpam_has_feature(mpam_feat_dspri_part_0_low, parent) !=
2731	mpam_has_feature(mpam_feat_dspri_part_0_low, child))) {
2732	pr_debug("%s cleared dspri_part\n", __func__);
2733	mpam_clear_feature(mpam_feat_dspri_part, parent);
2734	mpam_clear_feature(mpam_feat_dspri_part_0_low, parent);
2735	}
2736
2737	if (alias) {
2738	/* Merge features for aliased resources */
2739	bitmap_or(parent->features, parent->features, child->features, MPAM_FEATURE_LAST);
2740	} else {
2741	/* Clear missing features for non aliasing */
2742	bitmap_and(parent->features, parent->features, child->features, MPAM_FEATURE_LAST);
2743	}
2744	}
2745
2746	/*
2747	* If a vmsc doesn't match class feature/configuration, do the right thing(tm).
2748	* For 'num' properties we can just take the minimum.
2749	* For properties where the mismatched unused bits would make a difference, we
2750	* nobble the class feature, as we can't configure all the resources.
2751	* e.g. The L3 cache is composed of two resources with 13 and 17 portion
2752	* bitmaps respectively.
2753	* Quirks on an MSC will apply to all MSC in that class.
2754	*/
2755	static void
2756	__class_props_mismatch(struct mpam_class class, struct mpam_vmsc vmsc)
2757	{
2758	struct mpam_props *cprops = &class->props;
2759	struct mpam_props *vprops = &vmsc->props;
2760	struct device *dev = &vmsc->msc->pdev->dev;
2761
2762	lockdep_assert_held(&mpam_list_lock); /* we modify class */
2763
2764	dev_dbg(dev, "Merging features for class:0x%lx &= vmsc:0x%lx\n",
2765	(long)cprops->features, (long)vprops->features);
2766
2767	/* Merge quirks */
2768	class->quirks \|= vmsc->msc->quirks;
2769
2770	/* Take the safe value for any common features */
2771	__props_mismatch(cprops, vprops, false);
2772	}
2773
2774	static void
2775	__vmsc_props_mismatch(struct mpam_vmsc vmsc, struct mpam_msc_ris ris)
2776	{
2777	struct mpam_props *rprops = &ris->props;
2778	struct mpam_props *vprops = &vmsc->props;
2779	struct device *dev = &vmsc->msc->pdev->dev;
2780
2781	lockdep_assert_held(&mpam_list_lock); /* we modify vmsc */
2782
2783	dev_dbg(dev, "Merging features for vmsc:0x%lx \|= ris:0x%lx\n",
2784	(long)vprops->features, (long)rprops->features);
2785
2786	/*
2787	* Merge mismatched features - Copy any features that aren't common,
2788	* but take the safe value for any common features.
2789	*/
2790	__props_mismatch(vprops, rprops, true);
2791	}
2792
2793	/*
2794	* Copy the first component's first vMSC's properties and features to the
2795	* class. __class_props_mismatch() will remove conflicts.
2796	* It is not possible to have a class with no components, or a component with
2797	* no resources. The vMSC properties have already been built.
2798	*/
2799	static void mpam_enable_init_class_features(struct mpam_class *class)
2800	{
2801	struct mpam_vmsc *vmsc;
2802	struct mpam_component *comp;
2803
2804	comp = list_first_entry(&class->components,
2805	struct mpam_component, class_list);
2806	vmsc = list_first_entry(&comp->vmsc,
2807	struct mpam_vmsc, comp_list);
2808
2809	class->props = vmsc->props;
2810	}
2811
2812	static void mpam_enable_merge_vmsc_features(struct mpam_component *comp)
2813	{
2814	struct mpam_vmsc *vmsc;
2815	struct mpam_msc_ris *ris;
2816	struct mpam_class *class = comp->class;
2817
2818	list_for_each_entry(vmsc, &comp->vmsc, comp_list) {
2819	list_for_each_entry(ris, &vmsc->ris, vmsc_list) {
2820	__vmsc_props_mismatch(vmsc, ris);
2821	class->nrdy_usec = max(class->nrdy_usec,
2822	vmsc->msc->nrdy_usec);
2823	}
2824	}
2825	}
2826
2827	static void mpam_enable_merge_class_features(struct mpam_component *comp)
2828	{
2829	struct mpam_vmsc *vmsc;
2830	struct mpam_class *class = comp->class;
2831
2832	list_for_each_entry(vmsc, &comp->vmsc, comp_list)
2833	__class_props_mismatch(class, vmsc);
2834
2835	if (mpam_has_quirk(T241_FORCE_MBW_MIN_TO_ONE, class))
2836	mpam_clear_feature(mpam_feat_mbw_min, &class->props);
2837	}
2838
2839	/*
2840	* Merge all the common resource features into class.
2841	* vmsc features are bitwise-or'd together by mpam_enable_merge_vmsc_features()
2842	* as the first step so that mpam_enable_init_class_features() can initialise
2843	* the class with a representative set of features.
2844	* Next the mpam_enable_merge_class_features() bitwise-and's all the vmsc
2845	* features to form the class features.
2846	* Other features are the min/max as appropriate.
2847	*
2848	* To avoid walking the whole tree twice, the class->nrdy_usec property is
2849	* updated when working with the vmsc as it is a max(), and doesn't need
2850	* initialising first.
2851	*/
2852	static void mpam_enable_merge_features(struct list_head *all_classes_list)
2853	{
2854	struct mpam_class *class;
2855	struct mpam_component *comp;
2856
2857	lockdep_assert_held(&mpam_list_lock);
2858
2859	list_for_each_entry(class, all_classes_list, classes_list) {
2860	list_for_each_entry(comp, &class->components, class_list)
2861	mpam_enable_merge_vmsc_features(comp);
2862
2863	mpam_enable_init_class_features(class);
2864
2865	list_for_each_entry(comp, &class->components, class_list)
2866	mpam_enable_merge_class_features(comp);
2867	}
2868	}
2869
2870	static char *mpam_errcode_names[16] = {
2871	[MPAM_ERRCODE_NONE] = "No error",
2872	[MPAM_ERRCODE_PARTID_SEL_RANGE] = "PARTID_SEL_Range",
2873	[MPAM_ERRCODE_REQ_PARTID_RANGE] = "Req_PARTID_Range",
2874	[MPAM_ERRCODE_MSMONCFG_ID_RANGE] = "MSMONCFG_ID_RANGE",
2875	[MPAM_ERRCODE_REQ_PMG_RANGE] = "Req_PMG_Range",
2876	[MPAM_ERRCODE_MONITOR_RANGE] = "Monitor_Range",
2877	[MPAM_ERRCODE_INTPARTID_RANGE] = "intPARTID_Range",
2878	[MPAM_ERRCODE_UNEXPECTED_INTERNAL] = "Unexpected_INTERNAL",
2879	[MPAM_ERRCODE_UNDEFINED_RIS_PART_SEL] = "Undefined_RIS_PART_SEL",
2880	[MPAM_ERRCODE_RIS_NO_CONTROL] = "RIS_No_Control",
2881	[MPAM_ERRCODE_UNDEFINED_RIS_MON_SEL] = "Undefined_RIS_MON_SEL",
2882	[MPAM_ERRCODE_RIS_NO_MONITOR] = "RIS_No_Monitor",
2883	[12 ... 15] = "Reserved"
2884	};
2885
2886	/*
2887	* This will run as the threaded IRQ handler part when using MPAM-Fb, but
2888	* as the sole hard-IRQ handler for MMIO based accesses.
2889	*/
2890	static irqreturn_t __mpam_irq_handler(int irq, struct mpam_msc *msc)
2891	{
2892	u64 reg;
2893	int ret;
2894	u16 partid;
2895	u8 errcode, pmg, ris;
2896
2897	if (WARN_ON_ONCE(!msc))
2898	return IRQ_NONE;
2899
2900	if (msc->iface == MPAM_IFACE_MMIO &&
2901	WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(),
2902	&msc->accessibility)))
2903	return IRQ_NONE;
2904
2905	ret = mpam_msc_read_esr(msc, &reg);
2906	if (ret) {
2907	pr_err_ratelimited("unknown error irq from msc:%u\n", msc->id);
2908
2909	/* Try out best here ... */
2910	goto out_disable;
2911	}
2912
2913	errcode = FIELD_GET(MPAMF_ESR_ERRCODE, reg);
2914	if (!errcode)
2915	return IRQ_NONE;
2916
2917	/*
2918	* Clear the level triggered IRQ. If that fails, we cannot do anything
2919	* about it, so ignore any errors. We will disable the IRQ either on
2920	* the device side or on the irqchip level next anyway.
2921	*/
2922	mpam_msc_clear_esr(msc);
2923
2924	partid = FIELD_GET(MPAMF_ESR_PARTID_MON, reg);
2925	pmg = FIELD_GET(MPAMF_ESR_PMG, reg);
2926	ris = FIELD_GET(MPAMF_ESR_RIS, reg);
2927
2928	pr_err_ratelimited("error irq from msc:%u '%s', partid:%u, pmg: %u, ris: %u\n",
2929	msc->id, mpam_errcode_names[errcode], partid, pmg,
2930	ris);
2931
2932	out_disable:
2933	/*
2934	* Disable this interrupt on the device side. If that fails, disable
2935	* the IRQ on the irqchip level, as we must prevent further handler
2936	* invocations. We only take the interrupt once anyway, as we
2937	* are going to free the IRQ next, in mpam_disable().
2938	*/
2939	ret = mpam_disable_msc_ecr(msc);
2940	if (ret)
2941	disable_irq_nosync(irq);
2942
2943	/* Check whether we are racing with the thread disabling MPAM. */
2944	if (!mpam_is_enabled())
2945	return IRQ_HANDLED;
2946
2947	/*
2948	* Schedule the teardown work. We have to defer it as we can't
2949	* unregister the interrupt from the threaded part of a handler.
2950	*/
2951	mpam_disable_reason = "hardware error interrupt";
2952	schedule_work(&mpam_broken_work);
2953
2954	return IRQ_HANDLED;
2955	}
2956
2957	static irqreturn_t mpam_ppi_handler(int irq, void *dev_id)
2958	{
2959	struct mpam_msc msc = (struct mpam_msc **)dev_id;
2960
2961	return __mpam_irq_handler(irq, msc);
2962	}
2963
2964	/*
2965	* MMIO based MSC accesses must run in non-preemptible context, as they
2966	* might have affinity requirements to check.
2967	* MPAM-Fb based MSC accesses must NOT run in hard-IRQ context, as they
2968	* can sleep.
2969	* So split the IRQ handling up, depending on the MSC access type.
2970	*/
2971	static irqreturn_t mpam_shared_hard_irq(int irq, void *dev_id)
2972	{
2973	struct mpam_msc *msc = dev_id;
2974
2975	if (msc->iface == MPAM_IFACE_MMIO)
2976	return __mpam_irq_handler(irq, msc);
2977
2978	return IRQ_WAKE_THREAD;
2979	}
2980
2981	static irqreturn_t mpam_shared_thread_irq(int irq, void *dev_id)
2982	{
2983	struct mpam_msc *msc = dev_id;
2984
2985	if (msc->iface == MPAM_IFACE_MMIO)
2986	return IRQ_HANDLED;
2987
2988	return __mpam_irq_handler(irq, msc);
2989	}
2990
2991	static int mpam_register_irqs(void)
2992	{
2993	int err, irq;
2994	struct mpam_msc *msc;
2995
2996	lockdep_assert_cpus_held();
2997
2998	guard(srcu)(&mpam_srcu);
2999	list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list,
3000	srcu_read_lock_held(&mpam_srcu)) {
3001	irq = platform_get_irq_byname_optional(msc->pdev, "error");
3002	if (irq <= 0)
3003	continue;
3004
3005	/* The MPAM spec says the interrupt can be SPI, PPI or LPI */
3006	/* We anticipate sharing the interrupt with other MSCs */
3007	if (irq_is_percpu(irq)) {
3008	if (msc->iface != MPAM_IFACE_MMIO) {
3009	dev_err(&msc->pdev->dev,
3010	"Only MMIO MSCs can use per-CPU interrupts\n");
3011	return -EINVAL;
3012	}
3013	err = request_percpu_irq(irq, &mpam_ppi_handler,
3014	"mpam:msc:error",
3015	msc->error_dev_id);
3016	if (err)
3017	return err;
3018
3019	msc->reenable_error_ppi = irq;
3020	smp_call_function_many(&msc->accessibility,
3021	&_enable_percpu_irq, &irq,
3022	true);
3023	} else {
3024	err = devm_request_threaded_irq(&msc->pdev->dev, irq,
3025	&mpam_shared_hard_irq,
3026	&mpam_shared_thread_irq,
3027	IRQF_SHARED \| IRQF_ONESHOT,
3028	"mpam:msc:error", msc);
3029	if (err)
3030	return err;
3031	}
3032
3033	guard(mutex)(&msc->error_irq_lock);
3034	msc->error_irq_req = true;
3035	err = mpam_touch_msc(msc, mpam_enable_msc_ecr, msc);
3036	if (err)
3037	return err;
3038	msc->error_irq_hw_enabled = true;
3039	}
3040
3041	return 0;
3042	}
3043
3044	static void mpam_unregister_irqs(void)
3045	{
3046	int irq;
3047	struct mpam_msc *msc;
3048
3049	guard(cpus_read_lock)();
3050	guard(srcu)(&mpam_srcu);
3051	list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list,
3052	srcu_read_lock_held(&mpam_srcu)) {
3053	irq = platform_get_irq_byname_optional(msc->pdev, "error");
3054	if (irq <= 0)
3055	continue;
3056
3057	mutex_lock(&msc->error_irq_lock);
3058	if (msc->error_irq_hw_enabled) {
3059	if (!mpam_touch_msc(msc, mpam_disable_msc_ecr, msc))
3060	msc->error_irq_hw_enabled = false;
3061	}
3062
3063	if (msc->error_irq_req) {
3064	if (irq_is_percpu(irq)) {
3065	msc->reenable_error_ppi = 0;
3066	free_percpu_irq(irq, msc->error_dev_id);
3067	} else {
3068	devm_free_irq(&msc->pdev->dev, irq, msc);
3069	}
3070	msc->error_irq_req = false;
3071	}
3072	mutex_unlock(&msc->error_irq_lock);
3073	}
3074	}
3075
3076	static void __destroy_component_cfg(struct mpam_component *comp)
3077	{
3078	struct mpam_msc *msc;
3079	struct mpam_vmsc *vmsc;
3080	struct mpam_msc_ris *ris;
3081
3082	lockdep_assert_held(&mpam_list_lock);
3083
3084	if (!comp->cfg)
3085	return;
3086
3087	add_to_garbage(comp->cfg);
3088	list_for_each_entry(vmsc, &comp->vmsc, comp_list) {
3089	msc = vmsc->msc;
3090
3091	if (mpam_mon_sel_lock(msc)) {
3092	list_for_each_entry(ris, &vmsc->ris, vmsc_list) {
3093	if (ris->mbwu_state)
3094	add_to_garbage(ris->mbwu_state);
3095	}
3096	mpam_mon_sel_unlock(msc);
3097	}
3098	}
3099	}
3100
3101	static void mpam_reset_component_cfg(struct mpam_component *comp)
3102	{
3103	int i;
3104	struct mpam_props *cprops = &comp->class->props;
3105
3106	mpam_assert_partid_sizes_fixed();
3107
3108	if (!comp->cfg)
3109	return;
3110
3111	for (i = 0; i <= mpam_partid_max; i++) {
3112	comp->cfg[i] = (struct mpam_config) {};
3113	if (cprops->cpbm_wd)
3114	comp->cfg[i].cpbm = GENMASK(cprops->cpbm_wd - 1, 0);
3115	if (cprops->mbw_pbm_bits)
3116	comp->cfg[i].mbw_pbm = GENMASK(cprops->mbw_pbm_bits - 1, 0);
3117	if (cprops->bwa_wd)
3118	comp->cfg[i].mbw_max = GENMASK(15, 16 - cprops->bwa_wd);
3119	}
3120	}
3121
3122	static int __allocate_component_cfg(struct mpam_component *comp)
3123	{
3124	struct mpam_vmsc *vmsc;
3125
3126	mpam_assert_partid_sizes_fixed();
3127
3128	if (comp->cfg)
3129	return 0;
3130
3131	comp->cfg = kvzalloc_objs(*comp->cfg, mpam_partid_max + 1);
3132	if (!comp->cfg)
3133	return -ENOMEM;
3134
3135	/*
3136	* The array is free()d in one go, so only cfg[0]'s structure needs
3137	* to be initialised.
3138	*/
3139	init_garbage(&comp->cfg[0].garbage);
3140
3141	mpam_reset_component_cfg(comp);
3142
3143	list_for_each_entry(vmsc, &comp->vmsc, comp_list) {
3144	struct mpam_msc *msc;
3145	struct mpam_msc_ris *ris;
3146	struct msmon_mbwu_state *mbwu_state;
3147
3148	if (!vmsc->props.num_mbwu_mon)
3149	continue;
3150
3151	msc = vmsc->msc;
3152	list_for_each_entry(ris, &vmsc->ris, vmsc_list) {
3153	if (!ris->props.num_mbwu_mon)
3154	continue;
3155
3156	mbwu_state = kzalloc_objs(*ris->mbwu_state,
3157	ris->props.num_mbwu_mon);
3158	if (!mbwu_state) {
3159	__destroy_component_cfg(comp);
3160	return -ENOMEM;
3161	}
3162
3163	init_garbage(&mbwu_state[0].garbage);
3164
3165	if (mpam_mon_sel_lock(msc)) {
3166	ris->mbwu_state = mbwu_state;
3167	mpam_mon_sel_unlock(msc);
3168	}
3169	}
3170	}
3171
3172	return 0;
3173	}
3174
3175	static int mpam_allocate_config(void)
3176	{
3177	struct mpam_class *class;
3178	struct mpam_component *comp;
3179
3180	lockdep_assert_held(&mpam_list_lock);
3181
3182	list_for_each_entry(class, &mpam_classes, classes_list) {
3183	list_for_each_entry(comp, &class->components, class_list) {
3184	int err = __allocate_component_cfg(comp);
3185	if (err)
3186	return err;
3187	}
3188	}
3189
3190	return 0;
3191	}
3192
3193	static void mpam_enable_once(void)
3194	{
3195	int err;
3196
3197	/*
3198	* Once the cpuhp callbacks have been changed, mpam_partid_max can no
3199	* longer change.
3200	*/
3201	spin_lock(&partid_max_lock);
3202	partid_max_published = true;
3203	spin_unlock(&partid_max_lock);
3204
3205	/*
3206	* If all the MSC have been probed, enabling the IRQs happens next.
3207	* That involves cross-calling to a CPU that can reach the MSC, and
3208	* the locks must be taken in this order:
3209	*/
3210	cpus_read_lock();
3211	mutex_lock(&mpam_list_lock);
3212	do {
3213	mpam_enable_merge_features(&mpam_classes);
3214
3215	err = mpam_register_irqs();
3216	if (err) {
3217	pr_warn("Failed to register irqs: %d\n", err);
3218	break;
3219	}
3220
3221	err = mpam_allocate_config();
3222	if (err) {
3223	pr_err("Failed to allocate configuration arrays.\n");
3224	break;
3225	}
3226	} while (0);
3227	mutex_unlock(&mpam_list_lock);
3228	cpus_read_unlock();
3229
3230	if (!err) {
3231	err = mpam_resctrl_setup();
3232	if (err)
3233	pr_err("Failed to initialise resctrl: %d\n", err);
3234	}
3235
3236	if (err) {
3237	mpam_disable_reason = "Failed to enable.";
3238	schedule_work(&mpam_broken_work);
3239	return;
3240	}
3241
3242	static_branch_enable(&mpam_enabled);
3243	mpam_resctrl_enabled = true;
3244	mpam_register_cpuhp_callbacks(mpam_cpu_online, mpam_cpu_offline,
3245	"mpam:online");
3246
3247	/* Use printk() to avoid the pr_fmt adding the function name. */
3248	printk(KERN_INFO "MPAM enabled with %u PARTIDs and %u PMGs\n",
3249	mpam_partid_max + 1, mpam_pmg_max + 1);
3250	}
3251
3252	static int mpam_reset_component_locked(struct mpam_component *comp)
3253	{
3254	struct mpam_vmsc *vmsc;
3255
3256	lockdep_assert_cpus_held();
3257	mpam_assert_partid_sizes_fixed();
3258
3259	mpam_reset_component_cfg(comp);
3260
3261	guard(srcu)(&mpam_srcu);
3262	list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list,
3263	srcu_read_lock_held(&mpam_srcu)) {
3264	struct mpam_msc *msc = vmsc->msc;
3265	struct mpam_msc_ris *ris;
3266	int ret;
3267
3268	list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list,
3269	srcu_read_lock_held(&mpam_srcu)) {
3270	if (!ris->in_reset_state) {
3271	ret = mpam_touch_msc(msc, mpam_reset_ris, ris);
3272	if (ret)
3273	return ret;
3274	}
3275	ris->in_reset_state = true;
3276	}
3277	}
3278
3279	return 0;
3280	}
3281
3282	int mpam_reset_class_locked(struct mpam_class *class)
3283	{
3284	struct mpam_component *comp;
3285	int ret;
3286
3287	lockdep_assert_cpus_held();
3288
3289	guard(srcu)(&mpam_srcu);
3290	list_for_each_entry_srcu(comp, &class->components, class_list,
3291	srcu_read_lock_held(&mpam_srcu)) {
3292	ret = mpam_reset_component_locked(comp);
3293	if (ret)
3294	return ret;
3295	}
3296
3297	return 0;
3298	}
3299
3300	static void mpam_reset_class(struct mpam_class *class)
3301	{
3302	cpus_read_lock();
3303	mpam_reset_class_locked(class);
3304	cpus_read_unlock();
3305	}
3306
3307	/*
3308	* Called in response to an error IRQ.
3309	* All of MPAMs errors indicate a software bug, restore any modified
3310	* controls to their reset values.
3311	*/
3312	void mpam_disable(struct work_struct *ignored)
3313	{
3314	int idx;
3315	bool do_resctrl_exit;
3316	struct mpam_class *class;
3317	struct mpam_msc msc, tmp;
3318
3319	if (mpam_is_enabled())
3320	static_branch_disable(&mpam_enabled);
3321
3322	mutex_lock(&mpam_cpuhp_state_lock);
3323	if (mpam_cpuhp_state) {
3324	cpuhp_remove_state(mpam_cpuhp_state);
3325	mpam_cpuhp_state = 0;
3326	}
3327
3328	/*
3329	* Removing the cpuhp state called mpam_cpu_offline() and told resctrl
3330	* all the CPUs are offline.
3331	*/
3332	do_resctrl_exit = mpam_resctrl_enabled;
3333	mpam_resctrl_enabled = false;
3334	mutex_unlock(&mpam_cpuhp_state_lock);
3335
3336	if (do_resctrl_exit)
3337	mpam_resctrl_exit();
3338
3339	mpam_unregister_irqs();
3340
3341	idx = srcu_read_lock(&mpam_srcu);
3342	list_for_each_entry_srcu(class, &mpam_classes, classes_list,
3343	srcu_read_lock_held(&mpam_srcu)) {
3344	mpam_reset_class(class);
3345	if (do_resctrl_exit)
3346	mpam_resctrl_teardown_class(class);
3347	}
3348	srcu_read_unlock(&mpam_srcu, idx);
3349
3350	mutex_lock(&mpam_list_lock);
3351	list_for_each_entry_safe(msc, tmp, &mpam_all_msc, all_msc_list)
3352	mpam_msc_destroy(msc);
3353	mutex_unlock(&mpam_list_lock);
3354	mpam_free_garbage();
3355
3356	if (mpam_disable_errno \|\| mpam_disable_mpam_fb_err)
3357	pr_err_once("MPAM disabled due to %s: %s, MPAM-Fb error %d\n",
3358	mpam_disable_reason, errname(mpam_disable_errno),
3359	mpam_disable_mpam_fb_err);
3360	else
3361	pr_err_once("MPAM disabled due to %s\n", mpam_disable_reason);
3362	}
3363
3364	/*
3365	* Enable mpam once all devices have been probed.
3366	* Scheduled by mpam_discovery_cpu_online() once all devices have been created.
3367	* Also scheduled when new devices are probed when new CPUs come online.
3368	*/
3369	void mpam_enable(struct work_struct *work)
3370	{
3371	static atomic_t once;
3372	struct mpam_msc *msc;
3373	bool all_devices_probed = true;
3374
3375	/* Have we probed all the hw devices? */
3376	guard(srcu)(&mpam_srcu);
3377	list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list,
3378	srcu_read_lock_held(&mpam_srcu)) {
3379	mutex_lock(&msc->probe_lock);
3380	if (!msc->probed)
3381	all_devices_probed = false;
3382	mutex_unlock(&msc->probe_lock);
3383
3384	if (!all_devices_probed)
3385	break;
3386	}
3387
3388	if (all_devices_probed && !atomic_fetch_inc(&once))
3389	mpam_enable_once();
3390	}
3391
3392	#define maybe_update_config(cfg, feature, newcfg, member, changes) do { \
3393	if (mpam_has_feature(feature, newcfg) && \
3394	(newcfg)->member != (cfg)->member) { \
3395	(cfg)->member = (newcfg)->member; \
3396	mpam_set_feature(feature, cfg); \
3397	\
3398	(changes) = true; \
3399	} \
3400	} while (0)
3401
3402	static bool mpam_update_config(struct mpam_config *cfg,
3403	const struct mpam_config *newcfg)
3404	{
3405	bool has_changes = false;
3406
3407	maybe_update_config(cfg, mpam_feat_cpor_part, newcfg, cpbm, has_changes);
3408	maybe_update_config(cfg, mpam_feat_mbw_part, newcfg, mbw_pbm, has_changes);
3409	maybe_update_config(cfg, mpam_feat_mbw_max, newcfg, mbw_max, has_changes);
3410
3411	return has_changes;
3412	}
3413
3414	int mpam_apply_config(struct mpam_component *comp, u16 partid,
3415	struct mpam_config *cfg)
3416	{
3417	struct mpam_write_config_arg arg;
3418	struct mpam_msc_ris *ris;
3419	struct mpam_vmsc *vmsc;
3420	struct mpam_msc *msc;
3421	int ret;
3422
3423	lockdep_assert_cpus_held();
3424
3425	/* Don't pass in the current config! */
3426	WARN_ON_ONCE(&comp->cfg[partid] == cfg);
3427
3428	if (!mpam_update_config(&comp->cfg[partid], cfg))
3429	return 0;
3430
3431	arg.comp = comp;
3432	arg.partid = partid;
3433
3434	guard(srcu)(&mpam_srcu);
3435	list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list,
3436	srcu_read_lock_held(&mpam_srcu)) {
3437	msc = vmsc->msc;
3438
3439	guard(mutex)(&msc->cfg_lock);
3440	list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list,
3441	srcu_read_lock_held(&mpam_srcu)) {
3442	arg.ris = ris;
3443	ret = mpam_touch_msc(msc, __write_config, &arg);
3444	if (ret)
3445	return ret;
3446	ris->in_reset_state = false;
3447	}
3448	}
3449
3450	return 0;
3451	}
3452
3453	static int __init mpam_msc_driver_init(void)
3454	{
3455	if (!system_supports_mpam())
3456	return -EOPNOTSUPP;
3457
3458	init_srcu_struct(&mpam_srcu);
3459
3460	fw_num_msc = acpi_mpam_count_msc();
3461	if (fw_num_msc <= 0) {
3462	pr_err("No MSC devices found in firmware\n");
3463	return -EINVAL;
3464	}
3465
3466	return platform_driver_register(&mpam_msc_driver);
3467	}
3468
3469	/* Must occur after arm64_mpam_register_cpus() from arch_initcall() */
3470	subsys_initcall(mpam_msc_driver_init);
3471
3472	#ifdef CONFIG_MPAM_KUNIT_TEST
3473	#include "test_mpam_devices.c"
3474	#endif