projects/linux·blob master · refs

1987 lines · 54.1 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Resource Director Technology(RDT)
4 * - Monitoring code
5 *
6 * Copyright (C) 2017 Intel Corporation
7 *
8 * Author:
9 * Vikas Shivappa <[email protected]>
10 *
11 * This replaces the cqm.c based on perf but we reuse a lot of
12 * code and datastructures originally from Peter Zijlstra and Matt Fleming.
13 *
14 * More information about RDT be found in the Intel (R) x86 Architecture
15 * Software Developer Manual June 2016, volume 3, section 17.17.
16 */
17
18#define pr_fmt(fmt) "resctrl: " fmt
19
20#include <linux/cpu.h>
21#include <linux/resctrl.h>
22#include <linux/sizes.h>
23#include <linux/slab.h>
24
25#include "internal.h"
26
27#define CREATE_TRACE_POINTS
28
29#include "monitor_trace.h"
30
31/**
32 * struct rmid_entry - dirty tracking for all RMID.
33 * @closid: The CLOSID for this entry.
34 * @rmid: The RMID for this entry.
35 * @busy: The number of domains with cached data using this RMID.
36 * @list: Member of the rmid_free_lru list when busy == 0.
37 *
38 * Depending on the architecture the correct monitor is accessed using
39 * both @closid and @rmid, or @rmid only.
40 *
41 * Take the rdtgroup_mutex when accessing.
42 */
43struct rmid_entry {
44 u32 closid;
45 u32 rmid;
46 int busy;
47 struct list_head list;
48};
49
50/*
51 * @rmid_free_lru - A least recently used list of free RMIDs
52 * These RMIDs are guaranteed to have an occupancy less than the
53 * threshold occupancy
54 */
55static LIST_HEAD(rmid_free_lru);
56
57/*
58 * @closid_num_dirty_rmid The number of dirty RMID each CLOSID has.
59 * Only allocated when CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID is defined.
60 * Indexed by CLOSID. Protected by rdtgroup_mutex.
61 */
62static u32 *closid_num_dirty_rmid;
63
64/*
65 * @rmid_limbo_count - count of currently unused but (potentially)
66 * dirty RMIDs.
67 * This counts RMIDs that no one is currently using but that
68 * may have a occupancy value > resctrl_rmid_realloc_threshold. User can
69 * change the threshold occupancy value.
70 */
71static unsigned int rmid_limbo_count;
72
73/*
74 * @rmid_entry - The entry in the limbo and free lists.
75 */
76static struct rmid_entry *rmid_ptrs;
77
78/*
79 * This is the threshold cache occupancy in bytes at which we will consider an
80 * RMID available for re-allocation.
81 */
82unsigned int resctrl_rmid_realloc_threshold;
83
84/*
85 * This is the maximum value for the reallocation threshold, in bytes.
86 */
87unsigned int resctrl_rmid_realloc_limit;
88
89/*
90 * x86 and arm64 differ in their handling of monitoring.
91 * x86's RMID are independent numbers, there is only one source of traffic
92 * with an RMID value of '1'.
93 * arm64's PMG extends the PARTID/CLOSID space, there are multiple sources of
94 * traffic with a PMG value of '1', one for each CLOSID, meaning the RMID
95 * value is no longer unique.
96 * To account for this, resctrl uses an index. On x86 this is just the RMID,
97 * on arm64 it encodes the CLOSID and RMID. This gives a unique number.
98 *
99 * The domain's rmid_busy_llc and rmid_ptrs[] are sized by index. The arch code
100 * must accept an attempt to read every index.
101 */
102static inline struct rmid_entry *__rmid_entry(u32 idx)
103{
104 struct rmid_entry *entry;
105 u32 closid, rmid;
106
107 entry = &rmid_ptrs[idx];
108 resctrl_arch_rmid_idx_decode(idx, &closid, &rmid);
109
110 WARN_ON_ONCE(entry->closid != closid);
111 WARN_ON_ONCE(entry->rmid != rmid);
112
113 return entry;
114}
115
116static void limbo_release_entry(struct rmid_entry *entry)
117{
118 lockdep_assert_held(&rdtgroup_mutex);
119
120 rmid_limbo_count--;
121 list_add_tail(&entry->list, &rmid_free_lru);
122
123 if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID))
124 closid_num_dirty_rmid[entry->closid]--;
125}
126
127/*
128 * Check the RMIDs that are marked as busy for this domain. If the
129 * reported LLC occupancy is below the threshold clear the busy bit and
130 * decrement the count. If the busy count gets to zero on an RMID, we
131 * free the RMID
132 */
133void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free)
134{
135 struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3);
136 u32 idx_limit = resctrl_arch_system_num_rmid_idx();
137 struct rmid_entry *entry;
138 bool rmid_dirty = true;
139 u32 idx, cur_idx = 1;
140 void *arch_mon_ctx;
141 void *arch_priv;
142 u64 val = 0;
143
144 arch_priv = mon_event_all[QOS_L3_OCCUP_EVENT_ID].arch_priv;
145 arch_mon_ctx = resctrl_arch_mon_ctx_alloc(r, QOS_L3_OCCUP_EVENT_ID);
146 if (IS_ERR(arch_mon_ctx)) {
147 pr_warn_ratelimited("Failed to allocate monitor context: %pe",
148 arch_mon_ctx);
149 return;
150 }
151
152 /*
153 * Skip RMID 0 and start from RMID 1 and check all the RMIDs that
154 * are marked as busy for occupancy < threshold. If the occupancy
155 * is less than the threshold decrement the busy counter of the
156 * RMID and move it to the free list when the counter reaches 0.
157 */
158 for (;;) {
159 idx = find_next_bit(d->rmid_busy_llc, idx_limit, cur_idx);
160 if (idx >= idx_limit)
161 break;
162
163 entry = __rmid_entry(idx);
164 if (!force_free) {
165 if (resctrl_arch_rmid_read(r, &d->hdr, entry->closid,
166 entry->rmid, QOS_L3_OCCUP_EVENT_ID,
167 arch_priv, &val, arch_mon_ctx)) {
168 rmid_dirty = true;
169 } else {
170 rmid_dirty = (val >= resctrl_rmid_realloc_threshold);
171
172 /*
173 * x86's CLOSID and RMID are independent numbers,
174 * so the entry's CLOSID is an empty CLOSID
175 * (X86_RESCTRL_EMPTY_CLOSID). On Arm the RMID
176 * (PMG) extends the CLOSID (PARTID) space with
177 * bits that aren't used to select the configuration.
178 * It is thus necessary to track both CLOSID and
179 * RMID because there may be dependencies between
180 * them on some architectures.
181 */
182 trace_mon_llc_occupancy_limbo(entry->closid, entry->rmid,
183 d->hdr.id, val);
184 }
185 }
186
187 if (force_free || !rmid_dirty) {
188 clear_bit(idx, d->rmid_busy_llc);
189 if (!--entry->busy)
190 limbo_release_entry(entry);
191 }
192 cur_idx = idx + 1;
193 }
194
195 resctrl_arch_mon_ctx_free(r, QOS_L3_OCCUP_EVENT_ID, arch_mon_ctx);
196}
197
198bool has_busy_rmid(struct rdt_l3_mon_domain *d)
199{
200 u32 idx_limit = resctrl_arch_system_num_rmid_idx();
201
202 return find_first_bit(d->rmid_busy_llc, idx_limit) != idx_limit;
203}
204
205static struct rmid_entry *resctrl_find_free_rmid(u32 closid)
206{
207 struct rmid_entry *itr;
208 u32 itr_idx, cmp_idx;
209
210 if (list_empty(&rmid_free_lru))
211 return rmid_limbo_count ? ERR_PTR(-EBUSY) : ERR_PTR(-ENOSPC);
212
213 list_for_each_entry(itr, &rmid_free_lru, list) {
214 /*
215 * Get the index of this free RMID, and the index it would need
216 * to be if it were used with this CLOSID.
217 * If the CLOSID is irrelevant on this architecture, the two
218 * index values are always the same on every entry and thus the
219 * very first entry will be returned.
220 */
221 itr_idx = resctrl_arch_rmid_idx_encode(itr->closid, itr->rmid);
222 cmp_idx = resctrl_arch_rmid_idx_encode(closid, itr->rmid);
223
224 if (itr_idx == cmp_idx)
225 return itr;
226 }
227
228 return ERR_PTR(-ENOSPC);
229}
230
231/**
232 * resctrl_find_cleanest_closid() - Find a CLOSID where all the associated
233 * RMID are clean, or the CLOSID that has
234 * the most clean RMID.
235 *
236 * MPAM's equivalent of RMID are per-CLOSID, meaning a freshly allocated CLOSID
237 * may not be able to allocate clean RMID. To avoid this the allocator will
238 * choose the CLOSID with the most clean RMID.
239 *
240 * When the CLOSID and RMID are independent numbers, the first free CLOSID will
241 * be returned.
242 *
243 * Return: Free CLOSID on success, < 0 on failure.
244 */
245int resctrl_find_cleanest_closid(void)
246{
247 u32 cleanest_closid = ~0;
248 int i = 0;
249
250 lockdep_assert_held(&rdtgroup_mutex);
251
252 if (!IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID))
253 return -EIO;
254
255 for (i = 0; i < closids_supported(); i++) {
256 int num_dirty;
257
258 if (closid_allocated(i))
259 continue;
260
261 num_dirty = closid_num_dirty_rmid[i];
262 if (num_dirty == 0)
263 return i;
264
265 if (cleanest_closid == ~0)
266 cleanest_closid = i;
267
268 if (num_dirty < closid_num_dirty_rmid[cleanest_closid])
269 cleanest_closid = i;
270 }
271
272 if (cleanest_closid == ~0)
273 return -ENOSPC;
274
275 return cleanest_closid;
276}
277
278/*
279 * For MPAM the RMID value is not unique, and has to be considered with
280 * the CLOSID. The (CLOSID, RMID) pair is allocated on all domains, which
281 * allows all domains to be managed by a single free list.
282 * Each domain also has a rmid_busy_llc to reduce the work of the limbo handler.
283 */
284int alloc_rmid(u32 closid)
285{
286 struct rmid_entry *entry;
287
288 lockdep_assert_held(&rdtgroup_mutex);
289
290 entry = resctrl_find_free_rmid(closid);
291 if (IS_ERR(entry))
292 return PTR_ERR(entry);
293
294 list_del(&entry->list);
295 return entry->rmid;
296}
297
298static void add_rmid_to_limbo(struct rmid_entry *entry)
299{
300 struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3);
301 struct rdt_l3_mon_domain *d;
302 u32 idx;
303
304 lockdep_assert_held(&rdtgroup_mutex);
305
306 /* Walking r->domains, ensure it can't race with cpuhp */
307 lockdep_assert_cpus_held();
308
309 idx = resctrl_arch_rmid_idx_encode(entry->closid, entry->rmid);
310
311 entry->busy = 0;
312 list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
313 /*
314 * For the first limbo RMID in the domain,
315 * setup up the limbo worker.
316 */
317 if (!has_busy_rmid(d))
318 cqm_setup_limbo_handler(d, CQM_LIMBOCHECK_INTERVAL,
319 RESCTRL_PICK_ANY_CPU);
320 set_bit(idx, d->rmid_busy_llc);
321 entry->busy++;
322 }
323
324 rmid_limbo_count++;
325 if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID))
326 closid_num_dirty_rmid[entry->closid]++;
327}
328
329void free_rmid(u32 closid, u32 rmid)
330{
331 u32 idx = resctrl_arch_rmid_idx_encode(closid, rmid);
332 struct rmid_entry *entry;
333
334 lockdep_assert_held(&rdtgroup_mutex);
335
336 /*
337 * Do not allow the default rmid to be free'd. Comparing by index
338 * allows architectures that ignore the closid parameter to avoid an
339 * unnecessary check.
340 */
341 if (!resctrl_arch_mon_capable() ||
342 idx == resctrl_arch_rmid_idx_encode(RESCTRL_RESERVED_CLOSID,
343 RESCTRL_RESERVED_RMID))
344 return;
345
346 entry = __rmid_entry(idx);
347
348 if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID))
349 add_rmid_to_limbo(entry);
350 else
351 list_add_tail(&entry->list, &rmid_free_lru);
352}
353
354static struct mbm_state *get_mbm_state(struct rdt_l3_mon_domain *d, u32 closid,
355 u32 rmid, enum resctrl_event_id evtid)
356{
357 u32 idx = resctrl_arch_rmid_idx_encode(closid, rmid);
358 struct mbm_state *state;
359
360 if (!resctrl_is_mbm_event(evtid))
361 return NULL;
362
363 state = d->mbm_states[MBM_STATE_IDX(evtid)];
364
365 return state ? &state[idx] : NULL;
366}
367
368/*
369 * mbm_cntr_get() - Return the counter ID for the matching @evtid and @rdtgrp.
370 *
371 * Return:
372 * Valid counter ID on success, or -ENOENT on failure.
373 */
374static int mbm_cntr_get(struct rdt_resource *r, struct rdt_l3_mon_domain *d,
375 struct rdtgroup *rdtgrp, enum resctrl_event_id evtid)
376{
377 int cntr_id;
378
379 if (!r->mon.mbm_cntr_assignable)
380 return -ENOENT;
381
382 if (!resctrl_is_mbm_event(evtid))
383 return -ENOENT;
384
385 for (cntr_id = 0; cntr_id < r->mon.num_mbm_cntrs; cntr_id++) {
386 if (d->cntr_cfg[cntr_id].rdtgrp == rdtgrp &&
387 d->cntr_cfg[cntr_id].evtid == evtid)
388 return cntr_id;
389 }
390
391 return -ENOENT;
392}
393
394/*
395 * mbm_cntr_alloc() - Initialize and return a new counter ID in the domain @d.
396 * Caller must ensure that the specified event is not assigned already.
397 *
398 * Return:
399 * Valid counter ID on success, or -ENOSPC on failure.
400 */
401static int mbm_cntr_alloc(struct rdt_resource *r, struct rdt_l3_mon_domain *d,
402 struct rdtgroup *rdtgrp, enum resctrl_event_id evtid)
403{
404 int cntr_id;
405
406 for (cntr_id = 0; cntr_id < r->mon.num_mbm_cntrs; cntr_id++) {
407 if (!d->cntr_cfg[cntr_id].rdtgrp) {
408 d->cntr_cfg[cntr_id].rdtgrp = rdtgrp;
409 d->cntr_cfg[cntr_id].evtid = evtid;
410 return cntr_id;
411 }
412 }
413
414 return -ENOSPC;
415}
416
417/*
418 * mbm_cntr_free() - Clear the counter ID configuration details in the domain @d.
419 */
420static void mbm_cntr_free(struct rdt_l3_mon_domain *d, int cntr_id)
421{
422 memset(&d->cntr_cfg[cntr_id], 0, sizeof(*d->cntr_cfg));
423}
424
425static int __l3_mon_event_count(struct rdtgroup *rdtgrp, struct rmid_read *rr)
426{
427 int cpu = smp_processor_id();
428 u32 closid = rdtgrp->closid;
429 u32 rmid = rdtgrp->mon.rmid;
430 struct rdt_l3_mon_domain *d;
431 int cntr_id = -ENOENT;
432 struct mbm_state *m;
433 u64 tval = 0;
434
435 if (!domain_header_is_valid(rr->hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_L3)) {
436 rr->err = -EIO;
437 return -EINVAL;
438 }
439 d = container_of(rr->hdr, struct rdt_l3_mon_domain, hdr);
440
441 if (rr->is_mbm_cntr) {
442 cntr_id = mbm_cntr_get(rr->r, d, rdtgrp, rr->evt->evtid);
443 if (cntr_id < 0) {
444 rr->err = -ENOENT;
445 return -EINVAL;
446 }
447 }
448
449 if (rr->first) {
450 if (rr->is_mbm_cntr)
451 resctrl_arch_reset_cntr(rr->r, d, closid, rmid, cntr_id, rr->evt->evtid);
452 else
453 resctrl_arch_reset_rmid(rr->r, d, closid, rmid, rr->evt->evtid);
454 m = get_mbm_state(d, closid, rmid, rr->evt->evtid);
455 if (m)
456 memset(m, 0, sizeof(struct mbm_state));
457 return 0;
458 }
459
460 /* Reading a single domain, must be on a CPU in that domain. */
461 if (!cpumask_test_cpu(cpu, &d->hdr.cpu_mask)) {
462 rr->err = -EIO;
463 return -EINVAL;
464 }
465 if (rr->is_mbm_cntr)
466 rr->err = resctrl_arch_cntr_read(rr->r, d, closid, rmid, cntr_id,
467 rr->evt->evtid, &tval);
468 else
469 rr->err = resctrl_arch_rmid_read(rr->r, rr->hdr, closid, rmid,
470 rr->evt->evtid, rr->evt->arch_priv,
471 &tval, rr->arch_mon_ctx);
472 if (rr->err)
473 return rr->err;
474
475 rr->val += tval;
476
477 return 0;
478}
479
480static int __l3_mon_event_count_sum(struct rdtgroup *rdtgrp, struct rmid_read *rr)
481{
482 int cpu = smp_processor_id();
483 u32 closid = rdtgrp->closid;
484 u32 rmid = rdtgrp->mon.rmid;
485 struct rdt_l3_mon_domain *d;
486 u64 tval = 0;
487 int err, ret;
488
489 /*
490 * Summing across domains is only done for systems that implement
491 * Sub-NUMA Cluster. There is no overlap with systems that support
492 * assignable counters.
493 */
494 if (rr->is_mbm_cntr) {
495 pr_warn_once("Summing domains using assignable counters is not supported\n");
496 rr->err = -EINVAL;
497 return -EINVAL;
498 }
499
500 /* Summing domains that share a cache, must be on a CPU for that cache. */
501 if (!cpumask_test_cpu(cpu, &rr->ci->shared_cpu_map)) {
502 rr->err = -EIO;
503 return -EINVAL;
504 }
505
506 /*
507 * Legacy files must report the sum of an event across all
508 * domains that share the same L3 cache instance.
509 * Report success if a read from any domain succeeds, -EINVAL
510 * (translated to "Unavailable" for user space) if reading from
511 * all domains fail for any reason.
512 */
513 ret = -EINVAL;
514 /*
515 * RCU list being traversed with CPU hotplug lock held. lockdep
516 * unable to help prove this here since this work is scheduled via
517 * smp_call*(). Not called from MBM overflow handler.
518 */
519 list_for_each_entry(d, &rr->r->mon_domains, hdr.list) {
520 if (d->ci_id != rr->ci->id)
521 continue;
522 err = resctrl_arch_rmid_read(rr->r, &d->hdr, closid, rmid,
523 rr->evt->evtid, rr->evt->arch_priv,
524 &tval, rr->arch_mon_ctx);
525 if (!err) {
526 rr->val += tval;
527 ret = 0;
528 }
529 }
530
531 if (ret)
532 rr->err = ret;
533
534 return ret;
535}
536
537static int __mon_event_count(struct rdtgroup *rdtgrp, struct rmid_read *rr)
538{
539 switch (rr->r->rid) {
540 case RDT_RESOURCE_L3:
541 WARN_ON_ONCE(rr->evt->any_cpu);
542 if (rr->hdr)
543 return __l3_mon_event_count(rdtgrp, rr);
544 else
545 return __l3_mon_event_count_sum(rdtgrp, rr);
546 case RDT_RESOURCE_PERF_PKG: {
547 u64 tval = 0;
548
549 rr->err = resctrl_arch_rmid_read(rr->r, rr->hdr, rdtgrp->closid,
550 rdtgrp->mon.rmid, rr->evt->evtid,
551 rr->evt->arch_priv,
552 &tval, rr->arch_mon_ctx);
553 if (rr->err)
554 return rr->err;
555
556 rr->val += tval;
557
558 return 0;
559 }
560 default:
561 rr->err = -EINVAL;
562 return -EINVAL;
563 }
564}
565
566/*
567 * mbm_bw_count() - Update bw count from values previously read by
568 * __mon_event_count().
569 * @rdtgrp: resctrl group associated with the CLOSID and RMID to identify
570 * the cached mbm_state.
571 * @rr: The struct rmid_read populated by __mon_event_count().
572 *
573 * Supporting function to calculate the memory bandwidth
574 * and delta bandwidth in MBps. The chunks value previously read by
575 * __mon_event_count() is compared with the chunks value from the previous
576 * invocation. This must be called once per second to maintain values in MBps.
577 */
578static void mbm_bw_count(struct rdtgroup *rdtgrp, struct rmid_read *rr)
579{
580 u64 cur_bw, bytes, cur_bytes;
581 u32 closid = rdtgrp->closid;
582 u32 rmid = rdtgrp->mon.rmid;
583 struct rdt_l3_mon_domain *d;
584 struct mbm_state *m;
585
586 if (!domain_header_is_valid(rr->hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_L3))
587 return;
588 d = container_of(rr->hdr, struct rdt_l3_mon_domain, hdr);
589 m = get_mbm_state(d, closid, rmid, rr->evt->evtid);
590 if (WARN_ON_ONCE(!m))
591 return;
592
593 cur_bytes = rr->val;
594 bytes = cur_bytes - m->prev_bw_bytes;
595 m->prev_bw_bytes = cur_bytes;
596
597 cur_bw = bytes / SZ_1M;
598
599 m->prev_bw = cur_bw;
600}
601
602/*
603 * This is scheduled by mon_event_read() to read the CQM/MBM counters
604 * on a domain.
605 */
606void mon_event_count(void *info)
607{
608 struct rdtgroup *rdtgrp, *entry;
609 struct rmid_read *rr = info;
610 struct list_head *head;
611 int ret;
612
613 rdtgrp = rr->rgrp;
614
615 ret = __mon_event_count(rdtgrp, rr);
616
617 /*
618 * For Ctrl groups read data from child monitor groups and
619 * add them together. Count events which are read successfully.
620 * Discard the rmid_read's reporting errors.
621 */
622 head = &rdtgrp->mon.crdtgrp_list;
623
624 if (rdtgrp->type == RDTCTRL_GROUP) {
625 list_for_each_entry(entry, head, mon.crdtgrp_list) {
626 if (__mon_event_count(entry, rr) == 0)
627 ret = 0;
628 }
629 }
630
631 /*
632 * __mon_event_count() calls for newly created monitor groups may
633 * report -EINVAL/Unavailable if the monitor hasn't seen any traffic.
634 * Discard error if any of the monitor event reads succeeded.
635 */
636 if (ret == 0)
637 rr->err = 0;
638}
639
640/*
641 * Find the software controller's ctrl domain that contains @cpu on resource @r.
642 *
643 * Only called from the mbm_over worker via update_mba_bw() where the returned
644 * domain is kept alive by cancel_delayed_work_sync() in
645 * resctrl_offline_ctrl_domain(). This drains this worker and then waits on
646 * rdtgroup_mutex held here before the architecture can free the ctrl domain.
647 *
648 * Context: Call from RCU read-side critical section.
649 */
650static struct rdt_ctrl_domain *get_sc_ctrl_domain_from_cpu(int cpu,
651 struct rdt_resource *r)
652{
653 struct rdt_ctrl_domain *d;
654
655 list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list) {
656 /* Find the domain that contains this CPU */
657 if (cpumask_test_cpu(cpu, &d->hdr.cpu_mask))
658 return d;
659 }
660
661 return NULL;
662}
663
664/*
665 * Feedback loop for MBA software controller (mba_sc)
666 *
667 * mba_sc is a feedback loop where we periodically read MBM counters and
668 * adjust the bandwidth percentage values via the IA32_MBA_THRTL_MSRs so
669 * that:
670 *
671 * current bandwidth(cur_bw) < user specified bandwidth(user_bw)
672 *
673 * This uses the MBM counters to measure the bandwidth and MBA throttle
674 * MSRs to control the bandwidth for a particular rdtgrp. It builds on the
675 * fact that resctrl rdtgroups have both monitoring and control.
676 *
677 * The frequency of the checks is 1s and we just tag along the MBM overflow
678 * timer. Having 1s interval makes the calculation of bandwidth simpler.
679 *
680 * Although MBA's goal is to restrict the bandwidth to a maximum, there may
681 * be a need to increase the bandwidth to avoid unnecessarily restricting
682 * the L2 <-> L3 traffic.
683 *
684 * Since MBA controls the L2 external bandwidth where as MBM measures the
685 * L3 external bandwidth the following sequence could lead to such a
686 * situation.
687 *
688 * Consider an rdtgroup which had high L3 <-> memory traffic in initial
689 * phases -> mba_sc kicks in and reduced bandwidth percentage values -> but
690 * after some time rdtgroup has mostly L2 <-> L3 traffic.
691 *
692 * In this case we may restrict the rdtgroup's L2 <-> L3 traffic as its
693 * throttle MSRs already have low percentage values. To avoid
694 * unnecessarily restricting such rdtgroups, we also increase the bandwidth.
695 */
696static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_l3_mon_domain *dom_mbm)
697{
698 u32 closid, rmid, cur_msr_val, new_msr_val;
699 struct mbm_state *pmbm_data, *cmbm_data;
700 struct rdt_ctrl_domain *dom_mba;
701 enum resctrl_event_id evt_id;
702 struct rdt_resource *r_mba;
703 struct list_head *head;
704 struct rdtgroup *entry;
705 u32 cur_bw, user_bw;
706
707 r_mba = resctrl_arch_get_resource(RDT_RESOURCE_MBA);
708 evt_id = rgrp->mba_mbps_event;
709
710 closid = rgrp->closid;
711 rmid = rgrp->mon.rmid;
712 pmbm_data = get_mbm_state(dom_mbm, closid, rmid, evt_id);
713 if (WARN_ON_ONCE(!pmbm_data))
714 return;
715
716 guard(rcu)();
717 dom_mba = get_sc_ctrl_domain_from_cpu(smp_processor_id(), r_mba);
718 if (!dom_mba) {
719 pr_warn_once("Failure to get domain for MBA update\n");
720 return;
721 }
722
723 cur_bw = pmbm_data->prev_bw;
724 user_bw = dom_mba->mbps_val[closid];
725
726 /* MBA resource doesn't support CDP */
727 cur_msr_val = resctrl_arch_get_config(r_mba, dom_mba, closid, CDP_NONE);
728
729 /*
730 * For Ctrl groups read data from child monitor groups.
731 */
732 head = &rgrp->mon.crdtgrp_list;
733 list_for_each_entry(entry, head, mon.crdtgrp_list) {
734 cmbm_data = get_mbm_state(dom_mbm, entry->closid, entry->mon.rmid, evt_id);
735 if (WARN_ON_ONCE(!cmbm_data))
736 return;
737 cur_bw += cmbm_data->prev_bw;
738 }
739
740 /*
741 * Scale up/down the bandwidth linearly for the ctrl group. The
742 * bandwidth step is the bandwidth granularity specified by the
743 * hardware.
744 * Always increase throttling if current bandwidth is above the
745 * target set by user.
746 * But avoid thrashing up and down on every poll by checking
747 * whether a decrease in throttling is likely to push the group
748 * back over target. E.g. if currently throttling to 30% of bandwidth
749 * on a system with 10% granularity steps, check whether moving to
750 * 40% would go past the limit by multiplying current bandwidth by
751 * "(30 + 10) / 30".
752 */
753 if (cur_msr_val > r_mba->membw.min_bw && user_bw < cur_bw) {
754 new_msr_val = cur_msr_val - r_mba->membw.bw_gran;
755 } else if (cur_msr_val < MAX_MBA_BW &&
756 (user_bw > (cur_bw * (cur_msr_val + r_mba->membw.min_bw) / cur_msr_val))) {
757 new_msr_val = cur_msr_val + r_mba->membw.bw_gran;
758 } else {
759 return;
760 }
761
762 resctrl_arch_update_one(r_mba, dom_mba, closid, CDP_NONE, new_msr_val);
763}
764
765static void mbm_update_one_event(struct rdt_resource *r, struct rdt_l3_mon_domain *d,
766 struct rdtgroup *rdtgrp, enum resctrl_event_id evtid)
767{
768 struct rmid_read rr = {0};
769
770 rr.r = r;
771 rr.hdr = &d->hdr;
772 rr.evt = &mon_event_all[evtid];
773 if (resctrl_arch_mbm_cntr_assign_enabled(r)) {
774 rr.is_mbm_cntr = true;
775 } else {
776 rr.arch_mon_ctx = resctrl_arch_mon_ctx_alloc(rr.r, evtid);
777 if (IS_ERR(rr.arch_mon_ctx)) {
778 pr_warn_ratelimited("Failed to allocate monitor context: %pe",
779 rr.arch_mon_ctx);
780 return;
781 }
782 }
783
784 __mon_event_count(rdtgrp, &rr);
785
786 /*
787 * If the software controller is enabled, compute the
788 * bandwidth for this event id.
789 */
790 if (is_mba_sc(NULL))
791 mbm_bw_count(rdtgrp, &rr);
792
793 if (rr.arch_mon_ctx)
794 resctrl_arch_mon_ctx_free(rr.r, evtid, rr.arch_mon_ctx);
795}
796
797static void mbm_update(struct rdt_resource *r, struct rdt_l3_mon_domain *d,
798 struct rdtgroup *rdtgrp)
799{
800 /*
801 * This is protected from concurrent reads from user as both
802 * the user and overflow handler hold the global mutex.
803 */
804 if (resctrl_is_mon_event_enabled(QOS_L3_MBM_TOTAL_EVENT_ID))
805 mbm_update_one_event(r, d, rdtgrp, QOS_L3_MBM_TOTAL_EVENT_ID);
806
807 if (resctrl_is_mon_event_enabled(QOS_L3_MBM_LOCAL_EVENT_ID))
808 mbm_update_one_event(r, d, rdtgrp, QOS_L3_MBM_LOCAL_EVENT_ID);
809}
810
811/*
812 * Handler to scan the limbo list and move the RMIDs
813 * to free list whose occupancy < threshold_occupancy.
814 */
815void cqm_handle_limbo(struct work_struct *work)
816{
817 unsigned long delay = msecs_to_jiffies(CQM_LIMBOCHECK_INTERVAL);
818 struct rdt_l3_mon_domain *d;
819
820 /*
821 * Safe to run without CPU hotplug lock. Work is guaranteed to be
822 * canceled before the domain structure is removed.
823 */
824 mutex_lock(&rdtgroup_mutex);
825
826 /*
827 * Ensure the worker is dedicated to a CPU as intended and not
828 * relocated by workqueue subsystem as part of CPU going offline.
829 */
830 if (!is_percpu_thread())
831 goto out_unlock;
832
833 d = container_of(work, struct rdt_l3_mon_domain, cqm_limbo.work);
834
835 /* Domain is going offline */
836 if (cpumask_empty(&d->hdr.cpu_mask))
837 goto out_unlock;
838
839 __check_limbo(d, false);
840
841 if (has_busy_rmid(d)) {
842 d->cqm_work_cpu = cpumask_any_housekeeping(&d->hdr.cpu_mask,
843 RESCTRL_PICK_ANY_CPU);
844 schedule_delayed_work_on(d->cqm_work_cpu, &d->cqm_limbo,
845 delay);
846 }
847
848out_unlock:
849 mutex_unlock(&rdtgroup_mutex);
850}
851
852/**
853 * cqm_setup_limbo_handler() - Schedule the limbo handler to run for this
854 * domain.
855 * @dom: The domain the limbo handler should run for.
856 * @delay_ms: How far in the future the handler should run.
857 * @exclude_cpu: Which CPU the handler should not run on,
858 * RESCTRL_PICK_ANY_CPU to pick any CPU.
859 */
860void cqm_setup_limbo_handler(struct rdt_l3_mon_domain *dom, unsigned long delay_ms,
861 int exclude_cpu)
862{
863 unsigned long delay = msecs_to_jiffies(delay_ms);
864 int cpu;
865
866 cpu = cpumask_any_housekeeping(&dom->hdr.cpu_mask, exclude_cpu);
867 dom->cqm_work_cpu = cpu;
868
869 if (cpu < nr_cpu_ids)
870 schedule_delayed_work_on(cpu, &dom->cqm_limbo, delay);
871}
872
873void mbm_handle_overflow(struct work_struct *work)
874{
875 unsigned long delay = msecs_to_jiffies(MBM_OVERFLOW_INTERVAL);
876 struct rdtgroup *prgrp, *crgrp;
877 struct rdt_l3_mon_domain *d;
878 struct list_head *head;
879 struct rdt_resource *r;
880
881 /*
882 * Safe to run without CPU hotplug lock. Work is guaranteed to be
883 * canceled before the domain structure is removed.
884 */
885 mutex_lock(&rdtgroup_mutex);
886
887 /*
888 * If the filesystem has been unmounted this work no longer needs to
889 * run.
890 */
891 if (!resctrl_mounted || !resctrl_arch_mon_capable())
892 goto out_unlock;
893
894 /*
895 * Ensure the worker is dedicated to a CPU and not relocated by
896 * workqueue subsystem as part of CPU going offline since reading
897 * events depend on smp_processor_id(). After passing this check
898 * smp_processor_id() is valid for entire duration of this worker
899 * since it runs with rdtgroup_mutex held and the offline handler needs
900 * rdtgroup_mutex to offline the CPU being run on here.
901 */
902 if (!is_percpu_thread())
903 goto out_unlock;
904
905 r = resctrl_arch_get_resource(RDT_RESOURCE_L3);
906 d = container_of(work, struct rdt_l3_mon_domain, mbm_over.work);
907
908 /* Domain is going offline */
909 if (cpumask_empty(&d->hdr.cpu_mask))
910 goto out_unlock;
911
912 list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) {
913 mbm_update(r, d, prgrp);
914
915 head = &prgrp->mon.crdtgrp_list;
916 list_for_each_entry(crgrp, head, mon.crdtgrp_list)
917 mbm_update(r, d, crgrp);
918
919 if (is_mba_sc(NULL))
920 update_mba_bw(prgrp, d);
921 }
922
923 /*
924 * Re-check for housekeeping CPUs. This allows the overflow handler to
925 * move off a nohz_full CPU quickly.
926 */
927 d->mbm_work_cpu = cpumask_any_housekeeping(&d->hdr.cpu_mask,
928 RESCTRL_PICK_ANY_CPU);
929 schedule_delayed_work_on(d->mbm_work_cpu, &d->mbm_over, delay);
930
931out_unlock:
932 mutex_unlock(&rdtgroup_mutex);
933}
934
935/**
936 * mbm_setup_overflow_handler() - Schedule the overflow handler to run for this
937 * domain.
938 * @dom: The domain the overflow handler should run for.
939 * @delay_ms: How far in the future the handler should run.
940 * @exclude_cpu: Which CPU the handler should not run on,
941 * RESCTRL_PICK_ANY_CPU to pick any CPU.
942 */
943void mbm_setup_overflow_handler(struct rdt_l3_mon_domain *dom, unsigned long delay_ms,
944 int exclude_cpu)
945{
946 unsigned long delay = msecs_to_jiffies(delay_ms);
947 int cpu;
948
949 /*
950 * When a domain comes online there is no guarantee the filesystem is
951 * mounted. If not, there is no need to catch counter overflow.
952 */
953 if (!resctrl_mounted || !resctrl_arch_mon_capable())
954 return;
955 cpu = cpumask_any_housekeeping(&dom->hdr.cpu_mask, exclude_cpu);
956 dom->mbm_work_cpu = cpu;
957
958 if (cpu < nr_cpu_ids)
959 schedule_delayed_work_on(cpu, &dom->mbm_over, delay);
960}
961
962int setup_rmid_lru_list(void)
963{
964 struct rmid_entry *entry = NULL;
965 u32 idx_limit;
966 u32 idx;
967 int i;
968
969 if (!resctrl_arch_mon_capable())
970 return 0;
971
972 /*
973 * Called on every mount, but the number of RMIDs cannot change
974 * after the first mount, so keep using the same set of rmid_ptrs[]
975 * until resctrl_exit(). Note that the limbo handler continues to
976 * access rmid_ptrs[] after resctrl is unmounted.
977 */
978 if (rmid_ptrs)
979 return 0;
980
981 idx_limit = resctrl_arch_system_num_rmid_idx();
982 rmid_ptrs = kzalloc_objs(struct rmid_entry, idx_limit);
983 if (!rmid_ptrs)
984 return -ENOMEM;
985
986 for (i = 0; i < idx_limit; i++) {
987 entry = &rmid_ptrs[i];
988 INIT_LIST_HEAD(&entry->list);
989
990 resctrl_arch_rmid_idx_decode(i, &entry->closid, &entry->rmid);
991 list_add_tail(&entry->list, &rmid_free_lru);
992 }
993
994 /*
995 * RESCTRL_RESERVED_CLOSID and RESCTRL_RESERVED_RMID are special and
996 * are always allocated. These are used for the rdtgroup_default
997 * control group, which was setup earlier in rdtgroup_setup_default().
998 */
999 idx = resctrl_arch_rmid_idx_encode(RESCTRL_RESERVED_CLOSID,
1000 RESCTRL_RESERVED_RMID);
1001 entry = __rmid_entry(idx);
1002 list_del(&entry->list);
1003
1004 return 0;
1005}
1006
1007void free_rmid_lru_list(void)
1008{
1009 if (!resctrl_arch_mon_capable())
1010 return;
1011
1012 mutex_lock(&rdtgroup_mutex);
1013 kfree(rmid_ptrs);
1014 rmid_ptrs = NULL;
1015 mutex_unlock(&rdtgroup_mutex);
1016}
1017
1018#define MON_EVENT(_eventid, _name, _res, _fp) \
1019 [_eventid] = { \
1020 .name = _name, \
1021 .evtid = _eventid, \
1022 .rid = _res, \
1023 .is_floating_point = _fp, \
1024}
1025
1026/*
1027 * All available events. Architecture code marks the ones that
1028 * are supported by a system using resctrl_enable_mon_event()
1029 * to set .enabled.
1030 */
1031struct mon_evt mon_event_all[QOS_NUM_EVENTS] = {
1032 MON_EVENT(QOS_L3_OCCUP_EVENT_ID, "llc_occupancy", RDT_RESOURCE_L3, false),
1033 MON_EVENT(QOS_L3_MBM_TOTAL_EVENT_ID, "mbm_total_bytes", RDT_RESOURCE_L3, false),
1034 MON_EVENT(QOS_L3_MBM_LOCAL_EVENT_ID, "mbm_local_bytes", RDT_RESOURCE_L3, false),
1035 MON_EVENT(PMT_EVENT_ENERGY, "core_energy", RDT_RESOURCE_PERF_PKG, true),
1036 MON_EVENT(PMT_EVENT_ACTIVITY, "activity", RDT_RESOURCE_PERF_PKG, true),
1037 MON_EVENT(PMT_EVENT_STALLS_LLC_HIT, "stalls_llc_hit", RDT_RESOURCE_PERF_PKG, false),
1038 MON_EVENT(PMT_EVENT_C1_RES, "c1_res", RDT_RESOURCE_PERF_PKG, false),
1039 MON_EVENT(PMT_EVENT_UNHALTED_CORE_CYCLES, "unhalted_core_cycles", RDT_RESOURCE_PERF_PKG, false),
1040 MON_EVENT(PMT_EVENT_STALLS_LLC_MISS, "stalls_llc_miss", RDT_RESOURCE_PERF_PKG, false),
1041 MON_EVENT(PMT_EVENT_AUTO_C6_RES, "c6_res", RDT_RESOURCE_PERF_PKG, false),
1042 MON_EVENT(PMT_EVENT_UNHALTED_REF_CYCLES, "unhalted_ref_cycles", RDT_RESOURCE_PERF_PKG, false),
1043 MON_EVENT(PMT_EVENT_UOPS_RETIRED, "uops_retired", RDT_RESOURCE_PERF_PKG, false),
1044 MON_EVENT(PMT_EVENT_INST_RETIRED, "inst_retired", RDT_RESOURCE_PERF_PKG, false),
1045 MON_EVENT(PMT_EVENT_PCNT, "pcnt", RDT_RESOURCE_PERF_PKG, false),
1046};
1047
1048bool resctrl_enable_mon_event(enum resctrl_event_id eventid, bool any_cpu,
1049 unsigned int binary_bits, void *arch_priv)
1050{
1051 if (WARN_ON_ONCE(eventid < QOS_FIRST_EVENT || eventid >= QOS_NUM_EVENTS ||
1052 binary_bits > MAX_BINARY_BITS))
1053 return false;
1054 if (mon_event_all[eventid].enabled) {
1055 pr_warn("Duplicate enable for event %d\n", eventid);
1056 return false;
1057 }
1058 if (binary_bits && !mon_event_all[eventid].is_floating_point) {
1059 pr_warn("Event %d may not be floating point\n", eventid);
1060 return false;
1061 }
1062
1063 mon_event_all[eventid].any_cpu = any_cpu;
1064 mon_event_all[eventid].binary_bits = binary_bits;
1065 mon_event_all[eventid].arch_priv = arch_priv;
1066 mon_event_all[eventid].enabled = true;
1067
1068 return true;
1069}
1070
1071bool resctrl_is_mon_event_enabled(enum resctrl_event_id eventid)
1072{
1073 return eventid >= QOS_FIRST_EVENT && eventid < QOS_NUM_EVENTS &&
1074 mon_event_all[eventid].enabled;
1075}
1076
1077u32 resctrl_get_mon_evt_cfg(enum resctrl_event_id evtid)
1078{
1079 return mon_event_all[evtid].evt_cfg;
1080}
1081
1082/**
1083 * struct mbm_transaction - Memory transaction an MBM event can be configured with.
1084 * @name: Name of memory transaction (read, write ...).
1085 * @val: The bit (eg. READS_TO_LOCAL_MEM or READS_TO_REMOTE_MEM) used to
1086 * represent the memory transaction within an event's configuration.
1087 */
1088struct mbm_transaction {
1089 char name[32];
1090 u32 val;
1091};
1092
1093/* Decoded values for each type of memory transaction. */
1094static struct mbm_transaction mbm_transactions[NUM_MBM_TRANSACTIONS] = {
1095 {"local_reads", READS_TO_LOCAL_MEM},
1096 {"remote_reads", READS_TO_REMOTE_MEM},
1097 {"local_non_temporal_writes", NON_TEMP_WRITE_TO_LOCAL_MEM},
1098 {"remote_non_temporal_writes", NON_TEMP_WRITE_TO_REMOTE_MEM},
1099 {"local_reads_slow_memory", READS_TO_LOCAL_S_MEM},
1100 {"remote_reads_slow_memory", READS_TO_REMOTE_S_MEM},
1101 {"dirty_victim_writes_all", DIRTY_VICTIMS_TO_ALL_MEM},
1102};
1103
1104int event_filter_show(struct kernfs_open_file *of, struct seq_file *seq, void *v)
1105{
1106 struct mon_evt *mevt = rdt_kn_parent_priv(of->kn);
1107 struct rdt_resource *r;
1108 bool sep = false;
1109 int ret = 0, i;
1110
1111 if (!info_kn_lock(of->kn))
1112 return -ENOENT;
1113 rdt_last_cmd_clear();
1114
1115 r = resctrl_arch_get_resource(mevt->rid);
1116 if (!resctrl_arch_mbm_cntr_assign_enabled(r)) {
1117 rdt_last_cmd_puts("mbm_event counter assignment mode is not enabled\n");
1118 ret = -EINVAL;
1119 goto out_unlock;
1120 }
1121
1122 for (i = 0; i < NUM_MBM_TRANSACTIONS; i++) {
1123 if (mevt->evt_cfg & mbm_transactions[i].val) {
1124 if (sep)
1125 seq_putc(seq, ',');
1126 seq_printf(seq, "%s", mbm_transactions[i].name);
1127 sep = true;
1128 }
1129 }
1130 seq_putc(seq, '\n');
1131
1132out_unlock:
1133 info_kn_unlock(of->kn);
1134
1135 return ret;
1136}
1137
1138int resctrl_mbm_assign_on_mkdir_show(struct kernfs_open_file *of, struct seq_file *s,
1139 void *v)
1140{
1141 struct rdt_resource *r = rdt_kn_parent_priv(of->kn);
1142 int ret = 0;
1143
1144 if (!info_kn_lock(of->kn))
1145 return -ENOENT;
1146 rdt_last_cmd_clear();
1147
1148 if (!resctrl_arch_mbm_cntr_assign_enabled(r)) {
1149 rdt_last_cmd_puts("mbm_event counter assignment mode is not enabled\n");
1150 ret = -EINVAL;
1151 goto out_unlock;
1152 }
1153
1154 seq_printf(s, "%u\n", r->mon.mbm_assign_on_mkdir);
1155
1156out_unlock:
1157 info_kn_unlock(of->kn);
1158
1159 return ret;
1160}
1161
1162ssize_t resctrl_mbm_assign_on_mkdir_write(struct kernfs_open_file *of, char *buf,
1163 size_t nbytes, loff_t off)
1164{
1165 struct rdt_resource *r = rdt_kn_parent_priv(of->kn);
1166 bool value;
1167 int ret;
1168
1169 if (!info_kn_lock(of->kn))
1170 return -ENOENT;
1171 rdt_last_cmd_clear();
1172
1173 ret = kstrtobool(buf, &value);
1174 if (ret) {
1175 rdt_last_cmd_puts("mbm_assign_on_mkdir: Invalid input\n");
1176 goto out_unlock;
1177 }
1178
1179 if (!resctrl_arch_mbm_cntr_assign_enabled(r)) {
1180 rdt_last_cmd_puts("mbm_event counter assignment mode is not enabled\n");
1181 ret = -EINVAL;
1182 goto out_unlock;
1183 }
1184
1185 r->mon.mbm_assign_on_mkdir = value;
1186
1187out_unlock:
1188 info_kn_unlock(of->kn);
1189
1190 return ret ?: nbytes;
1191}
1192
1193/*
1194 * mbm_cntr_free_all() - Clear all the counter ID configuration details in the
1195 * domain @d. Called when mbm_assign_mode is changed.
1196 */
1197static void mbm_cntr_free_all(struct rdt_resource *r, struct rdt_l3_mon_domain *d)
1198{
1199 memset(d->cntr_cfg, 0, sizeof(*d->cntr_cfg) * r->mon.num_mbm_cntrs);
1200}
1201
1202/*
1203 * resctrl_reset_rmid_all() - Reset all non-architecture states for all the
1204 * supported RMIDs.
1205 */
1206static void resctrl_reset_rmid_all(struct rdt_resource *r, struct rdt_l3_mon_domain *d)
1207{
1208 u32 idx_limit = resctrl_arch_system_num_rmid_idx();
1209 enum resctrl_event_id evt;
1210 int idx;
1211
1212 for_each_mbm_event_id(evt) {
1213 if (!resctrl_is_mon_event_enabled(evt))
1214 continue;
1215 idx = MBM_STATE_IDX(evt);
1216 memset(d->mbm_states[idx], 0, sizeof(*d->mbm_states[0]) * idx_limit);
1217 }
1218}
1219
1220/*
1221 * rdtgroup_assign_cntr() - Assign/unassign the counter ID for the event, RMID
1222 * pair in the domain.
1223 *
1224 * Assign the counter if @assign is true else unassign the counter. Reset the
1225 * associated non-architectural state.
1226 */
1227static void rdtgroup_assign_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d,
1228 enum resctrl_event_id evtid, u32 rmid, u32 closid,
1229 u32 cntr_id, bool assign)
1230{
1231 struct mbm_state *m;
1232
1233 resctrl_arch_config_cntr(r, d, evtid, rmid, closid, cntr_id, assign);
1234
1235 m = get_mbm_state(d, closid, rmid, evtid);
1236 if (m)
1237 memset(m, 0, sizeof(*m));
1238}
1239
1240/*
1241 * rdtgroup_alloc_assign_cntr() - Allocate a counter ID and assign it to the event
1242 * pointed to by @mevt and the resctrl group @rdtgrp within the domain @d.
1243 *
1244 * Return:
1245 * 0 on success, < 0 on failure.
1246 */
1247static int rdtgroup_alloc_assign_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d,
1248 struct rdtgroup *rdtgrp, struct mon_evt *mevt)
1249{
1250 int cntr_id;
1251
1252 /* No action required if the counter is assigned already. */
1253 cntr_id = mbm_cntr_get(r, d, rdtgrp, mevt->evtid);
1254 if (cntr_id >= 0)
1255 return 0;
1256
1257 cntr_id = mbm_cntr_alloc(r, d, rdtgrp, mevt->evtid);
1258 if (cntr_id < 0) {
1259 rdt_last_cmd_printf("Failed to allocate counter for %s in domain %d\n",
1260 mevt->name, d->hdr.id);
1261 return cntr_id;
1262 }
1263
1264 rdtgroup_assign_cntr(r, d, mevt->evtid, rdtgrp->mon.rmid, rdtgrp->closid, cntr_id, true);
1265
1266 return 0;
1267}
1268
1269/*
1270 * rdtgroup_assign_cntr_event() - Assign a hardware counter for the event in
1271 * @mevt to the resctrl group @rdtgrp. Assign counters to all domains if @d is
1272 * NULL; otherwise, assign the counter to the specified domain @d.
1273 *
1274 * If all counters in a domain are already in use, rdtgroup_alloc_assign_cntr()
1275 * will fail. When attempting to assign counters to all domains, carry on trying
1276 * to assign counters after a failure since only some domains may have counters
1277 * and the goal is to assign counters where possible. If any counter assignment
1278 * fails, return the error from the last failing assignment.
1279 *
1280 * Return:
1281 * 0 on success, < 0 on failure.
1282 */
1283static int rdtgroup_assign_cntr_event(struct rdt_l3_mon_domain *d, struct rdtgroup *rdtgrp,
1284 struct mon_evt *mevt)
1285{
1286 struct rdt_resource *r = resctrl_arch_get_resource(mevt->rid);
1287 int ret = 0;
1288
1289 if (!d) {
1290 list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
1291 int err;
1292
1293 err = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt);
1294 if (err)
1295 ret = err;
1296 }
1297 } else {
1298 ret = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt);
1299 }
1300
1301 return ret;
1302}
1303
1304/*
1305 * rdtgroup_assign_cntrs() - Assign counters to MBM events. Called when
1306 * a new group is created.
1307 *
1308 * Each group can accommodate two counters per domain: one for the total
1309 * event and one for the local event. Assignments may fail due to the limited
1310 * number of counters. However, it is not necessary to fail the group creation
1311 * and thus no failure is returned. Users have the option to modify the
1312 * counter assignments after the group has been created.
1313 */
1314void rdtgroup_assign_cntrs(struct rdtgroup *rdtgrp)
1315{
1316 struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3);
1317
1318 if (!r->mon_capable || !resctrl_arch_mbm_cntr_assign_enabled(r) ||
1319 !r->mon.mbm_assign_on_mkdir)
1320 return;
1321
1322 if (resctrl_is_mon_event_enabled(QOS_L3_MBM_TOTAL_EVENT_ID))
1323 rdtgroup_assign_cntr_event(NULL, rdtgrp,
1324 &mon_event_all[QOS_L3_MBM_TOTAL_EVENT_ID]);
1325
1326 if (resctrl_is_mon_event_enabled(QOS_L3_MBM_LOCAL_EVENT_ID))
1327 rdtgroup_assign_cntr_event(NULL, rdtgrp,
1328 &mon_event_all[QOS_L3_MBM_LOCAL_EVENT_ID]);
1329}
1330
1331/*
1332 * rdtgroup_free_unassign_cntr() - Unassign and reset the counter ID configuration
1333 * for the event pointed to by @mevt within the domain @d and resctrl group @rdtgrp.
1334 */
1335static void rdtgroup_free_unassign_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d,
1336 struct rdtgroup *rdtgrp, struct mon_evt *mevt)
1337{
1338 int cntr_id;
1339
1340 cntr_id = mbm_cntr_get(r, d, rdtgrp, mevt->evtid);
1341
1342 /* If there is no cntr_id assigned, nothing to do */
1343 if (cntr_id < 0)
1344 return;
1345
1346 rdtgroup_assign_cntr(r, d, mevt->evtid, rdtgrp->mon.rmid, rdtgrp->closid, cntr_id, false);
1347
1348 mbm_cntr_free(d, cntr_id);
1349}
1350
1351/*
1352 * rdtgroup_unassign_cntr_event() - Unassign a hardware counter associated with
1353 * the event structure @mevt from the domain @d and the group @rdtgrp. Unassign
1354 * the counters from all the domains if @d is NULL else unassign from @d.
1355 */
1356static void rdtgroup_unassign_cntr_event(struct rdt_l3_mon_domain *d, struct rdtgroup *rdtgrp,
1357 struct mon_evt *mevt)
1358{
1359 struct rdt_resource *r = resctrl_arch_get_resource(mevt->rid);
1360
1361 if (!d) {
1362 list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held())
1363 rdtgroup_free_unassign_cntr(r, d, rdtgrp, mevt);
1364 } else {
1365 rdtgroup_free_unassign_cntr(r, d, rdtgrp, mevt);
1366 }
1367}
1368
1369/*
1370 * rdtgroup_unassign_cntrs() - Unassign the counters associated with MBM events.
1371 * Called when a group is deleted.
1372 */
1373void rdtgroup_unassign_cntrs(struct rdtgroup *rdtgrp)
1374{
1375 struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3);
1376
1377 if (!r->mon_capable || !resctrl_arch_mbm_cntr_assign_enabled(r))
1378 return;
1379
1380 if (resctrl_is_mon_event_enabled(QOS_L3_MBM_TOTAL_EVENT_ID))
1381 rdtgroup_unassign_cntr_event(NULL, rdtgrp,
1382 &mon_event_all[QOS_L3_MBM_TOTAL_EVENT_ID]);
1383
1384 if (resctrl_is_mon_event_enabled(QOS_L3_MBM_LOCAL_EVENT_ID))
1385 rdtgroup_unassign_cntr_event(NULL, rdtgrp,
1386 &mon_event_all[QOS_L3_MBM_LOCAL_EVENT_ID]);
1387}
1388
1389static int resctrl_parse_mem_transactions(char *tok, u32 *val)
1390{
1391 u32 temp_val = 0;
1392 char *evt_str;
1393 bool found;
1394 int i;
1395
1396next_config:
1397 if (!tok || tok[0] == '\0') {
1398 *val = temp_val;
1399 return 0;
1400 }
1401
1402 /* Start processing the strings for each memory transaction type */
1403 evt_str = strim(strsep(&tok, ","));
1404 found = false;
1405 for (i = 0; i < NUM_MBM_TRANSACTIONS; i++) {
1406 if (!strcmp(mbm_transactions[i].name, evt_str)) {
1407 temp_val |= mbm_transactions[i].val;
1408 found = true;
1409 break;
1410 }
1411 }
1412
1413 if (!found) {
1414 rdt_last_cmd_printf("Invalid memory transaction type %s\n", evt_str);
1415 return -EINVAL;
1416 }
1417
1418 goto next_config;
1419}
1420
1421/*
1422 * rdtgroup_update_cntr_event - Update the counter assignments for the event
1423 * in a group.
1424 * @r: Resource to which update needs to be done.
1425 * @rdtgrp: Resctrl group.
1426 * @evtid: MBM monitor event.
1427 */
1428static void rdtgroup_update_cntr_event(struct rdt_resource *r, struct rdtgroup *rdtgrp,
1429 enum resctrl_event_id evtid)
1430{
1431 struct rdt_l3_mon_domain *d;
1432 int cntr_id;
1433
1434 list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
1435 cntr_id = mbm_cntr_get(r, d, rdtgrp, evtid);
1436 if (cntr_id >= 0)
1437 rdtgroup_assign_cntr(r, d, evtid, rdtgrp->mon.rmid,
1438 rdtgrp->closid, cntr_id, true);
1439 }
1440}
1441
1442/*
1443 * resctrl_update_cntr_allrdtgrp - Update the counter assignments for the event
1444 * for all the groups.
1445 * @mevt MBM Monitor event.
1446 */
1447static void resctrl_update_cntr_allrdtgrp(struct mon_evt *mevt)
1448{
1449 struct rdt_resource *r = resctrl_arch_get_resource(mevt->rid);
1450 struct rdtgroup *prgrp, *crgrp;
1451
1452 /*
1453 * Find all the groups where the event is assigned and update the
1454 * configuration of existing assignments.
1455 */
1456 list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) {
1457 rdtgroup_update_cntr_event(r, prgrp, mevt->evtid);
1458
1459 list_for_each_entry(crgrp, &prgrp->mon.crdtgrp_list, mon.crdtgrp_list)
1460 rdtgroup_update_cntr_event(r, crgrp, mevt->evtid);
1461 }
1462}
1463
1464ssize_t event_filter_write(struct kernfs_open_file *of, char *buf, size_t nbytes,
1465 loff_t off)
1466{
1467 struct mon_evt *mevt = rdt_kn_parent_priv(of->kn);
1468 struct rdt_resource *r;
1469 u32 evt_cfg = 0;
1470 int ret = 0;
1471
1472 if (!info_kn_lock(of->kn))
1473 return -ENOENT;
1474
1475 rdt_last_cmd_clear();
1476
1477 /* Valid input requires a trailing newline */
1478 if (nbytes == 0 || buf[nbytes - 1] != '\n') {
1479 rdt_last_cmd_puts("event_filter: Invalid input\n");
1480 ret = -EINVAL;
1481 goto out_unlock;
1482 }
1483
1484 buf[nbytes - 1] = '\0';
1485
1486 r = resctrl_arch_get_resource(mevt->rid);
1487 if (!resctrl_arch_mbm_cntr_assign_enabled(r)) {
1488 rdt_last_cmd_puts("mbm_event counter assignment mode is not enabled\n");
1489 ret = -EINVAL;
1490 goto out_unlock;
1491 }
1492 if (!r->mon.mbm_cntr_configurable) {
1493 rdt_last_cmd_puts("event_filter is not configurable\n");
1494 ret = -EPERM;
1495 goto out_unlock;
1496 }
1497
1498 ret = resctrl_parse_mem_transactions(buf, &evt_cfg);
1499 if (!ret && mevt->evt_cfg != evt_cfg) {
1500 mevt->evt_cfg = evt_cfg;
1501 resctrl_update_cntr_allrdtgrp(mevt);
1502 }
1503
1504out_unlock:
1505 info_kn_unlock(of->kn);
1506
1507 return ret ?: nbytes;
1508}
1509
1510int resctrl_mbm_assign_mode_show(struct kernfs_open_file *of,
1511 struct seq_file *s, void *v)
1512{
1513 struct rdt_resource *r = rdt_kn_parent_priv(of->kn);
1514 bool enabled;
1515
1516 if (!info_kn_lock(of->kn))
1517 return -ENOENT;
1518 enabled = resctrl_arch_mbm_cntr_assign_enabled(r);
1519
1520 if (r->mon.mbm_cntr_assignable) {
1521 if (enabled)
1522 seq_puts(s, "[mbm_event]\n");
1523 else
1524 seq_puts(s, "[default]\n");
1525
1526 if (!r->mon.mbm_cntr_assign_fixed) {
1527 if (enabled)
1528 seq_puts(s, "default\n");
1529 else
1530 seq_puts(s, "mbm_event\n");
1531 }
1532 } else {
1533 seq_puts(s, "[default]\n");
1534 }
1535
1536 info_kn_unlock(of->kn);
1537
1538 return 0;
1539}
1540
1541ssize_t resctrl_mbm_assign_mode_write(struct kernfs_open_file *of, char *buf,
1542 size_t nbytes, loff_t off)
1543{
1544 struct rdt_resource *r = rdt_kn_parent_priv(of->kn);
1545 struct rdt_l3_mon_domain *d;
1546 int ret = 0;
1547 bool enable;
1548
1549 if (!info_kn_lock(of->kn))
1550 return -ENOENT;
1551
1552 rdt_last_cmd_clear();
1553
1554 /* Valid input requires a trailing newline */
1555 if (nbytes == 0 || buf[nbytes - 1] != '\n') {
1556 rdt_last_cmd_puts("mbm_assign_mode: Invalid input\n");
1557 ret = -EINVAL;
1558 goto out_unlock;
1559 }
1560
1561 buf[nbytes - 1] = '\0';
1562
1563 if (!strcmp(buf, "default")) {
1564 enable = 0;
1565 } else if (!strcmp(buf, "mbm_event")) {
1566 if (r->mon.mbm_cntr_assignable) {
1567 enable = 1;
1568 } else {
1569 ret = -EINVAL;
1570 rdt_last_cmd_puts("mbm_event mode is not supported\n");
1571 goto out_unlock;
1572 }
1573 } else {
1574 ret = -EINVAL;
1575 rdt_last_cmd_puts("Unsupported assign mode\n");
1576 goto out_unlock;
1577 }
1578
1579 if (enable != resctrl_arch_mbm_cntr_assign_enabled(r)) {
1580 if (r->mon.mbm_cntr_assign_fixed) {
1581 ret = -EINVAL;
1582 rdt_last_cmd_puts("Counter assignment mode is not configurable\n");
1583 goto out_unlock;
1584 }
1585
1586 ret = resctrl_arch_mbm_cntr_assign_set(r, enable);
1587 if (ret)
1588 goto out_unlock;
1589
1590 /* Update the visibility of BMEC related files */
1591 resctrl_bmec_files_show(r, NULL, !enable);
1592
1593 /*
1594 * Initialize the default memory transaction values for
1595 * total and local events.
1596 */
1597 if (resctrl_is_mon_event_enabled(QOS_L3_MBM_TOTAL_EVENT_ID))
1598 mon_event_all[QOS_L3_MBM_TOTAL_EVENT_ID].evt_cfg = r->mon.mbm_cfg_mask;
1599 if (resctrl_is_mon_event_enabled(QOS_L3_MBM_LOCAL_EVENT_ID))
1600 mon_event_all[QOS_L3_MBM_LOCAL_EVENT_ID].evt_cfg = r->mon.mbm_cfg_mask &
1601 (READS_TO_LOCAL_MEM |
1602 READS_TO_LOCAL_S_MEM |
1603 NON_TEMP_WRITE_TO_LOCAL_MEM);
1604 /* Enable auto assignment when switching to "mbm_event" mode */
1605 if (enable)
1606 r->mon.mbm_assign_on_mkdir = true;
1607 /*
1608 * Reset all the non-achitectural RMID state and assignable counters.
1609 */
1610 list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
1611 mbm_cntr_free_all(r, d);
1612 resctrl_reset_rmid_all(r, d);
1613 }
1614 }
1615
1616out_unlock:
1617 info_kn_unlock(of->kn);
1618
1619 return ret ?: nbytes;
1620}
1621
1622int resctrl_num_mbm_cntrs_show(struct kernfs_open_file *of,
1623 struct seq_file *s, void *v)
1624{
1625 struct rdt_resource *r = rdt_kn_parent_priv(of->kn);
1626 struct rdt_l3_mon_domain *dom;
1627 bool sep = false;
1628
1629 if (!info_kn_lock(of->kn))
1630 return -ENOENT;
1631
1632 list_for_each_entry_rcu(dom, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
1633 if (sep)
1634 seq_putc(s, ';');
1635
1636 seq_printf(s, "%d=%d", dom->hdr.id, r->mon.num_mbm_cntrs);
1637 sep = true;
1638 }
1639 seq_putc(s, '\n');
1640
1641 info_kn_unlock(of->kn);
1642 return 0;
1643}
1644
1645int resctrl_available_mbm_cntrs_show(struct kernfs_open_file *of,
1646 struct seq_file *s, void *v)
1647{
1648 struct rdt_resource *r = rdt_kn_parent_priv(of->kn);
1649 struct rdt_l3_mon_domain *dom;
1650 bool sep = false;
1651 u32 cntrs, i;
1652 int ret = 0;
1653
1654 if (!info_kn_lock(of->kn))
1655 return -ENOENT;
1656
1657 rdt_last_cmd_clear();
1658
1659 if (!resctrl_arch_mbm_cntr_assign_enabled(r)) {
1660 rdt_last_cmd_puts("mbm_event counter assignment mode is not enabled\n");
1661 ret = -EINVAL;
1662 goto out_unlock;
1663 }
1664
1665 list_for_each_entry_rcu(dom, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
1666 if (sep)
1667 seq_putc(s, ';');
1668
1669 cntrs = 0;
1670 for (i = 0; i < r->mon.num_mbm_cntrs; i++) {
1671 if (!dom->cntr_cfg[i].rdtgrp)
1672 cntrs++;
1673 }
1674
1675 seq_printf(s, "%d=%u", dom->hdr.id, cntrs);
1676 sep = true;
1677 }
1678 seq_putc(s, '\n');
1679
1680out_unlock:
1681 info_kn_unlock(of->kn);
1682
1683 return ret;
1684}
1685
1686int mbm_L3_assignments_show(struct kernfs_open_file *of, struct seq_file *s, void *v)
1687{
1688 struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3);
1689 struct rdt_l3_mon_domain *d;
1690 struct rdtgroup *rdtgrp;
1691 struct mon_evt *mevt;
1692 int ret = 0;
1693 bool sep;
1694
1695 rdtgrp = rdtgroup_kn_lock_live(of->kn);
1696 if (!rdtgrp) {
1697 ret = -ENOENT;
1698 goto out_unlock;
1699 }
1700
1701 if (!resctrl_arch_mbm_cntr_assign_enabled(r)) {
1702 rdt_last_cmd_puts("mbm_event counter assignment mode is not enabled\n");
1703 ret = -EINVAL;
1704 goto out_unlock;
1705 }
1706
1707 for_each_mon_event(mevt) {
1708 if (mevt->rid != r->rid || !mevt->enabled || !resctrl_is_mbm_event(mevt->evtid))
1709 continue;
1710
1711 sep = false;
1712 seq_printf(s, "%s:", mevt->name);
1713 list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
1714 if (sep)
1715 seq_putc(s, ';');
1716
1717 if (mbm_cntr_get(r, d, rdtgrp, mevt->evtid) < 0)
1718 seq_printf(s, "%d=_", d->hdr.id);
1719 else
1720 seq_printf(s, "%d=e", d->hdr.id);
1721
1722 sep = true;
1723 }
1724 seq_putc(s, '\n');
1725 }
1726
1727out_unlock:
1728 rdtgroup_kn_unlock(of->kn);
1729
1730 return ret;
1731}
1732
1733/*
1734 * mbm_get_mon_event_by_name() - Return the mon_evt entry for the matching
1735 * event name.
1736 */
1737static struct mon_evt *mbm_get_mon_event_by_name(struct rdt_resource *r, char *name)
1738{
1739 struct mon_evt *mevt;
1740
1741 for_each_mon_event(mevt) {
1742 if (mevt->rid == r->rid && mevt->enabled &&
1743 resctrl_is_mbm_event(mevt->evtid) &&
1744 !strcmp(mevt->name, name))
1745 return mevt;
1746 }
1747
1748 return NULL;
1749}
1750
1751static int rdtgroup_modify_assign_state(char *assign, struct rdt_l3_mon_domain *d,
1752 struct rdtgroup *rdtgrp, struct mon_evt *mevt)
1753{
1754 int ret = 0;
1755
1756 if (!assign || strlen(assign) != 1)
1757 return -EINVAL;
1758
1759 switch (*assign) {
1760 case 'e':
1761 ret = rdtgroup_assign_cntr_event(d, rdtgrp, mevt);
1762 break;
1763 case '_':
1764 rdtgroup_unassign_cntr_event(d, rdtgrp, mevt);
1765 break;
1766 default:
1767 ret = -EINVAL;
1768 break;
1769 }
1770
1771 return ret;
1772}
1773
1774static int resctrl_parse_mbm_assignment(struct rdt_resource *r, struct rdtgroup *rdtgrp,
1775 char *event, char *tok)
1776{
1777 struct rdt_l3_mon_domain *d;
1778 unsigned long dom_id = 0;
1779 char *dom_str, *id_str;
1780 struct mon_evt *mevt;
1781 int ret;
1782
1783 mevt = mbm_get_mon_event_by_name(r, event);
1784 if (!mevt) {
1785 rdt_last_cmd_printf("Invalid event %s\n", event);
1786 return -ENOENT;
1787 }
1788
1789next:
1790 if (!tok || tok[0] == '\0')
1791 return 0;
1792
1793 /* Start processing the strings for each domain */
1794 dom_str = strim(strsep(&tok, ";"));
1795
1796 id_str = strsep(&dom_str, "=");
1797
1798 /* Check for domain id '*' which means all domains */
1799 if (id_str && *id_str == '*') {
1800 ret = rdtgroup_modify_assign_state(dom_str, NULL, rdtgrp, mevt);
1801 if (ret)
1802 rdt_last_cmd_printf("Assign operation '%s:*=%s' failed\n",
1803 event, dom_str);
1804 return ret;
1805 } else if (!id_str || kstrtoul(id_str, 10, &dom_id)) {
1806 rdt_last_cmd_puts("Missing domain id\n");
1807 return -EINVAL;
1808 }
1809
1810 /* Verify if the dom_id is valid */
1811 list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
1812 if (d->hdr.id == dom_id) {
1813 ret = rdtgroup_modify_assign_state(dom_str, d, rdtgrp, mevt);
1814 if (ret) {
1815 rdt_last_cmd_printf("Assign operation '%s:%ld=%s' failed\n",
1816 event, dom_id, dom_str);
1817 return ret;
1818 }
1819 goto next;
1820 }
1821 }
1822
1823 rdt_last_cmd_printf("Invalid domain id %ld\n", dom_id);
1824 return -EINVAL;
1825}
1826
1827ssize_t mbm_L3_assignments_write(struct kernfs_open_file *of, char *buf,
1828 size_t nbytes, loff_t off)
1829{
1830 struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3);
1831 struct rdtgroup *rdtgrp;
1832 char *token, *event;
1833 int ret = 0;
1834
1835 rdtgrp = rdtgroup_kn_lock_live(of->kn);
1836 if (!rdtgrp) {
1837 rdtgroup_kn_unlock(of->kn);
1838 return -ENOENT;
1839 }
1840
1841 /* Valid input requires a trailing newline */
1842 if (nbytes == 0 || buf[nbytes - 1] != '\n') {
1843 rdt_last_cmd_puts("mbm_L3_assignments: Invalid input\n");
1844 ret = -EINVAL;
1845 goto out_unlock;
1846 }
1847
1848 buf[nbytes - 1] = '\0';
1849
1850 if (!resctrl_arch_mbm_cntr_assign_enabled(r)) {
1851 rdt_last_cmd_puts("mbm_event mode is not enabled\n");
1852 ret = -EINVAL;
1853 goto out_unlock;
1854 }
1855
1856 while ((token = strsep(&buf, "\n")) != NULL) {
1857 /*
1858 * The write command follows the following format:
1859 * "<Event>:<Domain ID>=<Assignment state>"
1860 * Extract the event name first.
1861 */
1862 event = strsep(&token, ":");
1863
1864 ret = resctrl_parse_mbm_assignment(r, rdtgrp, event, token);
1865 if (ret)
1866 break;
1867 }
1868
1869out_unlock:
1870 rdtgroup_kn_unlock(of->kn);
1871
1872 return ret ?: nbytes;
1873}
1874
1875static int closid_num_dirty_rmid_alloc(struct rdt_resource *r)
1876{
1877 if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) {
1878 u32 num_closid = resctrl_arch_get_num_closid(r);
1879 u32 *tmp;
1880
1881 /* For ARM memory ordering access to closid_num_dirty_rmid */
1882 mutex_lock(&rdtgroup_mutex);
1883
1884 /*
1885 * If the architecture hasn't provided a sanitised value here,
1886 * this may result in larger arrays than necessary. Resctrl will
1887 * use a smaller system wide value based on the resources in
1888 * use.
1889 */
1890 tmp = kcalloc(num_closid, sizeof(*tmp), GFP_KERNEL);
1891 if (!tmp) {
1892 mutex_unlock(&rdtgroup_mutex);
1893 return -ENOMEM;
1894 }
1895
1896 closid_num_dirty_rmid = tmp;
1897
1898 mutex_unlock(&rdtgroup_mutex);
1899 }
1900
1901 return 0;
1902}
1903
1904static void closid_num_dirty_rmid_free(void)
1905{
1906 if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) {
1907 mutex_lock(&rdtgroup_mutex);
1908 kfree(closid_num_dirty_rmid);
1909 closid_num_dirty_rmid = NULL;
1910 mutex_unlock(&rdtgroup_mutex);
1911 }
1912}
1913
1914/**
1915 * resctrl_l3_mon_resource_init() - Initialise global monitoring structures.
1916 *
1917 * Allocate and initialise global monitor resources that do not belong to a
1918 * specific domain. i.e. the closid_num_dirty_rmid[] used to find the CLOSID
1919 * with the cleanest set of RMIDs.
1920 * Called once during boot after the struct rdt_resource's have been configured
1921 * but before the filesystem is mounted.
1922 * Resctrl's cpuhp callbacks may be called before this point to bring a domain
1923 * online.
1924 *
1925 * Return: 0 for success, or -ENOMEM.
1926 */
1927int resctrl_l3_mon_resource_init(void)
1928{
1929 struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3);
1930 int ret;
1931
1932 if (!r->mon_capable)
1933 return 0;
1934
1935 ret = closid_num_dirty_rmid_alloc(r);
1936 if (ret)
1937 return ret;
1938
1939 if (resctrl_arch_is_evt_configurable(QOS_L3_MBM_TOTAL_EVENT_ID)) {
1940 mon_event_all[QOS_L3_MBM_TOTAL_EVENT_ID].configurable = true;
1941 resctrl_file_fflags_init("mbm_total_bytes_config",
1942 RFTYPE_MON_INFO | RFTYPE_RES_CACHE);
1943 }
1944 if (resctrl_arch_is_evt_configurable(QOS_L3_MBM_LOCAL_EVENT_ID)) {
1945 mon_event_all[QOS_L3_MBM_LOCAL_EVENT_ID].configurable = true;
1946 resctrl_file_fflags_init("mbm_local_bytes_config",
1947 RFTYPE_MON_INFO | RFTYPE_RES_CACHE);
1948 }
1949
1950 if (resctrl_is_mon_event_enabled(QOS_L3_MBM_LOCAL_EVENT_ID))
1951 mba_mbps_default_event = QOS_L3_MBM_LOCAL_EVENT_ID;
1952 else if (resctrl_is_mon_event_enabled(QOS_L3_MBM_TOTAL_EVENT_ID))
1953 mba_mbps_default_event = QOS_L3_MBM_TOTAL_EVENT_ID;
1954
1955 if (r->mon.mbm_cntr_assignable) {
1956 if (resctrl_is_mon_event_enabled(QOS_L3_MBM_TOTAL_EVENT_ID))
1957 mon_event_all[QOS_L3_MBM_TOTAL_EVENT_ID].evt_cfg = r->mon.mbm_cfg_mask;
1958 if (resctrl_is_mon_event_enabled(QOS_L3_MBM_LOCAL_EVENT_ID))
1959 mon_event_all[QOS_L3_MBM_LOCAL_EVENT_ID].evt_cfg = r->mon.mbm_cfg_mask &
1960 (READS_TO_LOCAL_MEM |
1961 READS_TO_LOCAL_S_MEM |
1962 NON_TEMP_WRITE_TO_LOCAL_MEM);
1963 r->mon.mbm_assign_on_mkdir = true;
1964 resctrl_file_fflags_init("num_mbm_cntrs",
1965 RFTYPE_MON_INFO | RFTYPE_RES_CACHE);
1966 resctrl_file_fflags_init("available_mbm_cntrs",
1967 RFTYPE_MON_INFO | RFTYPE_RES_CACHE);
1968 resctrl_file_fflags_init("event_filter", RFTYPE_ASSIGN_CONFIG);
1969 if (r->mon.mbm_cntr_configurable)
1970 resctrl_file_mode_init("event_filter", 0644);
1971 resctrl_file_fflags_init("mbm_assign_on_mkdir", RFTYPE_MON_INFO |
1972 RFTYPE_RES_CACHE);
1973 resctrl_file_fflags_init("mbm_L3_assignments", RFTYPE_MON_BASE);
1974 }
1975
1976 return 0;
1977}
1978
1979void resctrl_l3_mon_resource_exit(void)
1980{
1981 struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3);
1982
1983 if (!r->mon_capable)
1984 return;
1985
1986 closid_num_dirty_rmid_free();
1987}