projects/linux·blob master · refs

1094 lines · 27.3 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * cacheinfo support - processor cache information via sysfs
4 *
5 * Based on arch/x86/kernel/cpu/intel_cacheinfo.c
6 * Author: Sudeep Holla <[email protected]>
7 */
8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
10#include <linux/acpi.h>
11#include <linux/bitfield.h>
12#include <linux/bitops.h>
13#include <linux/cacheinfo.h>
14#include <linux/compiler.h>
15#include <linux/cpu.h>
16#include <linux/device.h>
17#include <linux/init.h>
18#include <linux/of.h>
19#include <linux/sched.h>
20#include <linux/sched/topology.h>
21#include <linux/slab.h>
22#include <linux/smp.h>
23#include <linux/sysfs.h>
24
25/* pointer to per cpu cacheinfo */
26static DEFINE_PER_CPU(struct cpu_cacheinfo, ci_cpu_cacheinfo);
27#define ci_cacheinfo(cpu) (&per_cpu(ci_cpu_cacheinfo, cpu))
28#define cache_leaves(cpu) (ci_cacheinfo(cpu)->num_leaves)
29#define per_cpu_cacheinfo(cpu) (ci_cacheinfo(cpu)->info_list)
30#define per_cpu_cacheinfo_idx(cpu, idx) \
31 (per_cpu_cacheinfo(cpu) + (idx))
32
33/* Set if no cache information is found in DT/ACPI. */
34static bool use_arch_info;
35
36struct cpu_cacheinfo *get_cpu_cacheinfo(unsigned int cpu)
37{
38 return ci_cacheinfo(cpu);
39}
40
41/**
42 * get_cpu_cacheinfo_id - Return the cache ID for a CPU and cache level
43 * @cpu: CPU number
44 * @level: Cache level
45 *
46 * The caller must hold the cpuhp lock.
47 *
48 * Return: Cache ID on success, or -1 if no matching cache exists.
49 */
50int get_cpu_cacheinfo_id(int cpu, int level)
51{
52 struct cacheinfo *ci = get_cpu_cacheinfo_level(cpu, level);
53
54 return ci ? ci->id : -1;
55}
56EXPORT_SYMBOL_GPL(get_cpu_cacheinfo_id);
57
58static inline bool cache_leaves_are_shared(struct cacheinfo *this_leaf,
59 struct cacheinfo *sib_leaf)
60{
61 /*
62 * For non DT/ACPI systems, assume unique level 1 caches,
63 * system-wide shared caches for all other levels.
64 */
65 if (!(IS_ENABLED(CONFIG_OF) || IS_ENABLED(CONFIG_ACPI)) ||
66 use_arch_info)
67 return (this_leaf->level != 1) && (sib_leaf->level != 1);
68
69 if ((sib_leaf->attributes & CACHE_ID) &&
70 (this_leaf->attributes & CACHE_ID))
71 return sib_leaf->id == this_leaf->id;
72
73 return sib_leaf->fw_token == this_leaf->fw_token;
74}
75
76bool last_level_cache_is_valid(unsigned int cpu)
77{
78 struct cacheinfo *llc;
79
80 if (!cache_leaves(cpu) || !per_cpu_cacheinfo(cpu))
81 return false;
82
83 llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1);
84
85 return (llc->attributes & CACHE_ID) || !!llc->fw_token;
86
87}
88
89/*
90 * Get the cacheinfo of the LLC associated with @cpu.
91 * Derived from update_per_cpu_data_slice_size_cpu().
92 */
93struct cacheinfo *get_cpu_cacheinfo_llc(unsigned int cpu)
94{
95 struct cacheinfo *llc;
96
97 if (!last_level_cache_is_valid(cpu))
98 return NULL;
99
100 llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1);
101 if (llc->type != CACHE_TYPE_DATA && llc->type != CACHE_TYPE_UNIFIED)
102 return NULL;
103
104 return llc;
105}
106
107bool last_level_cache_is_shared(unsigned int cpu_x, unsigned int cpu_y)
108{
109 struct cacheinfo *llc_x, *llc_y;
110
111 if (!last_level_cache_is_valid(cpu_x) ||
112 !last_level_cache_is_valid(cpu_y))
113 return false;
114
115 llc_x = per_cpu_cacheinfo_idx(cpu_x, cache_leaves(cpu_x) - 1);
116 llc_y = per_cpu_cacheinfo_idx(cpu_y, cache_leaves(cpu_y) - 1);
117
118 return cache_leaves_are_shared(llc_x, llc_y);
119}
120
121#ifdef CONFIG_OF
122
123static bool of_check_cache_nodes(struct device_node *np);
124
125/* OF properties to query for a given cache type */
126struct cache_type_info {
127 const char *size_prop;
128 const char *line_size_props[2];
129 const char *nr_sets_prop;
130};
131
132static const struct cache_type_info cache_type_info[] = {
133 {
134 .size_prop = "cache-size",
135 .line_size_props = { "cache-line-size",
136 "cache-block-size", },
137 .nr_sets_prop = "cache-sets",
138 }, {
139 .size_prop = "i-cache-size",
140 .line_size_props = { "i-cache-line-size",
141 "i-cache-block-size", },
142 .nr_sets_prop = "i-cache-sets",
143 }, {
144 .size_prop = "d-cache-size",
145 .line_size_props = { "d-cache-line-size",
146 "d-cache-block-size", },
147 .nr_sets_prop = "d-cache-sets",
148 },
149};
150
151static inline int get_cacheinfo_idx(enum cache_type type)
152{
153 if (type == CACHE_TYPE_UNIFIED)
154 return 0;
155 return type;
156}
157
158static void cache_size(struct cacheinfo *this_leaf, struct device_node *np)
159{
160 const char *propname;
161 int ct_idx;
162
163 ct_idx = get_cacheinfo_idx(this_leaf->type);
164 propname = cache_type_info[ct_idx].size_prop;
165
166 of_property_read_u32(np, propname, &this_leaf->size);
167}
168
169/* not cache_line_size() because that's a macro in include/linux/cache.h */
170static void cache_get_line_size(struct cacheinfo *this_leaf,
171 struct device_node *np)
172{
173 int i, lim, ct_idx;
174
175 ct_idx = get_cacheinfo_idx(this_leaf->type);
176 lim = ARRAY_SIZE(cache_type_info[ct_idx].line_size_props);
177
178 for (i = 0; i < lim; i++) {
179 int ret;
180 u32 line_size;
181 const char *propname;
182
183 propname = cache_type_info[ct_idx].line_size_props[i];
184 ret = of_property_read_u32(np, propname, &line_size);
185 if (!ret) {
186 this_leaf->coherency_line_size = line_size;
187 break;
188 }
189 }
190}
191
192static void cache_nr_sets(struct cacheinfo *this_leaf, struct device_node *np)
193{
194 const char *propname;
195 int ct_idx;
196
197 ct_idx = get_cacheinfo_idx(this_leaf->type);
198 propname = cache_type_info[ct_idx].nr_sets_prop;
199
200 of_property_read_u32(np, propname, &this_leaf->number_of_sets);
201}
202
203static void cache_associativity(struct cacheinfo *this_leaf)
204{
205 unsigned int line_size = this_leaf->coherency_line_size;
206 unsigned int nr_sets = this_leaf->number_of_sets;
207 unsigned int size = this_leaf->size;
208
209 /*
210 * If the cache is fully associative, there is no need to
211 * check the other properties.
212 */
213 if (!(nr_sets == 1) && (nr_sets > 0 && size > 0 && line_size > 0))
214 this_leaf->ways_of_associativity = (size / nr_sets) / line_size;
215}
216
217static bool cache_node_is_unified(struct cacheinfo *this_leaf,
218 struct device_node *np)
219{
220 return of_property_read_bool(np, "cache-unified");
221}
222
223static bool match_cache_node(struct device_node *cpu,
224 const struct device_node *cache_node)
225{
226 struct device_node *prev, *cache = of_find_next_cache_node(cpu);
227
228 while (cache) {
229 if (cache == cache_node) {
230 of_node_put(cache);
231 return true;
232 }
233
234 prev = cache;
235 cache = of_find_next_cache_node(cache);
236 of_node_put(prev);
237 }
238
239 return false;
240}
241
242#ifndef arch_compact_of_hwid
243#define arch_compact_of_hwid(_x) (_x)
244#endif
245
246static void cache_of_set_id(struct cacheinfo *this_leaf,
247 struct device_node *cache_node)
248{
249 struct device_node *cpu;
250 u32 min_id = ~0;
251
252 for_each_of_cpu_node(cpu) {
253 u64 id = of_get_cpu_hwid(cpu, 0);
254
255 id = arch_compact_of_hwid(id);
256 if (FIELD_GET(GENMASK_ULL(63, 32), id)) {
257 of_node_put(cpu);
258 return;
259 }
260
261 if (match_cache_node(cpu, cache_node))
262 min_id = min(min_id, id);
263 }
264
265 if (min_id != ~0) {
266 this_leaf->id = min_id;
267 this_leaf->attributes |= CACHE_ID;
268 }
269}
270
271static void cache_of_set_props(struct cacheinfo *this_leaf,
272 struct device_node *np)
273{
274 /*
275 * init_cache_level must setup the cache level correctly
276 * overriding the architecturally specified levels, so
277 * if type is NONE at this stage, it should be unified
278 */
279 if (this_leaf->type == CACHE_TYPE_NOCACHE &&
280 cache_node_is_unified(this_leaf, np))
281 this_leaf->type = CACHE_TYPE_UNIFIED;
282 cache_size(this_leaf, np);
283 cache_get_line_size(this_leaf, np);
284 cache_nr_sets(this_leaf, np);
285 cache_associativity(this_leaf);
286 cache_of_set_id(this_leaf, np);
287}
288
289static int cache_setup_of_node(unsigned int cpu)
290{
291 struct cacheinfo *this_leaf;
292 unsigned int index = 0;
293
294 struct device_node *np __free(device_node) = of_cpu_device_node_get(cpu);
295 if (!np) {
296 pr_err("Failed to find cpu%d device node\n", cpu);
297 return -ENOENT;
298 }
299
300 if (!of_check_cache_nodes(np)) {
301 return -ENOENT;
302 }
303
304 while (index < cache_leaves(cpu)) {
305 this_leaf = per_cpu_cacheinfo_idx(cpu, index);
306 if (this_leaf->level != 1) {
307 struct device_node *prev __free(device_node) = np;
308 np = of_find_next_cache_node(np);
309 if (!np)
310 break;
311 }
312 cache_of_set_props(this_leaf, np);
313 this_leaf->fw_token = np;
314 index++;
315 }
316
317 if (index != cache_leaves(cpu)) /* not all OF nodes populated */
318 return -ENOENT;
319
320 return 0;
321}
322
323static bool of_check_cache_nodes(struct device_node *np)
324{
325 if (of_property_present(np, "cache-size") ||
326 of_property_present(np, "i-cache-size") ||
327 of_property_present(np, "d-cache-size") ||
328 of_property_present(np, "cache-unified"))
329 return true;
330
331 struct device_node *next __free(device_node) = of_find_next_cache_node(np);
332 if (next) {
333 return true;
334 }
335
336 return false;
337}
338
339static int of_count_cache_leaves(struct device_node *np)
340{
341 unsigned int leaves = 0;
342
343 if (of_property_present(np, "cache-size"))
344 ++leaves;
345 if (of_property_present(np, "i-cache-size"))
346 ++leaves;
347 if (of_property_present(np, "d-cache-size"))
348 ++leaves;
349
350 if (!leaves) {
351 /* The '[i-|d-|]cache-size' property is required, but
352 * if absent, fallback on the 'cache-unified' property.
353 */
354 if (of_property_read_bool(np, "cache-unified"))
355 return 1;
356 else
357 return 2;
358 }
359
360 return leaves;
361}
362
363int init_of_cache_level(unsigned int cpu)
364{
365 struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
366 struct device_node *np __free(device_node) = of_cpu_device_node_get(cpu);
367 unsigned int levels = 0, leaves, level;
368
369 if (!of_check_cache_nodes(np)) {
370 return -ENOENT;
371 }
372
373 leaves = of_count_cache_leaves(np);
374 if (leaves > 0)
375 levels = 1;
376
377 while (1) {
378 struct device_node *prev __free(device_node) = np;
379 np = of_find_next_cache_node(np);
380 if (!np)
381 break;
382
383 if (!of_device_is_compatible(np, "cache"))
384 return -EINVAL;
385 if (of_property_read_u32(np, "cache-level", &level))
386 return -EINVAL;
387 if (level <= levels)
388 return -EINVAL;
389
390 leaves += of_count_cache_leaves(np);
391 levels = level;
392 }
393
394 this_cpu_ci->num_levels = levels;
395 this_cpu_ci->num_leaves = leaves;
396
397 return 0;
398}
399
400#else
401static inline int cache_setup_of_node(unsigned int cpu) { return 0; }
402int init_of_cache_level(unsigned int cpu) { return 0; }
403#endif
404
405int __weak cache_setup_acpi(unsigned int cpu)
406{
407 return -ENOTSUPP;
408}
409
410unsigned int coherency_max_size;
411
412static int cache_setup_properties(unsigned int cpu)
413{
414 int ret = 0;
415
416 if (of_have_populated_dt())
417 ret = cache_setup_of_node(cpu);
418 else if (!acpi_disabled)
419 ret = cache_setup_acpi(cpu);
420
421 /*
422 * No DT/ACPI cache nodes; fall back to arch-derived topology (e.g.
423 * arm64 CLIDR_EL1) and clear the error to avoid a spurious warning.
424 */
425 if (ret && use_arch_cache_info()) {
426 use_arch_info = true;
427 ret = 0;
428 }
429
430 return ret;
431}
432
433static int cache_shared_cpu_map_setup(unsigned int cpu)
434{
435 struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
436 struct cacheinfo *this_leaf, *sib_leaf;
437 unsigned int index, sib_index;
438 int ret = 0;
439
440 if (this_cpu_ci->cpu_map_populated)
441 return 0;
442
443 /*
444 * skip setting up cache properties if LLC is valid, just need
445 * to update the shared cpu_map if the cache attributes were
446 * populated early before all the cpus are brought online
447 */
448 if (!last_level_cache_is_valid(cpu) && !use_arch_info) {
449 ret = cache_setup_properties(cpu);
450 if (ret)
451 return ret;
452 }
453
454 for (index = 0; index < cache_leaves(cpu); index++) {
455 unsigned int i;
456
457 this_leaf = per_cpu_cacheinfo_idx(cpu, index);
458
459 cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map);
460 for_each_online_cpu(i) {
461 if (i == cpu || !per_cpu_cacheinfo(i))
462 continue;/* skip if itself or no cacheinfo */
463 for (sib_index = 0; sib_index < cache_leaves(i); sib_index++) {
464 sib_leaf = per_cpu_cacheinfo_idx(i, sib_index);
465
466 /*
467 * Comparing cache IDs only makes sense if the leaves
468 * belong to the same cache level of same type. Skip
469 * the check if level and type do not match.
470 */
471 if (sib_leaf->level != this_leaf->level ||
472 sib_leaf->type != this_leaf->type)
473 continue;
474
475 if (cache_leaves_are_shared(this_leaf, sib_leaf)) {
476 cpumask_set_cpu(cpu, &sib_leaf->shared_cpu_map);
477 cpumask_set_cpu(i, &this_leaf->shared_cpu_map);
478 break;
479 }
480 }
481 }
482 /* record the maximum cache line size */
483 if (this_leaf->coherency_line_size > coherency_max_size)
484 coherency_max_size = this_leaf->coherency_line_size;
485 }
486
487 /* shared_cpu_map is now populated for the cpu */
488 this_cpu_ci->cpu_map_populated = true;
489 return 0;
490}
491
492static void cache_shared_cpu_map_remove(unsigned int cpu)
493{
494 struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
495 struct cacheinfo *this_leaf, *sib_leaf;
496 unsigned int sibling, index, sib_index;
497
498 for (index = 0; index < cache_leaves(cpu); index++) {
499 this_leaf = per_cpu_cacheinfo_idx(cpu, index);
500 for_each_cpu(sibling, &this_leaf->shared_cpu_map) {
501 if (sibling == cpu || !per_cpu_cacheinfo(sibling))
502 continue;/* skip if itself or no cacheinfo */
503
504 for (sib_index = 0; sib_index < cache_leaves(sibling); sib_index++) {
505 sib_leaf = per_cpu_cacheinfo_idx(sibling, sib_index);
506
507 /*
508 * Comparing cache IDs only makes sense if the leaves
509 * belong to the same cache level of same type. Skip
510 * the check if level and type do not match.
511 */
512 if (sib_leaf->level != this_leaf->level ||
513 sib_leaf->type != this_leaf->type)
514 continue;
515
516 if (cache_leaves_are_shared(this_leaf, sib_leaf)) {
517 cpumask_clear_cpu(cpu, &sib_leaf->shared_cpu_map);
518 cpumask_clear_cpu(sibling, &this_leaf->shared_cpu_map);
519 break;
520 }
521 }
522 }
523 }
524
525 /* cpu is no longer populated in the shared map */
526 this_cpu_ci->cpu_map_populated = false;
527}
528
529static void free_cache_attributes(unsigned int cpu)
530{
531 if (!per_cpu_cacheinfo(cpu))
532 return;
533
534 cache_shared_cpu_map_remove(cpu);
535}
536
537int __weak early_cache_level(unsigned int cpu)
538{
539 return -ENOENT;
540}
541
542int __weak init_cache_level(unsigned int cpu)
543{
544 return -ENOENT;
545}
546
547int __weak populate_cache_leaves(unsigned int cpu)
548{
549 return -ENOENT;
550}
551
552static inline int allocate_cache_info(int cpu)
553{
554 per_cpu_cacheinfo(cpu) = kzalloc_objs(struct cacheinfo,
555 cache_leaves(cpu), GFP_ATOMIC);
556 if (!per_cpu_cacheinfo(cpu)) {
557 cache_leaves(cpu) = 0;
558 return -ENOMEM;
559 }
560
561 return 0;
562}
563
564int fetch_cache_info(unsigned int cpu)
565{
566 struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
567 unsigned int levels = 0, split_levels = 0;
568 int ret;
569
570 if (acpi_disabled) {
571 ret = init_of_cache_level(cpu);
572 } else {
573 ret = acpi_get_cache_info(cpu, &levels, &split_levels);
574 if (!ret) {
575 this_cpu_ci->num_levels = levels;
576 /*
577 * This assumes that:
578 * - there cannot be any split caches (data/instruction)
579 * above a unified cache
580 * - data/instruction caches come by pair
581 */
582 this_cpu_ci->num_leaves = levels + split_levels;
583 }
584 }
585
586 if (ret || !cache_leaves(cpu)) {
587 ret = early_cache_level(cpu);
588 if (ret)
589 return ret;
590
591 if (!cache_leaves(cpu))
592 return -ENOENT;
593
594 this_cpu_ci->early_ci_levels = true;
595 }
596
597 return allocate_cache_info(cpu);
598}
599
600static inline int init_level_allocate_ci(unsigned int cpu)
601{
602 unsigned int early_leaves = cache_leaves(cpu);
603
604 /* Since early initialization/allocation of the cacheinfo is allowed
605 * via fetch_cache_info() and this also gets called as CPU hotplug
606 * callbacks via cacheinfo_cpu_online, the init/alloc can be skipped
607 * as it will happen only once (the cacheinfo memory is never freed).
608 * Just populate the cacheinfo. However, if the cacheinfo has been
609 * allocated early through the arch-specific early_cache_level() call,
610 * there is a chance the info is wrong (this can happen on arm64). In
611 * that case, call init_cache_level() anyway to give the arch-specific
612 * code a chance to make things right.
613 */
614 if (per_cpu_cacheinfo(cpu) && !ci_cacheinfo(cpu)->early_ci_levels)
615 return 0;
616
617 if (init_cache_level(cpu) || !cache_leaves(cpu))
618 return -ENOENT;
619
620 /*
621 * Now that we have properly initialized the cache level info, make
622 * sure we don't try to do that again the next time we are called
623 * (e.g. as CPU hotplug callbacks).
624 */
625 ci_cacheinfo(cpu)->early_ci_levels = false;
626
627 /*
628 * Some architectures (e.g., x86) do not use early initialization.
629 * Allocate memory now in such case.
630 */
631 if (cache_leaves(cpu) <= early_leaves && per_cpu_cacheinfo(cpu))
632 return 0;
633
634 kfree(per_cpu_cacheinfo(cpu));
635 return allocate_cache_info(cpu);
636}
637
638int detect_cache_attributes(unsigned int cpu)
639{
640 int ret;
641
642 ret = init_level_allocate_ci(cpu);
643 if (ret)
644 return ret;
645
646 /*
647 * If LLC is valid the cache leaves were already populated so just go to
648 * update the cpu map.
649 */
650 if (!last_level_cache_is_valid(cpu)) {
651 /*
652 * populate_cache_leaves() may completely setup the cache leaves and
653 * shared_cpu_map or it may leave it partially setup.
654 */
655 ret = populate_cache_leaves(cpu);
656 if (ret)
657 goto free_ci;
658 }
659
660 /*
661 * For systems using DT for cache hierarchy, fw_token
662 * and shared_cpu_map will be set up here only if they are
663 * not populated already
664 */
665 ret = cache_shared_cpu_map_setup(cpu);
666 if (ret) {
667 pr_warn("Unable to detect cache hierarchy for CPU %d\n", cpu);
668 goto free_ci;
669 }
670
671 return 0;
672
673free_ci:
674 free_cache_attributes(cpu);
675 return ret;
676}
677
678/* pointer to cpuX/cache device */
679static DEFINE_PER_CPU(struct device *, ci_cache_dev);
680#define per_cpu_cache_dev(cpu) (per_cpu(ci_cache_dev, cpu))
681
682static cpumask_t cache_dev_map;
683
684/* pointer to array of devices for cpuX/cache/indexY */
685static DEFINE_PER_CPU(struct device **, ci_index_dev);
686#define per_cpu_index_dev(cpu) (per_cpu(ci_index_dev, cpu))
687#define per_cache_index_dev(cpu, idx) ((per_cpu_index_dev(cpu))[idx])
688
689#define show_one(file_name, object) \
690static ssize_t file_name##_show(struct device *dev, \
691 struct device_attribute *attr, char *buf) \
692{ \
693 struct cacheinfo *this_leaf = dev_get_drvdata(dev); \
694 return sysfs_emit(buf, "%u\n", this_leaf->object); \
695}
696
697show_one(id, id);
698show_one(level, level);
699show_one(coherency_line_size, coherency_line_size);
700show_one(number_of_sets, number_of_sets);
701show_one(physical_line_partition, physical_line_partition);
702show_one(ways_of_associativity, ways_of_associativity);
703
704static ssize_t size_show(struct device *dev,
705 struct device_attribute *attr, char *buf)
706{
707 struct cacheinfo *this_leaf = dev_get_drvdata(dev);
708
709 return sysfs_emit(buf, "%uK\n", this_leaf->size >> 10);
710}
711
712static ssize_t shared_cpu_map_show(struct device *dev,
713 struct device_attribute *attr, char *buf)
714{
715 struct cacheinfo *this_leaf = dev_get_drvdata(dev);
716 const struct cpumask *mask = &this_leaf->shared_cpu_map;
717
718 return sysfs_emit(buf, "%*pb\n", nr_cpu_ids, mask);
719}
720
721static ssize_t shared_cpu_list_show(struct device *dev,
722 struct device_attribute *attr, char *buf)
723{
724 struct cacheinfo *this_leaf = dev_get_drvdata(dev);
725 const struct cpumask *mask = &this_leaf->shared_cpu_map;
726
727 return sysfs_emit(buf, "%*pbl\n", nr_cpu_ids, mask);
728}
729
730static ssize_t type_show(struct device *dev,
731 struct device_attribute *attr, char *buf)
732{
733 struct cacheinfo *this_leaf = dev_get_drvdata(dev);
734 const char *output;
735
736 switch (this_leaf->type) {
737 case CACHE_TYPE_DATA:
738 output = "Data";
739 break;
740 case CACHE_TYPE_INST:
741 output = "Instruction";
742 break;
743 case CACHE_TYPE_UNIFIED:
744 output = "Unified";
745 break;
746 default:
747 return -EINVAL;
748 }
749
750 return sysfs_emit(buf, "%s\n", output);
751}
752
753static ssize_t allocation_policy_show(struct device *dev,
754 struct device_attribute *attr, char *buf)
755{
756 struct cacheinfo *this_leaf = dev_get_drvdata(dev);
757 unsigned int ci_attr = this_leaf->attributes;
758 const char *output;
759
760 if ((ci_attr & CACHE_READ_ALLOCATE) && (ci_attr & CACHE_WRITE_ALLOCATE))
761 output = "ReadWriteAllocate";
762 else if (ci_attr & CACHE_READ_ALLOCATE)
763 output = "ReadAllocate";
764 else if (ci_attr & CACHE_WRITE_ALLOCATE)
765 output = "WriteAllocate";
766 else
767 return 0;
768
769 return sysfs_emit(buf, "%s\n", output);
770}
771
772static ssize_t write_policy_show(struct device *dev,
773 struct device_attribute *attr, char *buf)
774{
775 struct cacheinfo *this_leaf = dev_get_drvdata(dev);
776 unsigned int ci_attr = this_leaf->attributes;
777 int n = 0;
778
779 if (ci_attr & CACHE_WRITE_THROUGH)
780 n = sysfs_emit(buf, "WriteThrough\n");
781 else if (ci_attr & CACHE_WRITE_BACK)
782 n = sysfs_emit(buf, "WriteBack\n");
783 return n;
784}
785
786static DEVICE_ATTR_RO(id);
787static DEVICE_ATTR_RO(level);
788static DEVICE_ATTR_RO(type);
789static DEVICE_ATTR_RO(coherency_line_size);
790static DEVICE_ATTR_RO(ways_of_associativity);
791static DEVICE_ATTR_RO(number_of_sets);
792static DEVICE_ATTR_RO(size);
793static DEVICE_ATTR_RO(allocation_policy);
794static DEVICE_ATTR_RO(write_policy);
795static DEVICE_ATTR_RO(shared_cpu_map);
796static DEVICE_ATTR_RO(shared_cpu_list);
797static DEVICE_ATTR_RO(physical_line_partition);
798
799static struct attribute *cache_default_attrs[] = {
800 &dev_attr_id.attr,
801 &dev_attr_type.attr,
802 &dev_attr_level.attr,
803 &dev_attr_shared_cpu_map.attr,
804 &dev_attr_shared_cpu_list.attr,
805 &dev_attr_coherency_line_size.attr,
806 &dev_attr_ways_of_associativity.attr,
807 &dev_attr_number_of_sets.attr,
808 &dev_attr_size.attr,
809 &dev_attr_allocation_policy.attr,
810 &dev_attr_write_policy.attr,
811 &dev_attr_physical_line_partition.attr,
812 NULL
813};
814
815static umode_t
816cache_default_attrs_is_visible(struct kobject *kobj,
817 struct attribute *attr, int unused)
818{
819 struct device *dev = kobj_to_dev(kobj);
820 struct cacheinfo *this_leaf = dev_get_drvdata(dev);
821 const struct cpumask *mask = &this_leaf->shared_cpu_map;
822 umode_t mode = attr->mode;
823
824 if ((attr == &dev_attr_id.attr) && (this_leaf->attributes & CACHE_ID))
825 return mode;
826 if ((attr == &dev_attr_type.attr) && this_leaf->type)
827 return mode;
828 if ((attr == &dev_attr_level.attr) && this_leaf->level)
829 return mode;
830 if ((attr == &dev_attr_shared_cpu_map.attr) && !cpumask_empty(mask))
831 return mode;
832 if ((attr == &dev_attr_shared_cpu_list.attr) && !cpumask_empty(mask))
833 return mode;
834 if ((attr == &dev_attr_coherency_line_size.attr) &&
835 this_leaf->coherency_line_size)
836 return mode;
837 if ((attr == &dev_attr_ways_of_associativity.attr) &&
838 this_leaf->size) /* allow 0 = full associativity */
839 return mode;
840 if ((attr == &dev_attr_number_of_sets.attr) &&
841 this_leaf->number_of_sets)
842 return mode;
843 if ((attr == &dev_attr_size.attr) && this_leaf->size)
844 return mode;
845 if ((attr == &dev_attr_write_policy.attr) &&
846 (this_leaf->attributes & CACHE_WRITE_POLICY_MASK))
847 return mode;
848 if ((attr == &dev_attr_allocation_policy.attr) &&
849 (this_leaf->attributes & CACHE_ALLOCATE_POLICY_MASK))
850 return mode;
851 if ((attr == &dev_attr_physical_line_partition.attr) &&
852 this_leaf->physical_line_partition)
853 return mode;
854
855 return 0;
856}
857
858static const struct attribute_group cache_default_group = {
859 .attrs = cache_default_attrs,
860 .is_visible = cache_default_attrs_is_visible,
861};
862
863static const struct attribute_group *cache_default_groups[] = {
864 &cache_default_group,
865 NULL,
866};
867
868static const struct attribute_group *cache_private_groups[] = {
869 &cache_default_group,
870 NULL, /* Place holder for private group */
871 NULL,
872};
873
874const struct attribute_group *
875__weak cache_get_priv_group(struct cacheinfo *this_leaf)
876{
877 return NULL;
878}
879
880static const struct attribute_group **
881cache_get_attribute_groups(struct cacheinfo *this_leaf)
882{
883 const struct attribute_group *priv_group =
884 cache_get_priv_group(this_leaf);
885
886 if (!priv_group)
887 return cache_default_groups;
888
889 if (!cache_private_groups[1])
890 cache_private_groups[1] = priv_group;
891
892 return cache_private_groups;
893}
894
895/* Add/Remove cache interface for CPU device */
896static void cpu_cache_sysfs_exit(unsigned int cpu)
897{
898 int i;
899 struct device *ci_dev;
900
901 if (per_cpu_index_dev(cpu)) {
902 for (i = 0; i < cache_leaves(cpu); i++) {
903 ci_dev = per_cache_index_dev(cpu, i);
904 if (!ci_dev)
905 continue;
906 device_unregister(ci_dev);
907 }
908 kfree(per_cpu_index_dev(cpu));
909 per_cpu_index_dev(cpu) = NULL;
910 }
911 device_unregister(per_cpu_cache_dev(cpu));
912 per_cpu_cache_dev(cpu) = NULL;
913}
914
915static int cpu_cache_sysfs_init(unsigned int cpu)
916{
917 struct device *dev = get_cpu_device(cpu);
918
919 if (per_cpu_cacheinfo(cpu) == NULL)
920 return -ENOENT;
921
922 per_cpu_cache_dev(cpu) = cpu_device_create(dev, NULL, NULL, "cache");
923 if (IS_ERR(per_cpu_cache_dev(cpu)))
924 return PTR_ERR(per_cpu_cache_dev(cpu));
925
926 /* Allocate all required memory */
927 per_cpu_index_dev(cpu) = kzalloc_objs(struct device *,
928 cache_leaves(cpu));
929 if (unlikely(per_cpu_index_dev(cpu) == NULL))
930 goto err_out;
931
932 return 0;
933
934err_out:
935 cpu_cache_sysfs_exit(cpu);
936 return -ENOMEM;
937}
938
939static int cache_add_dev(unsigned int cpu)
940{
941 unsigned int i;
942 int rc;
943 struct device *ci_dev, *parent;
944 struct cacheinfo *this_leaf;
945 const struct attribute_group **cache_groups;
946
947 rc = cpu_cache_sysfs_init(cpu);
948 if (unlikely(rc < 0))
949 return rc;
950
951 parent = per_cpu_cache_dev(cpu);
952 for (i = 0; i < cache_leaves(cpu); i++) {
953 this_leaf = per_cpu_cacheinfo_idx(cpu, i);
954 if (this_leaf->disable_sysfs)
955 continue;
956 if (this_leaf->type == CACHE_TYPE_NOCACHE)
957 break;
958 cache_groups = cache_get_attribute_groups(this_leaf);
959 ci_dev = cpu_device_create(parent, this_leaf, cache_groups,
960 "index%1u", i);
961 if (IS_ERR(ci_dev)) {
962 rc = PTR_ERR(ci_dev);
963 goto err;
964 }
965 per_cache_index_dev(cpu, i) = ci_dev;
966 }
967 cpumask_set_cpu(cpu, &cache_dev_map);
968
969 return 0;
970err:
971 cpu_cache_sysfs_exit(cpu);
972 return rc;
973}
974
975static unsigned int cpu_map_shared_cache(bool online, unsigned int cpu,
976 cpumask_t **map)
977{
978 struct cacheinfo *llc, *sib_llc;
979 unsigned int sibling;
980
981 if (!last_level_cache_is_valid(cpu))
982 return 0;
983
984 llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1);
985
986 if (llc->type != CACHE_TYPE_DATA && llc->type != CACHE_TYPE_UNIFIED)
987 return 0;
988
989 if (online) {
990 *map = &llc->shared_cpu_map;
991 return cpumask_weight(*map);
992 }
993
994 /* shared_cpu_map of offlined CPU will be cleared, so use sibling map */
995 for_each_cpu(sibling, &llc->shared_cpu_map) {
996 if (sibling == cpu || !last_level_cache_is_valid(sibling))
997 continue;
998 sib_llc = per_cpu_cacheinfo_idx(sibling, cache_leaves(sibling) - 1);
999 *map = &sib_llc->shared_cpu_map;
1000 return cpumask_weight(*map);
1001 }
1002
1003 return 0;
1004}
1005
1006/*
1007 * Calculate the size of the per-CPU data cache slice. This can be
1008 * used to estimate the size of the data cache slice that can be used
1009 * by one CPU under ideal circumstances. UNIFIED caches are counted
1010 * in addition to DATA caches. So, please consider code cache usage
1011 * when use the result.
1012 *
1013 * Because the cache inclusive/non-inclusive information isn't
1014 * available, we just use the size of the per-CPU slice of LLC to make
1015 * the result more predictable across architectures.
1016 */
1017static void update_per_cpu_data_slice_size_cpu(unsigned int cpu)
1018{
1019 struct cpu_cacheinfo *ci;
1020 struct cacheinfo *llc;
1021 unsigned int nr_shared;
1022
1023 if (!last_level_cache_is_valid(cpu))
1024 return;
1025
1026 ci = ci_cacheinfo(cpu);
1027 llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1);
1028
1029 if (llc->type != CACHE_TYPE_DATA && llc->type != CACHE_TYPE_UNIFIED)
1030 return;
1031
1032 nr_shared = cpumask_weight(&llc->shared_cpu_map);
1033 if (nr_shared)
1034 ci->per_cpu_data_slice_size = llc->size / nr_shared;
1035}
1036
1037static void update_per_cpu_data_slice_size(bool cpu_online, unsigned int cpu,
1038 cpumask_t *cpu_map)
1039{
1040 unsigned int icpu;
1041
1042 for_each_cpu(icpu, cpu_map) {
1043 if (!cpu_online && icpu == cpu)
1044 continue;
1045 update_per_cpu_data_slice_size_cpu(icpu);
1046 setup_pcp_cacheinfo(icpu);
1047 }
1048}
1049
1050static int cacheinfo_cpu_online(unsigned int cpu)
1051{
1052 int rc = detect_cache_attributes(cpu);
1053 cpumask_t *cpu_map;
1054
1055 if (rc)
1056 return rc;
1057 rc = cache_add_dev(cpu);
1058 if (rc)
1059 goto err;
1060 if (cpu_map_shared_cache(true, cpu, &cpu_map)) {
1061 update_per_cpu_data_slice_size(true, cpu, cpu_map);
1062 sched_update_llc_bytes(cpu_map);
1063 }
1064 return 0;
1065err:
1066 free_cache_attributes(cpu);
1067 return rc;
1068}
1069
1070static int cacheinfo_cpu_pre_down(unsigned int cpu)
1071{
1072 cpumask_t *cpu_map;
1073 unsigned int nr_shared;
1074
1075 nr_shared = cpu_map_shared_cache(false, cpu, &cpu_map);
1076 if (cpumask_test_and_clear_cpu(cpu, &cache_dev_map))
1077 cpu_cache_sysfs_exit(cpu);
1078
1079 free_cache_attributes(cpu);
1080 if (nr_shared > 1) {
1081 update_per_cpu_data_slice_size(false, cpu, cpu_map);
1082 sched_update_llc_bytes(cpu_map);
1083 }
1084
1085 return 0;
1086}
1087
1088static int __init cacheinfo_sysfs_init(void)
1089{
1090 return cpuhp_setup_state(CPUHP_AP_BASE_CACHEINFO_ONLINE,
1091 "base/cacheinfo:online",
1092 cacheinfo_cpu_online, cacheinfo_cpu_pre_down);
1093}
1094device_initcall(cacheinfo_sysfs_init);