projects/linux·blob assoc-array-fixes · refs

1077 lines · 26.9 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * cacheinfo support - processor cache information via sysfs
4 *
5 * Based on arch/x86/kernel/cpu/intel_cacheinfo.c
6 * Author: Sudeep Holla <[email protected]>
7 */
8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
10#include <linux/acpi.h>
11#include <linux/bitfield.h>
12#include <linux/bitops.h>
13#include <linux/cacheinfo.h>
14#include <linux/compiler.h>
15#include <linux/cpu.h>
16#include <linux/device.h>
17#include <linux/init.h>
18#include <linux/of.h>
19#include <linux/sched.h>
20#include <linux/sched/topology.h>
21#include <linux/slab.h>
22#include <linux/smp.h>
23#include <linux/sysfs.h>
24
25/* pointer to per cpu cacheinfo */
26static DEFINE_PER_CPU(struct cpu_cacheinfo, ci_cpu_cacheinfo);
27#define ci_cacheinfo(cpu) (&per_cpu(ci_cpu_cacheinfo, cpu))
28#define cache_leaves(cpu) (ci_cacheinfo(cpu)->num_leaves)
29#define per_cpu_cacheinfo(cpu) (ci_cacheinfo(cpu)->info_list)
30#define per_cpu_cacheinfo_idx(cpu, idx) \
31 (per_cpu_cacheinfo(cpu) + (idx))
32
33/* Set if no cache information is found in DT/ACPI. */
34static bool use_arch_info;
35
36struct cpu_cacheinfo *get_cpu_cacheinfo(unsigned int cpu)
37{
38 return ci_cacheinfo(cpu);
39}
40
41static inline bool cache_leaves_are_shared(struct cacheinfo *this_leaf,
42 struct cacheinfo *sib_leaf)
43{
44 /*
45 * For non DT/ACPI systems, assume unique level 1 caches,
46 * system-wide shared caches for all other levels.
47 */
48 if (!(IS_ENABLED(CONFIG_OF) || IS_ENABLED(CONFIG_ACPI)) ||
49 use_arch_info)
50 return (this_leaf->level != 1) && (sib_leaf->level != 1);
51
52 if ((sib_leaf->attributes & CACHE_ID) &&
53 (this_leaf->attributes & CACHE_ID))
54 return sib_leaf->id == this_leaf->id;
55
56 return sib_leaf->fw_token == this_leaf->fw_token;
57}
58
59bool last_level_cache_is_valid(unsigned int cpu)
60{
61 struct cacheinfo *llc;
62
63 if (!cache_leaves(cpu) || !per_cpu_cacheinfo(cpu))
64 return false;
65
66 llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1);
67
68 return (llc->attributes & CACHE_ID) || !!llc->fw_token;
69
70}
71
72/*
73 * Get the cacheinfo of the LLC associated with @cpu.
74 * Derived from update_per_cpu_data_slice_size_cpu().
75 */
76struct cacheinfo *get_cpu_cacheinfo_llc(unsigned int cpu)
77{
78 struct cacheinfo *llc;
79
80 if (!last_level_cache_is_valid(cpu))
81 return NULL;
82
83 llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1);
84 if (llc->type != CACHE_TYPE_DATA && llc->type != CACHE_TYPE_UNIFIED)
85 return NULL;
86
87 return llc;
88}
89
90bool last_level_cache_is_shared(unsigned int cpu_x, unsigned int cpu_y)
91{
92 struct cacheinfo *llc_x, *llc_y;
93
94 if (!last_level_cache_is_valid(cpu_x) ||
95 !last_level_cache_is_valid(cpu_y))
96 return false;
97
98 llc_x = per_cpu_cacheinfo_idx(cpu_x, cache_leaves(cpu_x) - 1);
99 llc_y = per_cpu_cacheinfo_idx(cpu_y, cache_leaves(cpu_y) - 1);
100
101 return cache_leaves_are_shared(llc_x, llc_y);
102}
103
104#ifdef CONFIG_OF
105
106static bool of_check_cache_nodes(struct device_node *np);
107
108/* OF properties to query for a given cache type */
109struct cache_type_info {
110 const char *size_prop;
111 const char *line_size_props[2];
112 const char *nr_sets_prop;
113};
114
115static const struct cache_type_info cache_type_info[] = {
116 {
117 .size_prop = "cache-size",
118 .line_size_props = { "cache-line-size",
119 "cache-block-size", },
120 .nr_sets_prop = "cache-sets",
121 }, {
122 .size_prop = "i-cache-size",
123 .line_size_props = { "i-cache-line-size",
124 "i-cache-block-size", },
125 .nr_sets_prop = "i-cache-sets",
126 }, {
127 .size_prop = "d-cache-size",
128 .line_size_props = { "d-cache-line-size",
129 "d-cache-block-size", },
130 .nr_sets_prop = "d-cache-sets",
131 },
132};
133
134static inline int get_cacheinfo_idx(enum cache_type type)
135{
136 if (type == CACHE_TYPE_UNIFIED)
137 return 0;
138 return type;
139}
140
141static void cache_size(struct cacheinfo *this_leaf, struct device_node *np)
142{
143 const char *propname;
144 int ct_idx;
145
146 ct_idx = get_cacheinfo_idx(this_leaf->type);
147 propname = cache_type_info[ct_idx].size_prop;
148
149 of_property_read_u32(np, propname, &this_leaf->size);
150}
151
152/* not cache_line_size() because that's a macro in include/linux/cache.h */
153static void cache_get_line_size(struct cacheinfo *this_leaf,
154 struct device_node *np)
155{
156 int i, lim, ct_idx;
157
158 ct_idx = get_cacheinfo_idx(this_leaf->type);
159 lim = ARRAY_SIZE(cache_type_info[ct_idx].line_size_props);
160
161 for (i = 0; i < lim; i++) {
162 int ret;
163 u32 line_size;
164 const char *propname;
165
166 propname = cache_type_info[ct_idx].line_size_props[i];
167 ret = of_property_read_u32(np, propname, &line_size);
168 if (!ret) {
169 this_leaf->coherency_line_size = line_size;
170 break;
171 }
172 }
173}
174
175static void cache_nr_sets(struct cacheinfo *this_leaf, struct device_node *np)
176{
177 const char *propname;
178 int ct_idx;
179
180 ct_idx = get_cacheinfo_idx(this_leaf->type);
181 propname = cache_type_info[ct_idx].nr_sets_prop;
182
183 of_property_read_u32(np, propname, &this_leaf->number_of_sets);
184}
185
186static void cache_associativity(struct cacheinfo *this_leaf)
187{
188 unsigned int line_size = this_leaf->coherency_line_size;
189 unsigned int nr_sets = this_leaf->number_of_sets;
190 unsigned int size = this_leaf->size;
191
192 /*
193 * If the cache is fully associative, there is no need to
194 * check the other properties.
195 */
196 if (!(nr_sets == 1) && (nr_sets > 0 && size > 0 && line_size > 0))
197 this_leaf->ways_of_associativity = (size / nr_sets) / line_size;
198}
199
200static bool cache_node_is_unified(struct cacheinfo *this_leaf,
201 struct device_node *np)
202{
203 return of_property_read_bool(np, "cache-unified");
204}
205
206static bool match_cache_node(struct device_node *cpu,
207 const struct device_node *cache_node)
208{
209 struct device_node *prev, *cache = of_find_next_cache_node(cpu);
210
211 while (cache) {
212 if (cache == cache_node) {
213 of_node_put(cache);
214 return true;
215 }
216
217 prev = cache;
218 cache = of_find_next_cache_node(cache);
219 of_node_put(prev);
220 }
221
222 return false;
223}
224
225#ifndef arch_compact_of_hwid
226#define arch_compact_of_hwid(_x) (_x)
227#endif
228
229static void cache_of_set_id(struct cacheinfo *this_leaf,
230 struct device_node *cache_node)
231{
232 struct device_node *cpu;
233 u32 min_id = ~0;
234
235 for_each_of_cpu_node(cpu) {
236 u64 id = of_get_cpu_hwid(cpu, 0);
237
238 id = arch_compact_of_hwid(id);
239 if (FIELD_GET(GENMASK_ULL(63, 32), id)) {
240 of_node_put(cpu);
241 return;
242 }
243
244 if (match_cache_node(cpu, cache_node))
245 min_id = min(min_id, id);
246 }
247
248 if (min_id != ~0) {
249 this_leaf->id = min_id;
250 this_leaf->attributes |= CACHE_ID;
251 }
252}
253
254static void cache_of_set_props(struct cacheinfo *this_leaf,
255 struct device_node *np)
256{
257 /*
258 * init_cache_level must setup the cache level correctly
259 * overriding the architecturally specified levels, so
260 * if type is NONE at this stage, it should be unified
261 */
262 if (this_leaf->type == CACHE_TYPE_NOCACHE &&
263 cache_node_is_unified(this_leaf, np))
264 this_leaf->type = CACHE_TYPE_UNIFIED;
265 cache_size(this_leaf, np);
266 cache_get_line_size(this_leaf, np);
267 cache_nr_sets(this_leaf, np);
268 cache_associativity(this_leaf);
269 cache_of_set_id(this_leaf, np);
270}
271
272static int cache_setup_of_node(unsigned int cpu)
273{
274 struct cacheinfo *this_leaf;
275 unsigned int index = 0;
276
277 struct device_node *np __free(device_node) = of_cpu_device_node_get(cpu);
278 if (!np) {
279 pr_err("Failed to find cpu%d device node\n", cpu);
280 return -ENOENT;
281 }
282
283 if (!of_check_cache_nodes(np)) {
284 return -ENOENT;
285 }
286
287 while (index < cache_leaves(cpu)) {
288 this_leaf = per_cpu_cacheinfo_idx(cpu, index);
289 if (this_leaf->level != 1) {
290 struct device_node *prev __free(device_node) = np;
291 np = of_find_next_cache_node(np);
292 if (!np)
293 break;
294 }
295 cache_of_set_props(this_leaf, np);
296 this_leaf->fw_token = np;
297 index++;
298 }
299
300 if (index != cache_leaves(cpu)) /* not all OF nodes populated */
301 return -ENOENT;
302
303 return 0;
304}
305
306static bool of_check_cache_nodes(struct device_node *np)
307{
308 if (of_property_present(np, "cache-size") ||
309 of_property_present(np, "i-cache-size") ||
310 of_property_present(np, "d-cache-size") ||
311 of_property_present(np, "cache-unified"))
312 return true;
313
314 struct device_node *next __free(device_node) = of_find_next_cache_node(np);
315 if (next) {
316 return true;
317 }
318
319 return false;
320}
321
322static int of_count_cache_leaves(struct device_node *np)
323{
324 unsigned int leaves = 0;
325
326 if (of_property_present(np, "cache-size"))
327 ++leaves;
328 if (of_property_present(np, "i-cache-size"))
329 ++leaves;
330 if (of_property_present(np, "d-cache-size"))
331 ++leaves;
332
333 if (!leaves) {
334 /* The '[i-|d-|]cache-size' property is required, but
335 * if absent, fallback on the 'cache-unified' property.
336 */
337 if (of_property_read_bool(np, "cache-unified"))
338 return 1;
339 else
340 return 2;
341 }
342
343 return leaves;
344}
345
346int init_of_cache_level(unsigned int cpu)
347{
348 struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
349 struct device_node *np __free(device_node) = of_cpu_device_node_get(cpu);
350 unsigned int levels = 0, leaves, level;
351
352 if (!of_check_cache_nodes(np)) {
353 return -ENOENT;
354 }
355
356 leaves = of_count_cache_leaves(np);
357 if (leaves > 0)
358 levels = 1;
359
360 while (1) {
361 struct device_node *prev __free(device_node) = np;
362 np = of_find_next_cache_node(np);
363 if (!np)
364 break;
365
366 if (!of_device_is_compatible(np, "cache"))
367 return -EINVAL;
368 if (of_property_read_u32(np, "cache-level", &level))
369 return -EINVAL;
370 if (level <= levels)
371 return -EINVAL;
372
373 leaves += of_count_cache_leaves(np);
374 levels = level;
375 }
376
377 this_cpu_ci->num_levels = levels;
378 this_cpu_ci->num_leaves = leaves;
379
380 return 0;
381}
382
383#else
384static inline int cache_setup_of_node(unsigned int cpu) { return 0; }
385int init_of_cache_level(unsigned int cpu) { return 0; }
386#endif
387
388int __weak cache_setup_acpi(unsigned int cpu)
389{
390 return -ENOTSUPP;
391}
392
393unsigned int coherency_max_size;
394
395static int cache_setup_properties(unsigned int cpu)
396{
397 int ret = 0;
398
399 if (of_have_populated_dt())
400 ret = cache_setup_of_node(cpu);
401 else if (!acpi_disabled)
402 ret = cache_setup_acpi(cpu);
403
404 /*
405 * No DT/ACPI cache nodes; fall back to arch-derived topology (e.g.
406 * arm64 CLIDR_EL1) and clear the error to avoid a spurious warning.
407 */
408 if (ret && use_arch_cache_info()) {
409 use_arch_info = true;
410 ret = 0;
411 }
412
413 return ret;
414}
415
416static int cache_shared_cpu_map_setup(unsigned int cpu)
417{
418 struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
419 struct cacheinfo *this_leaf, *sib_leaf;
420 unsigned int index, sib_index;
421 int ret = 0;
422
423 if (this_cpu_ci->cpu_map_populated)
424 return 0;
425
426 /*
427 * skip setting up cache properties if LLC is valid, just need
428 * to update the shared cpu_map if the cache attributes were
429 * populated early before all the cpus are brought online
430 */
431 if (!last_level_cache_is_valid(cpu) && !use_arch_info) {
432 ret = cache_setup_properties(cpu);
433 if (ret)
434 return ret;
435 }
436
437 for (index = 0; index < cache_leaves(cpu); index++) {
438 unsigned int i;
439
440 this_leaf = per_cpu_cacheinfo_idx(cpu, index);
441
442 cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map);
443 for_each_online_cpu(i) {
444 if (i == cpu || !per_cpu_cacheinfo(i))
445 continue;/* skip if itself or no cacheinfo */
446 for (sib_index = 0; sib_index < cache_leaves(i); sib_index++) {
447 sib_leaf = per_cpu_cacheinfo_idx(i, sib_index);
448
449 /*
450 * Comparing cache IDs only makes sense if the leaves
451 * belong to the same cache level of same type. Skip
452 * the check if level and type do not match.
453 */
454 if (sib_leaf->level != this_leaf->level ||
455 sib_leaf->type != this_leaf->type)
456 continue;
457
458 if (cache_leaves_are_shared(this_leaf, sib_leaf)) {
459 cpumask_set_cpu(cpu, &sib_leaf->shared_cpu_map);
460 cpumask_set_cpu(i, &this_leaf->shared_cpu_map);
461 break;
462 }
463 }
464 }
465 /* record the maximum cache line size */
466 if (this_leaf->coherency_line_size > coherency_max_size)
467 coherency_max_size = this_leaf->coherency_line_size;
468 }
469
470 /* shared_cpu_map is now populated for the cpu */
471 this_cpu_ci->cpu_map_populated = true;
472 return 0;
473}
474
475static void cache_shared_cpu_map_remove(unsigned int cpu)
476{
477 struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
478 struct cacheinfo *this_leaf, *sib_leaf;
479 unsigned int sibling, index, sib_index;
480
481 for (index = 0; index < cache_leaves(cpu); index++) {
482 this_leaf = per_cpu_cacheinfo_idx(cpu, index);
483 for_each_cpu(sibling, &this_leaf->shared_cpu_map) {
484 if (sibling == cpu || !per_cpu_cacheinfo(sibling))
485 continue;/* skip if itself or no cacheinfo */
486
487 for (sib_index = 0; sib_index < cache_leaves(sibling); sib_index++) {
488 sib_leaf = per_cpu_cacheinfo_idx(sibling, sib_index);
489
490 /*
491 * Comparing cache IDs only makes sense if the leaves
492 * belong to the same cache level of same type. Skip
493 * the check if level and type do not match.
494 */
495 if (sib_leaf->level != this_leaf->level ||
496 sib_leaf->type != this_leaf->type)
497 continue;
498
499 if (cache_leaves_are_shared(this_leaf, sib_leaf)) {
500 cpumask_clear_cpu(cpu, &sib_leaf->shared_cpu_map);
501 cpumask_clear_cpu(sibling, &this_leaf->shared_cpu_map);
502 break;
503 }
504 }
505 }
506 }
507
508 /* cpu is no longer populated in the shared map */
509 this_cpu_ci->cpu_map_populated = false;
510}
511
512static void free_cache_attributes(unsigned int cpu)
513{
514 if (!per_cpu_cacheinfo(cpu))
515 return;
516
517 cache_shared_cpu_map_remove(cpu);
518}
519
520int __weak early_cache_level(unsigned int cpu)
521{
522 return -ENOENT;
523}
524
525int __weak init_cache_level(unsigned int cpu)
526{
527 return -ENOENT;
528}
529
530int __weak populate_cache_leaves(unsigned int cpu)
531{
532 return -ENOENT;
533}
534
535static inline int allocate_cache_info(int cpu)
536{
537 per_cpu_cacheinfo(cpu) = kzalloc_objs(struct cacheinfo,
538 cache_leaves(cpu), GFP_ATOMIC);
539 if (!per_cpu_cacheinfo(cpu)) {
540 cache_leaves(cpu) = 0;
541 return -ENOMEM;
542 }
543
544 return 0;
545}
546
547int fetch_cache_info(unsigned int cpu)
548{
549 struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
550 unsigned int levels = 0, split_levels = 0;
551 int ret;
552
553 if (acpi_disabled) {
554 ret = init_of_cache_level(cpu);
555 } else {
556 ret = acpi_get_cache_info(cpu, &levels, &split_levels);
557 if (!ret) {
558 this_cpu_ci->num_levels = levels;
559 /*
560 * This assumes that:
561 * - there cannot be any split caches (data/instruction)
562 * above a unified cache
563 * - data/instruction caches come by pair
564 */
565 this_cpu_ci->num_leaves = levels + split_levels;
566 }
567 }
568
569 if (ret || !cache_leaves(cpu)) {
570 ret = early_cache_level(cpu);
571 if (ret)
572 return ret;
573
574 if (!cache_leaves(cpu))
575 return -ENOENT;
576
577 this_cpu_ci->early_ci_levels = true;
578 }
579
580 return allocate_cache_info(cpu);
581}
582
583static inline int init_level_allocate_ci(unsigned int cpu)
584{
585 unsigned int early_leaves = cache_leaves(cpu);
586
587 /* Since early initialization/allocation of the cacheinfo is allowed
588 * via fetch_cache_info() and this also gets called as CPU hotplug
589 * callbacks via cacheinfo_cpu_online, the init/alloc can be skipped
590 * as it will happen only once (the cacheinfo memory is never freed).
591 * Just populate the cacheinfo. However, if the cacheinfo has been
592 * allocated early through the arch-specific early_cache_level() call,
593 * there is a chance the info is wrong (this can happen on arm64). In
594 * that case, call init_cache_level() anyway to give the arch-specific
595 * code a chance to make things right.
596 */
597 if (per_cpu_cacheinfo(cpu) && !ci_cacheinfo(cpu)->early_ci_levels)
598 return 0;
599
600 if (init_cache_level(cpu) || !cache_leaves(cpu))
601 return -ENOENT;
602
603 /*
604 * Now that we have properly initialized the cache level info, make
605 * sure we don't try to do that again the next time we are called
606 * (e.g. as CPU hotplug callbacks).
607 */
608 ci_cacheinfo(cpu)->early_ci_levels = false;
609
610 /*
611 * Some architectures (e.g., x86) do not use early initialization.
612 * Allocate memory now in such case.
613 */
614 if (cache_leaves(cpu) <= early_leaves && per_cpu_cacheinfo(cpu))
615 return 0;
616
617 kfree(per_cpu_cacheinfo(cpu));
618 return allocate_cache_info(cpu);
619}
620
621int detect_cache_attributes(unsigned int cpu)
622{
623 int ret;
624
625 ret = init_level_allocate_ci(cpu);
626 if (ret)
627 return ret;
628
629 /*
630 * If LLC is valid the cache leaves were already populated so just go to
631 * update the cpu map.
632 */
633 if (!last_level_cache_is_valid(cpu)) {
634 /*
635 * populate_cache_leaves() may completely setup the cache leaves and
636 * shared_cpu_map or it may leave it partially setup.
637 */
638 ret = populate_cache_leaves(cpu);
639 if (ret)
640 goto free_ci;
641 }
642
643 /*
644 * For systems using DT for cache hierarchy, fw_token
645 * and shared_cpu_map will be set up here only if they are
646 * not populated already
647 */
648 ret = cache_shared_cpu_map_setup(cpu);
649 if (ret) {
650 pr_warn("Unable to detect cache hierarchy for CPU %d\n", cpu);
651 goto free_ci;
652 }
653
654 return 0;
655
656free_ci:
657 free_cache_attributes(cpu);
658 return ret;
659}
660
661/* pointer to cpuX/cache device */
662static DEFINE_PER_CPU(struct device *, ci_cache_dev);
663#define per_cpu_cache_dev(cpu) (per_cpu(ci_cache_dev, cpu))
664
665static cpumask_t cache_dev_map;
666
667/* pointer to array of devices for cpuX/cache/indexY */
668static DEFINE_PER_CPU(struct device **, ci_index_dev);
669#define per_cpu_index_dev(cpu) (per_cpu(ci_index_dev, cpu))
670#define per_cache_index_dev(cpu, idx) ((per_cpu_index_dev(cpu))[idx])
671
672#define show_one(file_name, object) \
673static ssize_t file_name##_show(struct device *dev, \
674 struct device_attribute *attr, char *buf) \
675{ \
676 struct cacheinfo *this_leaf = dev_get_drvdata(dev); \
677 return sysfs_emit(buf, "%u\n", this_leaf->object); \
678}
679
680show_one(id, id);
681show_one(level, level);
682show_one(coherency_line_size, coherency_line_size);
683show_one(number_of_sets, number_of_sets);
684show_one(physical_line_partition, physical_line_partition);
685show_one(ways_of_associativity, ways_of_associativity);
686
687static ssize_t size_show(struct device *dev,
688 struct device_attribute *attr, char *buf)
689{
690 struct cacheinfo *this_leaf = dev_get_drvdata(dev);
691
692 return sysfs_emit(buf, "%uK\n", this_leaf->size >> 10);
693}
694
695static ssize_t shared_cpu_map_show(struct device *dev,
696 struct device_attribute *attr, char *buf)
697{
698 struct cacheinfo *this_leaf = dev_get_drvdata(dev);
699 const struct cpumask *mask = &this_leaf->shared_cpu_map;
700
701 return sysfs_emit(buf, "%*pb\n", nr_cpu_ids, mask);
702}
703
704static ssize_t shared_cpu_list_show(struct device *dev,
705 struct device_attribute *attr, char *buf)
706{
707 struct cacheinfo *this_leaf = dev_get_drvdata(dev);
708 const struct cpumask *mask = &this_leaf->shared_cpu_map;
709
710 return sysfs_emit(buf, "%*pbl\n", nr_cpu_ids, mask);
711}
712
713static ssize_t type_show(struct device *dev,
714 struct device_attribute *attr, char *buf)
715{
716 struct cacheinfo *this_leaf = dev_get_drvdata(dev);
717 const char *output;
718
719 switch (this_leaf->type) {
720 case CACHE_TYPE_DATA:
721 output = "Data";
722 break;
723 case CACHE_TYPE_INST:
724 output = "Instruction";
725 break;
726 case CACHE_TYPE_UNIFIED:
727 output = "Unified";
728 break;
729 default:
730 return -EINVAL;
731 }
732
733 return sysfs_emit(buf, "%s\n", output);
734}
735
736static ssize_t allocation_policy_show(struct device *dev,
737 struct device_attribute *attr, char *buf)
738{
739 struct cacheinfo *this_leaf = dev_get_drvdata(dev);
740 unsigned int ci_attr = this_leaf->attributes;
741 const char *output;
742
743 if ((ci_attr & CACHE_READ_ALLOCATE) && (ci_attr & CACHE_WRITE_ALLOCATE))
744 output = "ReadWriteAllocate";
745 else if (ci_attr & CACHE_READ_ALLOCATE)
746 output = "ReadAllocate";
747 else if (ci_attr & CACHE_WRITE_ALLOCATE)
748 output = "WriteAllocate";
749 else
750 return 0;
751
752 return sysfs_emit(buf, "%s\n", output);
753}
754
755static ssize_t write_policy_show(struct device *dev,
756 struct device_attribute *attr, char *buf)
757{
758 struct cacheinfo *this_leaf = dev_get_drvdata(dev);
759 unsigned int ci_attr = this_leaf->attributes;
760 int n = 0;
761
762 if (ci_attr & CACHE_WRITE_THROUGH)
763 n = sysfs_emit(buf, "WriteThrough\n");
764 else if (ci_attr & CACHE_WRITE_BACK)
765 n = sysfs_emit(buf, "WriteBack\n");
766 return n;
767}
768
769static DEVICE_ATTR_RO(id);
770static DEVICE_ATTR_RO(level);
771static DEVICE_ATTR_RO(type);
772static DEVICE_ATTR_RO(coherency_line_size);
773static DEVICE_ATTR_RO(ways_of_associativity);
774static DEVICE_ATTR_RO(number_of_sets);
775static DEVICE_ATTR_RO(size);
776static DEVICE_ATTR_RO(allocation_policy);
777static DEVICE_ATTR_RO(write_policy);
778static DEVICE_ATTR_RO(shared_cpu_map);
779static DEVICE_ATTR_RO(shared_cpu_list);
780static DEVICE_ATTR_RO(physical_line_partition);
781
782static struct attribute *cache_default_attrs[] = {
783 &dev_attr_id.attr,
784 &dev_attr_type.attr,
785 &dev_attr_level.attr,
786 &dev_attr_shared_cpu_map.attr,
787 &dev_attr_shared_cpu_list.attr,
788 &dev_attr_coherency_line_size.attr,
789 &dev_attr_ways_of_associativity.attr,
790 &dev_attr_number_of_sets.attr,
791 &dev_attr_size.attr,
792 &dev_attr_allocation_policy.attr,
793 &dev_attr_write_policy.attr,
794 &dev_attr_physical_line_partition.attr,
795 NULL
796};
797
798static umode_t
799cache_default_attrs_is_visible(struct kobject *kobj,
800 struct attribute *attr, int unused)
801{
802 struct device *dev = kobj_to_dev(kobj);
803 struct cacheinfo *this_leaf = dev_get_drvdata(dev);
804 const struct cpumask *mask = &this_leaf->shared_cpu_map;
805 umode_t mode = attr->mode;
806
807 if ((attr == &dev_attr_id.attr) && (this_leaf->attributes & CACHE_ID))
808 return mode;
809 if ((attr == &dev_attr_type.attr) && this_leaf->type)
810 return mode;
811 if ((attr == &dev_attr_level.attr) && this_leaf->level)
812 return mode;
813 if ((attr == &dev_attr_shared_cpu_map.attr) && !cpumask_empty(mask))
814 return mode;
815 if ((attr == &dev_attr_shared_cpu_list.attr) && !cpumask_empty(mask))
816 return mode;
817 if ((attr == &dev_attr_coherency_line_size.attr) &&
818 this_leaf->coherency_line_size)
819 return mode;
820 if ((attr == &dev_attr_ways_of_associativity.attr) &&
821 this_leaf->size) /* allow 0 = full associativity */
822 return mode;
823 if ((attr == &dev_attr_number_of_sets.attr) &&
824 this_leaf->number_of_sets)
825 return mode;
826 if ((attr == &dev_attr_size.attr) && this_leaf->size)
827 return mode;
828 if ((attr == &dev_attr_write_policy.attr) &&
829 (this_leaf->attributes & CACHE_WRITE_POLICY_MASK))
830 return mode;
831 if ((attr == &dev_attr_allocation_policy.attr) &&
832 (this_leaf->attributes & CACHE_ALLOCATE_POLICY_MASK))
833 return mode;
834 if ((attr == &dev_attr_physical_line_partition.attr) &&
835 this_leaf->physical_line_partition)
836 return mode;
837
838 return 0;
839}
840
841static const struct attribute_group cache_default_group = {
842 .attrs = cache_default_attrs,
843 .is_visible = cache_default_attrs_is_visible,
844};
845
846static const struct attribute_group *cache_default_groups[] = {
847 &cache_default_group,
848 NULL,
849};
850
851static const struct attribute_group *cache_private_groups[] = {
852 &cache_default_group,
853 NULL, /* Place holder for private group */
854 NULL,
855};
856
857const struct attribute_group *
858__weak cache_get_priv_group(struct cacheinfo *this_leaf)
859{
860 return NULL;
861}
862
863static const struct attribute_group **
864cache_get_attribute_groups(struct cacheinfo *this_leaf)
865{
866 const struct attribute_group *priv_group =
867 cache_get_priv_group(this_leaf);
868
869 if (!priv_group)
870 return cache_default_groups;
871
872 if (!cache_private_groups[1])
873 cache_private_groups[1] = priv_group;
874
875 return cache_private_groups;
876}
877
878/* Add/Remove cache interface for CPU device */
879static void cpu_cache_sysfs_exit(unsigned int cpu)
880{
881 int i;
882 struct device *ci_dev;
883
884 if (per_cpu_index_dev(cpu)) {
885 for (i = 0; i < cache_leaves(cpu); i++) {
886 ci_dev = per_cache_index_dev(cpu, i);
887 if (!ci_dev)
888 continue;
889 device_unregister(ci_dev);
890 }
891 kfree(per_cpu_index_dev(cpu));
892 per_cpu_index_dev(cpu) = NULL;
893 }
894 device_unregister(per_cpu_cache_dev(cpu));
895 per_cpu_cache_dev(cpu) = NULL;
896}
897
898static int cpu_cache_sysfs_init(unsigned int cpu)
899{
900 struct device *dev = get_cpu_device(cpu);
901
902 if (per_cpu_cacheinfo(cpu) == NULL)
903 return -ENOENT;
904
905 per_cpu_cache_dev(cpu) = cpu_device_create(dev, NULL, NULL, "cache");
906 if (IS_ERR(per_cpu_cache_dev(cpu)))
907 return PTR_ERR(per_cpu_cache_dev(cpu));
908
909 /* Allocate all required memory */
910 per_cpu_index_dev(cpu) = kzalloc_objs(struct device *,
911 cache_leaves(cpu));
912 if (unlikely(per_cpu_index_dev(cpu) == NULL))
913 goto err_out;
914
915 return 0;
916
917err_out:
918 cpu_cache_sysfs_exit(cpu);
919 return -ENOMEM;
920}
921
922static int cache_add_dev(unsigned int cpu)
923{
924 unsigned int i;
925 int rc;
926 struct device *ci_dev, *parent;
927 struct cacheinfo *this_leaf;
928 const struct attribute_group **cache_groups;
929
930 rc = cpu_cache_sysfs_init(cpu);
931 if (unlikely(rc < 0))
932 return rc;
933
934 parent = per_cpu_cache_dev(cpu);
935 for (i = 0; i < cache_leaves(cpu); i++) {
936 this_leaf = per_cpu_cacheinfo_idx(cpu, i);
937 if (this_leaf->disable_sysfs)
938 continue;
939 if (this_leaf->type == CACHE_TYPE_NOCACHE)
940 break;
941 cache_groups = cache_get_attribute_groups(this_leaf);
942 ci_dev = cpu_device_create(parent, this_leaf, cache_groups,
943 "index%1u", i);
944 if (IS_ERR(ci_dev)) {
945 rc = PTR_ERR(ci_dev);
946 goto err;
947 }
948 per_cache_index_dev(cpu, i) = ci_dev;
949 }
950 cpumask_set_cpu(cpu, &cache_dev_map);
951
952 return 0;
953err:
954 cpu_cache_sysfs_exit(cpu);
955 return rc;
956}
957
958static unsigned int cpu_map_shared_cache(bool online, unsigned int cpu,
959 cpumask_t **map)
960{
961 struct cacheinfo *llc, *sib_llc;
962 unsigned int sibling;
963
964 if (!last_level_cache_is_valid(cpu))
965 return 0;
966
967 llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1);
968
969 if (llc->type != CACHE_TYPE_DATA && llc->type != CACHE_TYPE_UNIFIED)
970 return 0;
971
972 if (online) {
973 *map = &llc->shared_cpu_map;
974 return cpumask_weight(*map);
975 }
976
977 /* shared_cpu_map of offlined CPU will be cleared, so use sibling map */
978 for_each_cpu(sibling, &llc->shared_cpu_map) {
979 if (sibling == cpu || !last_level_cache_is_valid(sibling))
980 continue;
981 sib_llc = per_cpu_cacheinfo_idx(sibling, cache_leaves(sibling) - 1);
982 *map = &sib_llc->shared_cpu_map;
983 return cpumask_weight(*map);
984 }
985
986 return 0;
987}
988
989/*
990 * Calculate the size of the per-CPU data cache slice. This can be
991 * used to estimate the size of the data cache slice that can be used
992 * by one CPU under ideal circumstances. UNIFIED caches are counted
993 * in addition to DATA caches. So, please consider code cache usage
994 * when use the result.
995 *
996 * Because the cache inclusive/non-inclusive information isn't
997 * available, we just use the size of the per-CPU slice of LLC to make
998 * the result more predictable across architectures.
999 */
1000static void update_per_cpu_data_slice_size_cpu(unsigned int cpu)
1001{
1002 struct cpu_cacheinfo *ci;
1003 struct cacheinfo *llc;
1004 unsigned int nr_shared;
1005
1006 if (!last_level_cache_is_valid(cpu))
1007 return;
1008
1009 ci = ci_cacheinfo(cpu);
1010 llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1);
1011
1012 if (llc->type != CACHE_TYPE_DATA && llc->type != CACHE_TYPE_UNIFIED)
1013 return;
1014
1015 nr_shared = cpumask_weight(&llc->shared_cpu_map);
1016 if (nr_shared)
1017 ci->per_cpu_data_slice_size = llc->size / nr_shared;
1018}
1019
1020static void update_per_cpu_data_slice_size(bool cpu_online, unsigned int cpu,
1021 cpumask_t *cpu_map)
1022{
1023 unsigned int icpu;
1024
1025 for_each_cpu(icpu, cpu_map) {
1026 if (!cpu_online && icpu == cpu)
1027 continue;
1028 update_per_cpu_data_slice_size_cpu(icpu);
1029 setup_pcp_cacheinfo(icpu);
1030 }
1031}
1032
1033static int cacheinfo_cpu_online(unsigned int cpu)
1034{
1035 int rc = detect_cache_attributes(cpu);
1036 cpumask_t *cpu_map;
1037
1038 if (rc)
1039 return rc;
1040 rc = cache_add_dev(cpu);
1041 if (rc)
1042 goto err;
1043 if (cpu_map_shared_cache(true, cpu, &cpu_map)) {
1044 update_per_cpu_data_slice_size(true, cpu, cpu_map);
1045 sched_update_llc_bytes(cpu_map);
1046 }
1047 return 0;
1048err:
1049 free_cache_attributes(cpu);
1050 return rc;
1051}
1052
1053static int cacheinfo_cpu_pre_down(unsigned int cpu)
1054{
1055 cpumask_t *cpu_map;
1056 unsigned int nr_shared;
1057
1058 nr_shared = cpu_map_shared_cache(false, cpu, &cpu_map);
1059 if (cpumask_test_and_clear_cpu(cpu, &cache_dev_map))
1060 cpu_cache_sysfs_exit(cpu);
1061
1062 free_cache_attributes(cpu);
1063 if (nr_shared > 1) {
1064 update_per_cpu_data_slice_size(false, cpu, cpu_map);
1065 sched_update_llc_bytes(cpu_map);
1066 }
1067
1068 return 0;
1069}
1070
1071static int __init cacheinfo_sysfs_init(void)
1072{
1073 return cpuhp_setup_state(CPUHP_AP_BASE_CACHEINFO_ONLINE,
1074 "base/cacheinfo:online",
1075 cacheinfo_cpu_online, cacheinfo_cpu_pre_down);
1076}
1077device_initcall(cacheinfo_sysfs_init);