projects/linux·blob master · refs

354 lines · 9.4 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * SiFive composable cache controller Driver
4 *
5 * Copyright (C) 2018-2022 SiFive, Inc.
6 *
7 */
8
9#define pr_fmt(fmt) "CCACHE: " fmt
10
11#include <linux/align.h>
12#include <linux/debugfs.h>
13#include <linux/interrupt.h>
14#include <linux/of_irq.h>
15#include <linux/of_address.h>
16#include <linux/device.h>
17#include <linux/bitfield.h>
18#include <linux/platform_device.h>
19#include <linux/property.h>
20#include <asm/cacheflush.h>
21#include <asm/cacheinfo.h>
22#include <asm/dma-noncoherent.h>
23#include <soc/sifive/sifive_ccache.h>
24
25#define SIFIVE_CCACHE_DIRECCFIX_LOW 0x100
26#define SIFIVE_CCACHE_DIRECCFIX_HIGH 0x104
27#define SIFIVE_CCACHE_DIRECCFIX_COUNT 0x108
28
29#define SIFIVE_CCACHE_DIRECCFAIL_LOW 0x120
30#define SIFIVE_CCACHE_DIRECCFAIL_HIGH 0x124
31#define SIFIVE_CCACHE_DIRECCFAIL_COUNT 0x128
32
33#define SIFIVE_CCACHE_DATECCFIX_LOW 0x140
34#define SIFIVE_CCACHE_DATECCFIX_HIGH 0x144
35#define SIFIVE_CCACHE_DATECCFIX_COUNT 0x148
36
37#define SIFIVE_CCACHE_DATECCFAIL_LOW 0x160
38#define SIFIVE_CCACHE_DATECCFAIL_HIGH 0x164
39#define SIFIVE_CCACHE_DATECCFAIL_COUNT 0x168
40
41#define SIFIVE_CCACHE_CONFIG 0x00
42#define SIFIVE_CCACHE_CONFIG_BANK_MASK GENMASK_ULL(7, 0)
43#define SIFIVE_CCACHE_CONFIG_WAYS_MASK GENMASK_ULL(15, 8)
44#define SIFIVE_CCACHE_CONFIG_SETS_MASK GENMASK_ULL(23, 16)
45#define SIFIVE_CCACHE_CONFIG_BLKS_MASK GENMASK_ULL(31, 24)
46
47#define SIFIVE_CCACHE_FLUSH64 0x200
48#define SIFIVE_CCACHE_FLUSH32 0x240
49
50#define SIFIVE_CCACHE_WAYENABLE 0x08
51#define SIFIVE_CCACHE_ECCINJECTERR 0x40
52
53#define SIFIVE_CCACHE_MAX_ECCINTR 4
54#define SIFIVE_CCACHE_LINE_SIZE 64
55
56static void __iomem *ccache_base;
57static int g_irq[SIFIVE_CCACHE_MAX_ECCINTR];
58static struct riscv_cacheinfo_ops ccache_cache_ops;
59static int level;
60
61enum {
62 DIR_CORR = 0,
63 DATA_CORR,
64 DATA_UNCORR,
65 DIR_UNCORR,
66};
67
68enum {
69 QUIRK_NONSTANDARD_CACHE_OPS = BIT(0),
70 QUIRK_BROKEN_DATA_UNCORR = BIT(1),
71};
72
73#ifdef CONFIG_DEBUG_FS
74static struct dentry *sifive_test;
75
76static ssize_t ccache_write(struct file *file, const char __user *data,
77 size_t count, loff_t *ppos)
78{
79 unsigned int val;
80
81 if (kstrtouint_from_user(data, count, 0, &val))
82 return -EINVAL;
83 if ((val < 0xFF) || (val >= 0x10000 && val < 0x100FF))
84 writel(val, ccache_base + SIFIVE_CCACHE_ECCINJECTERR);
85 else
86 return -EINVAL;
87 return count;
88}
89
90static const struct file_operations ccache_fops = {
91 .owner = THIS_MODULE,
92 .open = simple_open,
93 .write = ccache_write
94};
95
96static void setup_sifive_debug(void)
97{
98 sifive_test = debugfs_create_dir("sifive_ccache_cache", NULL);
99
100 debugfs_create_file("sifive_debug_inject_error", 0200,
101 sifive_test, NULL, &ccache_fops);
102}
103#endif
104
105static void ccache_config_read(void)
106{
107 u32 cfg;
108
109 cfg = readl(ccache_base + SIFIVE_CCACHE_CONFIG);
110 pr_info("%llu banks, %llu ways, sets/bank=%llu, bytes/block=%llu\n",
111 FIELD_GET(SIFIVE_CCACHE_CONFIG_BANK_MASK, cfg),
112 FIELD_GET(SIFIVE_CCACHE_CONFIG_WAYS_MASK, cfg),
113 BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_SETS_MASK, cfg)),
114 BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_BLKS_MASK, cfg)));
115
116 cfg = readl(ccache_base + SIFIVE_CCACHE_WAYENABLE);
117 pr_info("Index of the largest way enabled: %u\n", cfg);
118}
119
120static const struct of_device_id sifive_ccache_ids[] = {
121 { .compatible = "eswin,eic7700-l3-cache",
122 .data = (void *)(QUIRK_NONSTANDARD_CACHE_OPS) },
123 { .compatible = "sifive,fu540-c000-ccache" },
124 { .compatible = "sifive,fu740-c000-ccache" },
125 { .compatible = "starfive,jh7100-ccache",
126 .data = (void *)(QUIRK_NONSTANDARD_CACHE_OPS | QUIRK_BROKEN_DATA_UNCORR) },
127 { .compatible = "starfive,jh7110-ccache",
128 .data = (void *)(QUIRK_NONSTANDARD_CACHE_OPS) },
129 { .compatible = "sifive,ccache0" },
130 { /* end of table */ }
131};
132
133static ATOMIC_NOTIFIER_HEAD(ccache_err_chain);
134
135int register_sifive_ccache_error_notifier(struct notifier_block *nb)
136{
137 return atomic_notifier_chain_register(&ccache_err_chain, nb);
138}
139EXPORT_SYMBOL_GPL(register_sifive_ccache_error_notifier);
140
141int unregister_sifive_ccache_error_notifier(struct notifier_block *nb)
142{
143 return atomic_notifier_chain_unregister(&ccache_err_chain, nb);
144}
145EXPORT_SYMBOL_GPL(unregister_sifive_ccache_error_notifier);
146
147#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
148static void ccache_flush_range(phys_addr_t start, size_t len)
149{
150 phys_addr_t end = start + len;
151 phys_addr_t line;
152
153 if (!len)
154 return;
155
156 mb(); /* complete earlier memory accesses before the cache flush */
157 for (line = ALIGN_DOWN(start, SIFIVE_CCACHE_LINE_SIZE); line < end;
158 line += SIFIVE_CCACHE_LINE_SIZE) {
159#ifdef CONFIG_32BIT
160 writel_relaxed(line >> 4, ccache_base + SIFIVE_CCACHE_FLUSH32);
161#else
162 writeq_relaxed(line, ccache_base + SIFIVE_CCACHE_FLUSH64);
163#endif
164 }
165 mb(); /* issue later memory accesses after the cache flush */
166}
167
168static const struct riscv_nonstd_cache_ops ccache_mgmt_ops __initconst = {
169 .wback = &ccache_flush_range,
170 .inv = &ccache_flush_range,
171 .wback_inv = &ccache_flush_range,
172};
173#endif /* CONFIG_RISCV_NONSTANDARD_CACHE_OPS */
174
175static int ccache_largest_wayenabled(void)
176{
177 return readl(ccache_base + SIFIVE_CCACHE_WAYENABLE) & 0xFF;
178}
179
180static ssize_t number_of_ways_enabled_show(struct device *dev,
181 struct device_attribute *attr,
182 char *buf)
183{
184 return sprintf(buf, "%u\n", ccache_largest_wayenabled());
185}
186
187static DEVICE_ATTR_RO(number_of_ways_enabled);
188
189static struct attribute *priv_attrs[] = {
190 &dev_attr_number_of_ways_enabled.attr,
191 NULL,
192};
193
194static const struct attribute_group priv_attr_group = {
195 .attrs = priv_attrs,
196};
197
198static const struct attribute_group *ccache_get_priv_group(struct cacheinfo
199 *this_leaf)
200{
201 /* We want to use private group for composable cache only */
202 if (this_leaf->level == level)
203 return &priv_attr_group;
204 else
205 return NULL;
206}
207
208static irqreturn_t ccache_int_handler(int irq, void *device)
209{
210 unsigned int add_h, add_l;
211
212 if (irq == g_irq[DIR_CORR]) {
213 add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_HIGH);
214 add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_LOW);
215 pr_err("DirError @ 0x%08X.%08X\n", add_h, add_l);
216 /* Reading this register clears the DirError interrupt sig */
217 readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_COUNT);
218 atomic_notifier_call_chain(&ccache_err_chain,
219 SIFIVE_CCACHE_ERR_TYPE_CE,
220 "DirECCFix");
221 }
222 if (irq == g_irq[DIR_UNCORR]) {
223 add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_HIGH);
224 add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_LOW);
225 /* Reading this register clears the DirFail interrupt sig */
226 readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_COUNT);
227 atomic_notifier_call_chain(&ccache_err_chain,
228 SIFIVE_CCACHE_ERR_TYPE_UE,
229 "DirECCFail");
230 panic("CCACHE: DirFail @ 0x%08X.%08X\n", add_h, add_l);
231 }
232 if (irq == g_irq[DATA_CORR]) {
233 add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_HIGH);
234 add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_LOW);
235 pr_err("DataError @ 0x%08X.%08X\n", add_h, add_l);
236 /* Reading this register clears the DataError interrupt sig */
237 readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_COUNT);
238 atomic_notifier_call_chain(&ccache_err_chain,
239 SIFIVE_CCACHE_ERR_TYPE_CE,
240 "DatECCFix");
241 }
242 if (irq == g_irq[DATA_UNCORR]) {
243 add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_HIGH);
244 add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_LOW);
245 pr_err("DataFail @ 0x%08X.%08X\n", add_h, add_l);
246 /* Reading this register clears the DataFail interrupt sig */
247 readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_COUNT);
248 atomic_notifier_call_chain(&ccache_err_chain,
249 SIFIVE_CCACHE_ERR_TYPE_UE,
250 "DatECCFail");
251 }
252
253 return IRQ_HANDLED;
254}
255
256static int sifive_ccache_probe(struct platform_device *pdev)
257{
258 struct device *dev = &pdev->dev;
259 unsigned long quirks;
260 int intr_num, rc;
261
262 quirks = (unsigned long)device_get_match_data(dev);
263
264 intr_num = platform_irq_count(pdev);
265 if (!intr_num)
266 return dev_err_probe(dev, -ENODEV, "No interrupts property\n");
267
268 for (int i = 0; i < intr_num; i++) {
269 if (i == DATA_UNCORR && (quirks & QUIRK_BROKEN_DATA_UNCORR))
270 continue;
271
272 g_irq[i] = platform_get_irq(pdev, i);
273 if (g_irq[i] < 0)
274 return g_irq[i];
275
276 rc = devm_request_irq(dev, g_irq[i], ccache_int_handler, 0, "ccache_ecc", NULL);
277 if (rc)
278 return dev_err_probe(dev, rc, "Could not request IRQ %d\n", g_irq[i]);
279 }
280
281 return 0;
282}
283
284static struct platform_driver sifive_ccache_driver = {
285 .probe = sifive_ccache_probe,
286 .driver = {
287 .name = "sifive_ccache",
288 .of_match_table = sifive_ccache_ids,
289 },
290};
291
292static int __init sifive_ccache_init(void)
293{
294 struct device_node *np;
295 struct resource res;
296 const struct of_device_id *match;
297 unsigned long quirks __maybe_unused;
298 int rc;
299
300 np = of_find_matching_node_and_match(NULL, sifive_ccache_ids, &match);
301 if (!np)
302 return -ENODEV;
303
304 quirks = (uintptr_t)match->data;
305
306 if (of_address_to_resource(np, 0, &res)) {
307 rc = -ENODEV;
308 goto err_node_put;
309 }
310
311 ccache_base = ioremap(res.start, resource_size(&res));
312 if (!ccache_base) {
313 rc = -ENOMEM;
314 goto err_node_put;
315 }
316
317 if (of_property_read_u32(np, "cache-level", &level)) {
318 rc = -ENOENT;
319 goto err_unmap;
320 }
321
322#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
323 if (quirks & QUIRK_NONSTANDARD_CACHE_OPS) {
324 riscv_cbom_block_size = SIFIVE_CCACHE_LINE_SIZE;
325 riscv_noncoherent_supported();
326 riscv_noncoherent_register_cache_ops(&ccache_mgmt_ops);
327 }
328#endif
329
330 ccache_config_read();
331
332 ccache_cache_ops.get_priv_group = ccache_get_priv_group;
333 riscv_set_cacheinfo_ops(&ccache_cache_ops);
334
335#ifdef CONFIG_DEBUG_FS
336 setup_sifive_debug();
337#endif
338
339 rc = platform_driver_register(&sifive_ccache_driver);
340 if (rc)
341 goto err_unmap;
342
343 of_node_put(np);
344
345 return 0;
346
347err_unmap:
348 iounmap(ccache_base);
349err_node_put:
350 of_node_put(np);
351 return rc;
352}
353
354arch_initcall(sifive_ccache_init);