projects/linux·blob master · refs

399 lines · 8.0 KiB

1// SPDX-License-Identifier: GPL-2.0
2//
3// Register cache access API - maple tree based cache
4//
5// Copyright 2023 Arm, Ltd
6//
7// Author: Mark Brown <[email protected]>
8
9#include <linux/debugfs.h>
10#include <linux/device.h>
11#include <linux/maple_tree.h>
12#include <linux/slab.h>
13
14#include "internal.h"
15
16static int regcache_maple_read(struct regmap *map,
17 unsigned int reg, unsigned int *value)
18{
19 struct maple_tree *mt = map->cache;
20 MA_STATE(mas, mt, reg, reg);
21 unsigned long *entry;
22
23 rcu_read_lock();
24
25 entry = mas_walk(&mas);
26 if (!entry) {
27 rcu_read_unlock();
28 return -ENOENT;
29 }
30
31 *value = entry[reg - mas.index];
32
33 rcu_read_unlock();
34
35 return 0;
36}
37
38static int regcache_maple_write(struct regmap *map, unsigned int reg,
39 unsigned int val)
40{
41 struct maple_tree *mt = map->cache;
42 MA_STATE(mas, mt, reg, reg);
43 unsigned long *entry, *upper, *lower;
44 unsigned long index, last;
45 size_t lower_sz, upper_sz;
46 int ret;
47
48 rcu_read_lock();
49
50 entry = mas_walk(&mas);
51 if (entry) {
52 entry[reg - mas.index] = val;
53 rcu_read_unlock();
54 return 0;
55 }
56
57 /* Any adjacent entries to extend/merge? */
58 mas_set_range(&mas, reg - 1, reg + 1);
59 index = reg;
60 last = reg;
61
62 lower = mas_find(&mas, reg - 1);
63 if (lower) {
64 index = mas.index;
65 lower_sz = (mas.last - mas.index + 1) * sizeof(unsigned long);
66 }
67
68 upper = mas_find(&mas, reg + 1);
69 if (upper) {
70 last = mas.last;
71 upper_sz = (mas.last - mas.index + 1) * sizeof(unsigned long);
72 }
73
74 rcu_read_unlock();
75
76 entry = kmalloc_array(last - index + 1, sizeof(*entry), map->alloc_flags);
77 if (!entry)
78 return -ENOMEM;
79
80 if (lower)
81 memcpy(entry, lower, lower_sz);
82 entry[reg - index] = val;
83 if (upper)
84 memcpy(&entry[reg - index + 1], upper, upper_sz);
85
86 /*
87 * This is safe because the regmap lock means the Maple lock
88 * is redundant, but we need to take it due to lockdep asserts
89 * in the maple tree code.
90 */
91 mas_lock(&mas);
92
93 mas_set_range(&mas, index, last);
94 ret = mas_store_gfp(&mas, entry, map->alloc_flags);
95
96 mas_unlock(&mas);
97
98 if (ret) {
99 kfree(entry);
100 return ret;
101 }
102 kfree(lower);
103 kfree(upper);
104 return 0;
105}
106
107static int regcache_maple_drop(struct regmap *map, unsigned int min,
108 unsigned int max)
109{
110 struct maple_tree *mt = map->cache;
111 MA_STATE(mas, mt, min, max);
112 unsigned long *entry, *lower, *upper;
113 /* initialized to work around false-positive -Wuninitialized warning */
114 unsigned long lower_index = 0, lower_last = 0;
115 unsigned long upper_index = 0, upper_last = 0;
116 int ret = 0;
117
118 lower = NULL;
119 upper = NULL;
120
121 mas_lock(&mas);
122
123 mas_for_each(&mas, entry, max) {
124 /*
125 * This is safe because the regmap lock means the
126 * Maple lock is redundant, but we need to take it due
127 * to lockdep asserts in the maple tree code.
128 */
129 mas_unlock(&mas);
130
131 /* Do we need to save any of this entry? */
132 if (mas.index < min) {
133 lower_index = mas.index;
134 lower_last = min -1;
135
136 lower = kmemdup_array(entry,
137 min - mas.index, sizeof(*lower),
138 map->alloc_flags);
139 if (!lower) {
140 ret = -ENOMEM;
141 goto out_unlocked;
142 }
143 }
144
145 if (mas.last > max) {
146 upper_index = max + 1;
147 upper_last = mas.last;
148
149 upper = kmemdup_array(&entry[max - mas.index + 1],
150 mas.last - max, sizeof(*upper),
151 map->alloc_flags);
152 if (!upper) {
153 ret = -ENOMEM;
154 goto out_unlocked;
155 }
156 }
157
158 kfree(entry);
159 mas_lock(&mas);
160 mas_erase(&mas);
161
162 /* Insert new nodes with the saved data */
163 if (lower) {
164 mas_set_range(&mas, lower_index, lower_last);
165 ret = mas_store_gfp(&mas, lower, map->alloc_flags);
166 if (ret != 0)
167 goto out;
168 lower = NULL;
169 }
170
171 if (upper) {
172 mas_set_range(&mas, upper_index, upper_last);
173 ret = mas_store_gfp(&mas, upper, map->alloc_flags);
174 if (ret != 0)
175 goto out;
176 upper = NULL;
177 }
178 }
179
180out:
181 mas_unlock(&mas);
182out_unlocked:
183 kfree(lower);
184 kfree(upper);
185
186 return ret;
187}
188
189static int regcache_maple_sync_block(struct regmap *map, unsigned long *entry,
190 struct ma_state *mas,
191 unsigned int min, unsigned int max)
192{
193 void *buf;
194 unsigned long r;
195 size_t val_bytes = map->format.val_bytes;
196 int ret = 0;
197
198 mas_pause(mas);
199 rcu_read_unlock();
200
201 /*
202 * Use a raw write if writing more than one register to a
203 * device that supports raw writes to reduce transaction
204 * overheads.
205 */
206 if (max - min > 1 && regmap_can_raw_write(map)) {
207 buf = kmalloc_array(max - min, val_bytes, map->alloc_flags);
208 if (!buf) {
209 ret = -ENOMEM;
210 goto out;
211 }
212
213 /* Render the data for a raw write */
214 for (r = min; r < max; r++) {
215 regcache_set_val(map, buf, r - min,
216 entry[r - mas->index]);
217 }
218
219 ret = _regmap_raw_write(map, min, buf, (max - min) * val_bytes,
220 false);
221
222 kfree(buf);
223 } else {
224 for (r = min; r < max; r++) {
225 ret = _regmap_write(map, r,
226 entry[r - mas->index]);
227 if (ret != 0)
228 goto out;
229 }
230 }
231
232out:
233 rcu_read_lock();
234
235 return ret;
236}
237
238static int regcache_maple_sync(struct regmap *map, unsigned int min,
239 unsigned int max)
240{
241 struct maple_tree *mt = map->cache;
242 unsigned long *entry;
243 MA_STATE(mas, mt, min, max);
244 unsigned long lmin = min;
245 unsigned long lmax = max;
246 unsigned int r, v, sync_start;
247 int ret = 0;
248 bool sync_needed = false;
249
250 map->cache_bypass = true;
251
252 rcu_read_lock();
253
254 mas_for_each(&mas, entry, max) {
255 for (r = max(mas.index, lmin); r <= min(mas.last, lmax); r++) {
256 v = entry[r - mas.index];
257
258 if (regcache_reg_needs_sync(map, r, v)) {
259 if (!sync_needed) {
260 sync_start = r;
261 sync_needed = true;
262 }
263 continue;
264 }
265
266 if (!sync_needed)
267 continue;
268
269 ret = regcache_maple_sync_block(map, entry, &mas,
270 sync_start, r);
271 if (ret != 0)
272 goto out;
273 sync_needed = false;
274 }
275
276 if (sync_needed) {
277 ret = regcache_maple_sync_block(map, entry, &mas,
278 sync_start, r);
279 if (ret != 0)
280 goto out;
281 sync_needed = false;
282 }
283 }
284
285out:
286 rcu_read_unlock();
287
288 map->cache_bypass = false;
289
290 return ret;
291}
292
293static int regcache_maple_init(struct regmap *map)
294{
295 struct maple_tree *mt;
296
297 mt = kmalloc_obj(*mt, map->alloc_flags);
298 if (!mt)
299 return -ENOMEM;
300 map->cache = mt;
301
302 mt_init(mt);
303
304 if (!mt_external_lock(mt) && map->lock_key)
305 lockdep_set_class_and_subclass(&mt->ma_lock, map->lock_key, 1);
306
307 return 0;
308}
309
310static void regcache_maple_exit(struct regmap *map)
311{
312 struct maple_tree *mt = map->cache;
313 MA_STATE(mas, mt, 0, UINT_MAX);
314 unsigned int *entry;
315 unsigned long flags;
316
317 /* if we've already been called then just return */
318 if (!mt)
319 return;
320
321 /*
322 * Register access may take the Maple lock from hard IRQ context, so
323 * preserve its IRQ-safe lockdep state during teardown.
324 */
325 mas_lock_irqsave(&mas, flags);
326 mas_for_each(&mas, entry, UINT_MAX)
327 kfree(entry);
328 __mt_destroy(mt);
329 mas_unlock_irqrestore(&mas, flags);
330
331 kfree(mt);
332 map->cache = NULL;
333}
334
335static int regcache_maple_insert_block(struct regmap *map, int first,
336 int last)
337{
338 struct maple_tree *mt = map->cache;
339 MA_STATE(mas, mt, first, last);
340 unsigned long *entry;
341 int i, ret;
342
343 entry = kmalloc_array(last - first + 1, sizeof(*entry), map->alloc_flags);
344 if (!entry)
345 return -ENOMEM;
346
347 for (i = 0; i < last - first + 1; i++)
348 entry[i] = map->reg_defaults[first + i].def;
349
350 mas_lock(&mas);
351
352 mas_set_range(&mas, map->reg_defaults[first].reg,
353 map->reg_defaults[last].reg);
354 ret = mas_store_gfp(&mas, entry, map->alloc_flags);
355
356 mas_unlock(&mas);
357
358 if (ret)
359 kfree(entry);
360
361 return ret;
362}
363
364static int regcache_maple_populate(struct regmap *map)
365{
366 int i;
367 int ret;
368 int range_start;
369
370 range_start = 0;
371
372 /* Scan for ranges of contiguous registers */
373 for (i = 1; i < map->num_reg_defaults; i++) {
374 if (map->reg_defaults[i].reg !=
375 map->reg_defaults[i - 1].reg + 1) {
376 ret = regcache_maple_insert_block(map, range_start,
377 i - 1);
378 if (ret != 0)
379 return ret;
380
381 range_start = i;
382 }
383 }
384
385 /* Add the last block */
386 return regcache_maple_insert_block(map, range_start, map->num_reg_defaults - 1);
387}
388
389struct regcache_ops regcache_maple_ops = {
390 .type = REGCACHE_MAPLE,
391 .name = "maple",
392 .init = regcache_maple_init,
393 .exit = regcache_maple_exit,
394 .populate = regcache_maple_populate,
395 .read = regcache_maple_read,
396 .write = regcache_maple_write,
397 .drop = regcache_maple_drop,
398 .sync = regcache_maple_sync,
399};