projects/linux·blob master · refs

478 lines · 12.1 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2#include <linux/blk_types.h>
3
4#include "cache.h"
5#include "cache_dev.h"
6#include "backing_dev.h"
7#include "dm_pcache.h"
8
9struct kmem_cache *key_cache;
10
11static inline struct pcache_cache_info *get_cache_info_addr(struct pcache_cache *cache)
12{
13 return (struct pcache_cache_info *)((char *)cache->cache_info_addr +
14 (size_t)cache->info_index * PCACHE_CACHE_INFO_SIZE);
15}
16
17static void cache_info_write(struct pcache_cache *cache)
18{
19 struct pcache_cache_info *cache_info = &cache->cache_info;
20
21 cache_info->header.seq++;
22 cache_info->header.crc = pcache_meta_crc(&cache_info->header,
23 sizeof(struct pcache_cache_info));
24
25 cache->info_index = (cache->info_index + 1) % PCACHE_META_INDEX_MAX;
26 memcpy_flushcache(get_cache_info_addr(cache), cache_info,
27 sizeof(struct pcache_cache_info));
28 pmem_wmb();
29}
30
31static void cache_info_init_default(struct pcache_cache *cache);
32static int cache_info_init(struct pcache_cache *cache, struct pcache_cache_options *opts)
33{
34 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
35 struct pcache_cache_info *cache_info_addr;
36
37 cache_info_addr = pcache_meta_find_latest(&cache->cache_info_addr->header,
38 sizeof(struct pcache_cache_info),
39 PCACHE_CACHE_INFO_SIZE,
40 &cache->cache_info);
41 if (IS_ERR(cache_info_addr))
42 return PTR_ERR(cache_info_addr);
43
44 if (cache_info_addr) {
45 if (opts->data_crc !=
46 (cache->cache_info.flags & PCACHE_CACHE_FLAGS_DATA_CRC)) {
47 pcache_dev_err(pcache, "invalid option for data_crc: %s, expected: %s",
48 opts->data_crc ? "true" : "false",
49 cache->cache_info.flags & PCACHE_CACHE_FLAGS_DATA_CRC ? "true" : "false");
50 return -EINVAL;
51 }
52
53 cache->info_index = ((char *)cache_info_addr - (char *)cache->cache_info_addr) / PCACHE_CACHE_INFO_SIZE;
54
55 return 0;
56 }
57
58 /* init cache_info for new cache */
59 cache_info_init_default(cache);
60 cache_mode_set(cache, opts->cache_mode);
61 if (opts->data_crc)
62 cache->cache_info.flags |= PCACHE_CACHE_FLAGS_DATA_CRC;
63
64 return 0;
65}
66
67static void cache_info_set_gc_percent(struct pcache_cache_info *cache_info, u8 percent)
68{
69 cache_info->flags &= ~PCACHE_CACHE_FLAGS_GC_PERCENT_MASK;
70 cache_info->flags |= FIELD_PREP(PCACHE_CACHE_FLAGS_GC_PERCENT_MASK, percent);
71}
72
73int pcache_cache_set_gc_percent(struct pcache_cache *cache, u8 percent)
74{
75 if (percent > PCACHE_CACHE_GC_PERCENT_MAX || percent < PCACHE_CACHE_GC_PERCENT_MIN)
76 return -EINVAL;
77
78 mutex_lock(&cache->cache_info_lock);
79 cache_info_set_gc_percent(&cache->cache_info, percent);
80
81 cache_info_write(cache);
82 mutex_unlock(&cache->cache_info_lock);
83
84 return 0;
85}
86
87void cache_pos_encode(struct pcache_cache *cache,
88 struct pcache_cache_pos_onmedia *pos_onmedia_base,
89 struct pcache_cache_pos *pos, u64 seq, u32 *index)
90{
91 struct pcache_cache_pos_onmedia pos_onmedia;
92 struct pcache_cache_pos_onmedia *pos_onmedia_addr = pos_onmedia_base + *index;
93
94 pos_onmedia.cache_seg_id = pos->cache_seg->cache_seg_id;
95 pos_onmedia.seg_off = pos->seg_off;
96 pos_onmedia.header.seq = seq;
97 pos_onmedia.header.crc = cache_pos_onmedia_crc(&pos_onmedia);
98
99 *index = (*index + 1) % PCACHE_META_INDEX_MAX;
100
101 memcpy_flushcache(pos_onmedia_addr, &pos_onmedia, sizeof(struct pcache_cache_pos_onmedia));
102 pmem_wmb();
103}
104
105int cache_pos_decode(struct pcache_cache *cache,
106 struct pcache_cache_pos_onmedia *pos_onmedia,
107 struct pcache_cache_pos *pos, u64 *seq, u32 *index)
108{
109 struct pcache_cache_pos_onmedia latest, *latest_addr;
110
111 latest_addr = pcache_meta_find_latest(&pos_onmedia->header,
112 sizeof(struct pcache_cache_pos_onmedia),
113 sizeof(struct pcache_cache_pos_onmedia),
114 &latest);
115 if (IS_ERR(latest_addr))
116 return PTR_ERR(latest_addr);
117
118 if (!latest_addr)
119 return -EIO;
120
121 if (!cache_seg_id_valid(cache, latest.cache_seg_id))
122 return -EIO;
123
124 pos->cache_seg = &cache->segments[latest.cache_seg_id];
125
126 if (latest.seg_off >= pos->cache_seg->segment.data_size)
127 return -EIO;
128
129 pos->seg_off = latest.seg_off;
130 *seq = latest.header.seq;
131 *index = (latest_addr - pos_onmedia);
132
133 return 0;
134}
135
136static inline void cache_info_set_seg_id(struct pcache_cache *cache, u32 seg_id)
137{
138 cache->cache_info.seg_id = seg_id;
139}
140
141static int cache_init(struct dm_pcache *pcache)
142{
143 struct pcache_cache *cache = &pcache->cache;
144 struct pcache_backing_dev *backing_dev = &pcache->backing_dev;
145 struct pcache_cache_dev *cache_dev = &pcache->cache_dev;
146 int ret;
147
148 cache->segments = kvzalloc_objs(struct pcache_cache_segment,
149 cache_dev->seg_num);
150 if (!cache->segments) {
151 ret = -ENOMEM;
152 goto err;
153 }
154
155 cache->seg_map = kvcalloc(BITS_TO_LONGS(cache_dev->seg_num), sizeof(unsigned long), GFP_KERNEL);
156 if (!cache->seg_map) {
157 ret = -ENOMEM;
158 goto free_segments;
159 }
160
161 cache->backing_dev = backing_dev;
162 cache->cache_dev = &pcache->cache_dev;
163 cache->n_segs = cache_dev->seg_num;
164 atomic_set(&cache->gc_errors, 0);
165 atomic_set(&cache->writeback_errors, 0);
166 spin_lock_init(&cache->seg_map_lock);
167 spin_lock_init(&cache->key_head_lock);
168
169 mutex_init(&cache->cache_info_lock);
170 mutex_init(&cache->key_tail_lock);
171 mutex_init(&cache->dirty_tail_lock);
172 mutex_init(&cache->writeback_lock);
173
174 INIT_DELAYED_WORK(&cache->writeback_work, cache_writeback_fn);
175 INIT_DELAYED_WORK(&cache->gc_work, pcache_cache_gc_fn);
176 INIT_WORK(&cache->clean_work, clean_fn);
177
178 return 0;
179
180free_segments:
181 kvfree(cache->segments);
182err:
183 return ret;
184}
185
186static void cache_exit(struct pcache_cache *cache)
187{
188 kvfree(cache->seg_map);
189 kvfree(cache->segments);
190}
191
192static void cache_info_init_default(struct pcache_cache *cache)
193{
194 struct pcache_cache_info *cache_info = &cache->cache_info;
195
196 memset(cache_info, 0, sizeof(*cache_info));
197 cache_info->n_segs = cache->cache_dev->seg_num;
198 cache_info_set_gc_percent(cache_info, PCACHE_CACHE_GC_PERCENT_DEFAULT);
199}
200
201static int cache_tail_init(struct pcache_cache *cache)
202{
203 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
204 bool new_cache = !(cache->cache_info.flags & PCACHE_CACHE_FLAGS_INIT_DONE);
205 int ret;
206
207 if (new_cache) {
208 __set_bit(0, cache->seg_map);
209
210 cache->key_head.cache_seg = &cache->segments[0];
211 cache->key_head.seg_off = 0;
212 cache_pos_copy(&cache->key_tail, &cache->key_head);
213 cache_pos_copy(&cache->dirty_tail, &cache->key_head);
214
215 cache_encode_dirty_tail(cache);
216 cache_encode_key_tail(cache);
217 } else {
218 if (cache_decode_key_tail(cache) || cache_decode_dirty_tail(cache)) {
219 pcache_dev_err(pcache, "Corrupted key tail or dirty tail.\n");
220 return -EIO;
221 }
222
223 ret = cache_verify_dirty_tail(cache);
224 if (ret) {
225 pcache_dev_err(pcache, "dirty tail chain does not terminate (crafted cache image?)\n");
226 return ret;
227 }
228 }
229
230 return 0;
231}
232
233static int get_seg_id(struct pcache_cache *cache,
234 struct pcache_cache_segment *prev_cache_seg,
235 bool new_cache, u32 *seg_id)
236{
237 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
238 struct pcache_cache_dev *cache_dev = cache->cache_dev;
239 int ret;
240
241 if (new_cache) {
242 ret = cache_dev_get_empty_segment_id(cache_dev, seg_id);
243 if (ret) {
244 pcache_dev_err(pcache, "no available segment\n");
245 goto err;
246 }
247
248 if (prev_cache_seg)
249 cache_seg_set_next_seg(prev_cache_seg, *seg_id);
250 else
251 cache_info_set_seg_id(cache, *seg_id);
252 } else {
253 if (prev_cache_seg) {
254 struct pcache_segment_info *prev_seg_info;
255
256 prev_seg_info = &prev_cache_seg->cache_seg_info;
257 if (!segment_info_has_next(prev_seg_info)) {
258 ret = -EFAULT;
259 goto err;
260 }
261 *seg_id = prev_cache_seg->cache_seg_info.next_seg;
262 } else {
263 *seg_id = cache->cache_info.seg_id;
264 }
265
266 if (*seg_id >= cache_dev->seg_num) {
267 pcache_dev_err(pcache, "invalid segment id %u from cache device (seg_num %u)\n",
268 *seg_id, cache_dev->seg_num);
269 ret = -EIO;
270 goto err;
271 }
272 }
273 return 0;
274err:
275 return ret;
276}
277
278static int cache_segs_init(struct pcache_cache *cache)
279{
280 struct pcache_cache_segment *prev_cache_seg = NULL;
281 struct pcache_cache_info *cache_info = &cache->cache_info;
282 bool new_cache = !(cache->cache_info.flags & PCACHE_CACHE_FLAGS_INIT_DONE);
283 u32 seg_id;
284 int ret;
285 u32 i;
286
287 if (cache_info->n_segs > cache->cache_dev->seg_num) {
288 pcache_dev_err(CACHE_TO_PCACHE(cache),
289 "cache_info n_segs %u exceeds cache device segments %u\n",
290 cache_info->n_segs, cache->cache_dev->seg_num);
291 return -EIO;
292 }
293
294 for (i = 0; i < cache_info->n_segs; i++) {
295 ret = get_seg_id(cache, prev_cache_seg, new_cache, &seg_id);
296 if (ret)
297 goto err;
298
299 ret = cache_seg_init(cache, seg_id, i, new_cache);
300 if (ret)
301 goto err;
302
303 prev_cache_seg = &cache->segments[i];
304 }
305 return 0;
306err:
307 return ret;
308}
309
310static int cache_init_req_keys(struct pcache_cache *cache, u32 n_paral)
311{
312 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
313 u32 n_subtrees;
314 int ret;
315 u32 i, cpu;
316
317 /* Calculate number of cache trees based on the device size */
318 n_subtrees = DIV_ROUND_UP(cache->dev_size << SECTOR_SHIFT, PCACHE_CACHE_SUBTREE_SIZE);
319 ret = cache_tree_init(cache, &cache->req_key_tree, n_subtrees);
320 if (ret)
321 goto err;
322
323 cache->n_ksets = n_paral;
324 cache->ksets = kvcalloc(cache->n_ksets, PCACHE_KSET_SIZE, GFP_KERNEL);
325 if (!cache->ksets) {
326 ret = -ENOMEM;
327 goto req_tree_exit;
328 }
329
330 /*
331 * Initialize each kset with a spinlock and delayed work for flushing.
332 * Each kset is associated with one queue to ensure independent handling
333 * of cache keys across multiple queues, maximizing multiqueue concurrency.
334 */
335 for (i = 0; i < cache->n_ksets; i++) {
336 struct pcache_cache_kset *kset = get_kset(cache, i);
337
338 kset->cache = cache;
339 spin_lock_init(&kset->kset_lock);
340 INIT_DELAYED_WORK(&kset->flush_work, kset_flush_fn);
341 }
342
343 cache->data_heads = alloc_percpu(struct pcache_cache_data_head);
344 if (!cache->data_heads) {
345 ret = -ENOMEM;
346 goto free_kset;
347 }
348
349 for_each_possible_cpu(cpu) {
350 struct pcache_cache_data_head *h =
351 per_cpu_ptr(cache->data_heads, cpu);
352 h->head_pos.cache_seg = NULL;
353 }
354
355 /*
356 * Replay persisted cache keys using cache_replay.
357 * This function loads and replays cache keys from previously stored
358 * ksets, allowing the cache to restore its state after a restart.
359 */
360 ret = cache_replay(cache);
361 if (ret) {
362 pcache_dev_err(pcache, "failed to replay keys\n");
363 goto free_heads;
364 }
365
366 return 0;
367
368free_heads:
369 free_percpu(cache->data_heads);
370free_kset:
371 kvfree(cache->ksets);
372req_tree_exit:
373 cache_tree_exit(&cache->req_key_tree);
374err:
375 return ret;
376}
377
378static void cache_destroy_req_keys(struct pcache_cache *cache)
379{
380 u32 i;
381
382 for (i = 0; i < cache->n_ksets; i++) {
383 struct pcache_cache_kset *kset = get_kset(cache, i);
384
385 cancel_delayed_work_sync(&kset->flush_work);
386 }
387
388 free_percpu(cache->data_heads);
389 kvfree(cache->ksets);
390 cache_tree_exit(&cache->req_key_tree);
391}
392
393int pcache_cache_start(struct dm_pcache *pcache)
394{
395 struct pcache_backing_dev *backing_dev = &pcache->backing_dev;
396 struct pcache_cache *cache = &pcache->cache;
397 struct pcache_cache_options *opts = &pcache->opts;
398 int ret;
399
400 ret = cache_init(pcache);
401 if (ret)
402 return ret;
403
404 cache->cache_info_addr = CACHE_DEV_CACHE_INFO(cache->cache_dev);
405 cache->cache_ctrl = CACHE_DEV_CACHE_CTRL(cache->cache_dev);
406 backing_dev->cache = cache;
407 cache->dev_size = backing_dev->dev_size;
408
409 ret = cache_info_init(cache, opts);
410 if (ret)
411 goto cache_exit;
412
413 ret = cache_segs_init(cache);
414 if (ret)
415 goto cache_exit;
416
417 ret = cache_tail_init(cache);
418 if (ret)
419 goto cache_exit;
420
421 ret = cache_init_req_keys(cache, num_online_cpus());
422 if (ret)
423 goto cache_exit;
424
425 ret = cache_writeback_init(cache);
426 if (ret)
427 goto destroy_keys;
428
429 cache->cache_info.flags |= PCACHE_CACHE_FLAGS_INIT_DONE;
430 cache_info_write(cache);
431 queue_delayed_work(cache_get_wq(cache), &cache->gc_work, 0);
432
433 return 0;
434
435destroy_keys:
436 cache_destroy_req_keys(cache);
437cache_exit:
438 cache_exit(cache);
439
440 return ret;
441}
442
443void pcache_cache_stop(struct dm_pcache *pcache)
444{
445 struct pcache_cache *cache = &pcache->cache;
446
447 pcache_cache_flush(cache);
448
449 cancel_delayed_work_sync(&cache->gc_work);
450 flush_work(&cache->clean_work);
451 cache_writeback_exit(cache);
452
453 if (cache->req_key_tree.n_subtrees)
454 cache_destroy_req_keys(cache);
455
456 cache_exit(cache);
457}
458
459struct workqueue_struct *cache_get_wq(struct pcache_cache *cache)
460{
461 struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
462
463 return pcache->task_wq;
464}
465
466int pcache_cache_init(void)
467{
468 key_cache = KMEM_CACHE(pcache_cache_key, 0);
469 if (!key_cache)
470 return -ENOMEM;
471
472 return 0;
473}
474
475void pcache_cache_exit(void)
476{
477 kmem_cache_destroy(key_cache);
478}