projects/linux·blob master · refs

505 lines · 11.2 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2#include <linux/module.h>
3#include <linux/blkdev.h>
4#include <linux/bio.h>
5
6#include "../dm-core.h"
7#include "cache_dev.h"
8#include "backing_dev.h"
9#include "cache.h"
10#include "dm_pcache.h"
11
12void pcache_defer_reqs_kick(struct dm_pcache *pcache)
13{
14 struct pcache_cache *cache = &pcache->cache;
15
16 spin_lock(&cache->seg_map_lock);
17 if (!cache->cache_full)
18 queue_work(pcache->task_wq, &pcache->defered_req_work);
19 spin_unlock(&cache->seg_map_lock);
20}
21
22static void defer_req(struct pcache_request *pcache_req)
23{
24 struct dm_pcache *pcache = pcache_req->pcache;
25
26 BUG_ON(!list_empty(&pcache_req->list_node));
27
28 spin_lock(&pcache->defered_req_list_lock);
29 list_add(&pcache_req->list_node, &pcache->defered_req_list);
30 pcache_defer_reqs_kick(pcache);
31 spin_unlock(&pcache->defered_req_list_lock);
32}
33
34static void defered_req_fn(struct work_struct *work)
35{
36 struct dm_pcache *pcache = container_of(work, struct dm_pcache, defered_req_work);
37 struct pcache_request *pcache_req;
38 LIST_HEAD(tmp_list);
39 int ret;
40
41 if (pcache_is_stopping(pcache))
42 return;
43
44 spin_lock(&pcache->defered_req_list_lock);
45 list_splice_init(&pcache->defered_req_list, &tmp_list);
46 spin_unlock(&pcache->defered_req_list_lock);
47
48 while (!list_empty(&tmp_list)) {
49 pcache_req = list_first_entry(&tmp_list,
50 struct pcache_request, list_node);
51 list_del_init(&pcache_req->list_node);
52 pcache_req->ret = 0;
53 ret = pcache_cache_handle_req(&pcache->cache, pcache_req);
54 if (ret == -EBUSY)
55 defer_req(pcache_req);
56 else
57 pcache_req_put(pcache_req, ret);
58 }
59}
60
61void pcache_req_get(struct pcache_request *pcache_req)
62{
63 kref_get(&pcache_req->ref);
64}
65
66static void end_req(struct kref *ref)
67{
68 struct pcache_request *pcache_req = container_of(ref, struct pcache_request, ref);
69 struct dm_pcache *pcache = pcache_req->pcache;
70 struct bio *bio = pcache_req->bio;
71 int ret = pcache_req->ret;
72
73 if (ret == -EBUSY) {
74 pcache_req_get(pcache_req);
75 defer_req(pcache_req);
76 } else {
77 bio->bi_status = errno_to_blk_status(ret);
78 bio_endio(bio);
79
80 if (atomic_dec_and_test(&pcache->inflight_reqs))
81 wake_up(&pcache->inflight_wq);
82 }
83}
84
85void pcache_req_put(struct pcache_request *pcache_req, int ret)
86{
87 /* Set the return status if it is not already set */
88 if (ret && !pcache_req->ret)
89 pcache_req->ret = ret;
90
91 kref_put(&pcache_req->ref, end_req);
92}
93
94static bool at_least_one_arg(struct dm_arg_set *as, char **error)
95{
96 if (!as->argc) {
97 *error = "Insufficient args";
98 return false;
99 }
100
101 return true;
102}
103
104static int parse_cache_dev(struct dm_pcache *pcache, struct dm_arg_set *as,
105 char **error)
106{
107 int ret;
108
109 if (!at_least_one_arg(as, error))
110 return -EINVAL;
111 ret = dm_get_device(pcache->ti, dm_shift_arg(as),
112 BLK_OPEN_READ | BLK_OPEN_WRITE,
113 &pcache->cache_dev.dm_dev);
114 if (ret) {
115 *error = "Error opening cache device";
116 return ret;
117 }
118
119 return 0;
120}
121
122static int parse_backing_dev(struct dm_pcache *pcache, struct dm_arg_set *as,
123 char **error)
124{
125 int ret;
126
127 if (!at_least_one_arg(as, error))
128 return -EINVAL;
129
130 ret = dm_get_device(pcache->ti, dm_shift_arg(as),
131 BLK_OPEN_READ | BLK_OPEN_WRITE,
132 &pcache->backing_dev.dm_dev);
133 if (ret) {
134 *error = "Error opening backing device";
135 return ret;
136 }
137
138 return 0;
139}
140
141static void pcache_init_opts(struct pcache_cache_options *opts)
142{
143 opts->cache_mode = PCACHE_CACHE_MODE_WRITEBACK;
144 opts->data_crc = false;
145}
146
147static int parse_cache_opts(struct dm_pcache *pcache, struct dm_arg_set *as,
148 char **error)
149{
150 struct pcache_cache_options *opts = &pcache->opts;
151 static const struct dm_arg _args[] = {
152 {0, 4, "Invalid number of cache option arguments"},
153 };
154 unsigned int argc;
155 const char *arg;
156 int ret;
157
158 pcache_init_opts(opts);
159 if (!as->argc)
160 return 0;
161
162 ret = dm_read_arg_group(_args, as, &argc, error);
163 if (ret)
164 return -EINVAL;
165
166 while (argc) {
167 arg = dm_shift_arg(as);
168 argc--;
169
170 if (!strcmp(arg, "cache_mode")) {
171 if (!argc) {
172 *error = "Missing value for cache_mode";
173 return -EINVAL;
174 }
175 arg = dm_shift_arg(as);
176 if (!strcmp(arg, "writeback")) {
177 opts->cache_mode = PCACHE_CACHE_MODE_WRITEBACK;
178 } else {
179 *error = "Invalid cache mode parameter";
180 return -EINVAL;
181 }
182 argc--;
183 } else if (!strcmp(arg, "data_crc")) {
184 if (!argc) {
185 *error = "Missing value for data_crc";
186 return -EINVAL;
187 }
188 arg = dm_shift_arg(as);
189 if (!strcmp(arg, "true")) {
190 opts->data_crc = true;
191 } else if (!strcmp(arg, "false")) {
192 opts->data_crc = false;
193 } else {
194 *error = "Invalid data crc parameter";
195 return -EINVAL;
196 }
197 argc--;
198 } else {
199 *error = "Unrecognised cache option requested";
200 return -EINVAL;
201 }
202 }
203
204 return 0;
205}
206
207static int pcache_start(struct dm_pcache *pcache, char **error)
208{
209 int ret;
210
211 ret = cache_dev_start(pcache);
212 if (ret) {
213 *error = "Failed to start cache dev";
214 return ret;
215 }
216
217 ret = backing_dev_start(pcache);
218 if (ret) {
219 *error = "Failed to start backing dev";
220 goto stop_cache;
221 }
222
223 ret = pcache_cache_start(pcache);
224 if (ret) {
225 *error = "Failed to start pcache";
226 goto stop_backing;
227 }
228
229 return 0;
230stop_backing:
231 backing_dev_stop(pcache);
232stop_cache:
233 cache_dev_stop(pcache);
234
235 return ret;
236}
237
238static void pcache_destroy_args(struct dm_pcache *pcache)
239{
240 if (pcache->cache_dev.dm_dev)
241 dm_put_device(pcache->ti, pcache->cache_dev.dm_dev);
242 if (pcache->backing_dev.dm_dev)
243 dm_put_device(pcache->ti, pcache->backing_dev.dm_dev);
244}
245
246static int pcache_parse_args(struct dm_pcache *pcache, unsigned int argc, char **argv,
247 char **error)
248{
249 struct dm_arg_set as;
250 int ret;
251
252 as.argc = argc;
253 as.argv = argv;
254
255 /*
256 * Parse cache device
257 */
258 ret = parse_cache_dev(pcache, &as, error);
259 if (ret)
260 return ret;
261 /*
262 * Parse backing device
263 */
264 ret = parse_backing_dev(pcache, &as, error);
265 if (ret)
266 goto out;
267 /*
268 * Parse optional arguments
269 */
270 ret = parse_cache_opts(pcache, &as, error);
271 if (ret)
272 goto out;
273
274 return 0;
275out:
276 pcache_destroy_args(pcache);
277 return ret;
278}
279
280static int dm_pcache_ctr(struct dm_target *ti, unsigned int argc, char **argv)
281{
282 struct mapped_device *md = ti->table->md;
283 struct dm_pcache *pcache;
284 int ret;
285
286 if (md->map) {
287 ti->error = "Don't support table loading for live md";
288 return -EOPNOTSUPP;
289 }
290
291 /* Allocate memory for the cache structure */
292 pcache = kzalloc_obj(struct dm_pcache);
293 if (!pcache)
294 return -ENOMEM;
295
296 pcache->task_wq = alloc_workqueue("pcache-%s-wq", WQ_UNBOUND | WQ_MEM_RECLAIM,
297 0, md->name);
298 if (!pcache->task_wq) {
299 ret = -ENOMEM;
300 goto free_pcache;
301 }
302
303 spin_lock_init(&pcache->defered_req_list_lock);
304 INIT_LIST_HEAD(&pcache->defered_req_list);
305 INIT_WORK(&pcache->defered_req_work, defered_req_fn);
306 pcache->ti = ti;
307
308 ret = pcache_parse_args(pcache, argc, argv, &ti->error);
309 if (ret)
310 goto destroy_wq;
311
312 ret = pcache_start(pcache, &ti->error);
313 if (ret)
314 goto destroy_args;
315
316 ti->num_flush_bios = 1;
317 ti->flush_supported = true;
318 ti->per_io_data_size = sizeof(struct pcache_request);
319 ti->private = pcache;
320 atomic_set(&pcache->inflight_reqs, 0);
321 atomic_set(&pcache->state, PCACHE_STATE_RUNNING);
322 init_waitqueue_head(&pcache->inflight_wq);
323
324 return 0;
325destroy_args:
326 pcache_destroy_args(pcache);
327destroy_wq:
328 destroy_workqueue(pcache->task_wq);
329free_pcache:
330 kfree(pcache);
331
332 return ret;
333}
334
335static void defer_req_stop(struct dm_pcache *pcache)
336{
337 struct pcache_request *pcache_req;
338 LIST_HEAD(tmp_list);
339
340 flush_work(&pcache->defered_req_work);
341
342 spin_lock(&pcache->defered_req_list_lock);
343 list_splice_init(&pcache->defered_req_list, &tmp_list);
344 spin_unlock(&pcache->defered_req_list_lock);
345
346 while (!list_empty(&tmp_list)) {
347 pcache_req = list_first_entry(&tmp_list,
348 struct pcache_request, list_node);
349 list_del_init(&pcache_req->list_node);
350 pcache_req_put(pcache_req, -EIO);
351 }
352}
353
354static void dm_pcache_dtr(struct dm_target *ti)
355{
356 struct dm_pcache *pcache;
357
358 pcache = ti->private;
359 atomic_set(&pcache->state, PCACHE_STATE_STOPPING);
360 defer_req_stop(pcache);
361
362 wait_event(pcache->inflight_wq,
363 atomic_read(&pcache->inflight_reqs) == 0);
364
365 pcache_cache_stop(pcache);
366 backing_dev_stop(pcache);
367 cache_dev_stop(pcache);
368
369 pcache_destroy_args(pcache);
370 drain_workqueue(pcache->task_wq);
371 destroy_workqueue(pcache->task_wq);
372
373 kfree(pcache);
374}
375
376static int dm_pcache_map_bio(struct dm_target *ti, struct bio *bio)
377{
378 struct pcache_request *pcache_req = dm_per_bio_data(bio, sizeof(struct pcache_request));
379 struct dm_pcache *pcache = ti->private;
380 int ret;
381
382 pcache_req->pcache = pcache;
383 kref_init(&pcache_req->ref);
384 pcache_req->ret = 0;
385 pcache_req->bio = bio;
386 pcache_req->off = (u64)bio->bi_iter.bi_sector << SECTOR_SHIFT;
387 pcache_req->data_len = bio->bi_iter.bi_size;
388 INIT_LIST_HEAD(&pcache_req->list_node);
389 atomic_inc(&pcache->inflight_reqs);
390
391 ret = pcache_cache_handle_req(&pcache->cache, pcache_req);
392 if (ret == -EBUSY)
393 defer_req(pcache_req);
394 else
395 pcache_req_put(pcache_req, ret);
396
397 return DM_MAPIO_SUBMITTED;
398}
399
400static void dm_pcache_status(struct dm_target *ti, status_type_t type,
401 unsigned int status_flags, char *result,
402 unsigned int maxlen)
403{
404 struct dm_pcache *pcache = ti->private;
405 struct pcache_cache_dev *cache_dev = &pcache->cache_dev;
406 struct pcache_backing_dev *backing_dev = &pcache->backing_dev;
407 struct pcache_cache *cache = &pcache->cache;
408 unsigned int sz = 0;
409
410 switch (type) {
411 case STATUSTYPE_INFO:
412 DMEMIT("%x %u %u %u %u %x %u:%u %u:%u %u:%u",
413 cache_dev->sb_flags,
414 cache_dev->seg_num,
415 cache->n_segs,
416 bitmap_weight(cache->seg_map, cache->n_segs),
417 pcache_cache_get_gc_percent(cache),
418 cache->cache_info.flags,
419 cache->key_head.cache_seg->cache_seg_id,
420 cache->key_head.seg_off,
421 cache->dirty_tail.cache_seg->cache_seg_id,
422 cache->dirty_tail.seg_off,
423 cache->key_tail.cache_seg->cache_seg_id,
424 cache->key_tail.seg_off);
425 break;
426 case STATUSTYPE_TABLE:
427 DMEMIT("%s %s 4 cache_mode writeback crc %s",
428 cache_dev->dm_dev->name,
429 backing_dev->dm_dev->name,
430 cache_data_crc_on(cache) ? "true" : "false");
431 break;
432 case STATUSTYPE_IMA:
433 *result = '\0';
434 break;
435 }
436}
437
438static int dm_pcache_message(struct dm_target *ti, unsigned int argc,
439 char **argv, char *result, unsigned int maxlen)
440{
441 struct dm_pcache *pcache = ti->private;
442 u8 val;
443
444 if (argc != 2)
445 goto err;
446
447 if (!strcasecmp(argv[0], "gc_percent")) {
448 if (kstrtou8(argv[1], 10, &val))
449 goto err;
450
451 return pcache_cache_set_gc_percent(&pcache->cache, val);
452 }
453err:
454 return -EINVAL;
455}
456
457static struct target_type dm_pcache_target = {
458 .name = "pcache",
459 .version = {0, 1, 0},
460 .module = THIS_MODULE,
461 .features = DM_TARGET_SINGLETON,
462 .ctr = dm_pcache_ctr,
463 .dtr = dm_pcache_dtr,
464 .map = dm_pcache_map_bio,
465 .status = dm_pcache_status,
466 .message = dm_pcache_message,
467};
468
469static int __init dm_pcache_init(void)
470{
471 int ret;
472
473 ret = pcache_backing_init();
474 if (ret)
475 goto err;
476
477 ret = pcache_cache_init();
478 if (ret)
479 goto backing_exit;
480
481 ret = dm_register_target(&dm_pcache_target);
482 if (ret)
483 goto cache_exit;
484 return 0;
485
486cache_exit:
487 pcache_cache_exit();
488backing_exit:
489 pcache_backing_exit();
490err:
491 return ret;
492}
493module_init(dm_pcache_init);
494
495static void __exit dm_pcache_exit(void)
496{
497 dm_unregister_target(&dm_pcache_target);
498 pcache_cache_exit();
499 pcache_backing_exit();
500}
501module_exit(dm_pcache_exit);
502
503MODULE_DESCRIPTION("dm-pcache Persistent Cache for block device");
504MODULE_AUTHOR("Dongsheng Yang <[email protected]>");
505MODULE_LICENSE("GPL");