projects/linux·blob master · refs

3615 lines · 86.4 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2012 Red Hat. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm.h"
9#include "dm-bio-prison-v2.h"
10#include "dm-bio-record.h"
11#include "dm-cache-metadata.h"
12#include "dm-io-tracker.h"
13#include "dm-cache-background-tracker.h"
14
15#include <linux/dm-io.h>
16#include <linux/dm-kcopyd.h>
17#include <linux/jiffies.h>
18#include <linux/init.h>
19#include <linux/kstrtox.h>
20#include <linux/mempool.h>
21#include <linux/module.h>
22#include <linux/rwsem.h>
23#include <linux/slab.h>
24#include <linux/vmalloc.h>
25
26#define DM_MSG_PREFIX "cache"
27
28DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(cache_copy_throttle,
29 "A percentage of time allocated for copying to and/or from cache");
30
31/*----------------------------------------------------------------*/
32
33/*
34 * Glossary:
35 *
36 * oblock: index of an origin block
37 * cblock: index of a cache block
38 * promotion: movement of a block from origin to cache
39 * demotion: movement of a block from cache to origin
40 * migration: movement of a block between the origin and cache device,
41 * either direction
42 */
43
44/*----------------------------------------------------------------*/
45
46/*
47 * Represents a chunk of future work. 'input' allows continuations to pass
48 * values between themselves, typically error values.
49 */
50struct continuation {
51 struct work_struct ws;
52 blk_status_t input;
53};
54
55static inline void init_continuation(struct continuation *k,
56 void (*fn)(struct work_struct *))
57{
58 INIT_WORK(&k->ws, fn);
59 k->input = 0;
60}
61
62static inline void queue_continuation(struct workqueue_struct *wq,
63 struct continuation *k)
64{
65 queue_work(wq, &k->ws);
66}
67
68/*----------------------------------------------------------------*/
69
70/*
71 * The batcher collects together pieces of work that need a particular
72 * operation to occur before they can proceed (typically a commit).
73 */
74struct batcher {
75 /*
76 * The operation that everyone is waiting for.
77 */
78 blk_status_t (*commit_op)(void *context);
79 void *commit_context;
80
81 /*
82 * This is how bios should be issued once the commit op is complete
83 * (accounted_request).
84 */
85 void (*issue_op)(struct bio *bio, void *context);
86 void *issue_context;
87
88 /*
89 * Queued work gets put on here after commit.
90 */
91 struct workqueue_struct *wq;
92
93 spinlock_t lock;
94 struct list_head work_items;
95 struct bio_list bios;
96 struct work_struct commit_work;
97
98 bool commit_scheduled;
99};
100
101static void __commit(struct work_struct *_ws)
102{
103 struct batcher *b = container_of(_ws, struct batcher, commit_work);
104 blk_status_t r;
105 struct list_head work_items;
106 struct work_struct *ws, *tmp;
107 struct continuation *k;
108 struct bio *bio;
109 struct bio_list bios;
110
111 INIT_LIST_HEAD(&work_items);
112 bio_list_init(&bios);
113
114 /*
115 * We have to grab these before the commit_op to avoid a race
116 * condition.
117 */
118 spin_lock_irq(&b->lock);
119 list_splice_init(&b->work_items, &work_items);
120 bio_list_merge_init(&bios, &b->bios);
121 b->commit_scheduled = false;
122 spin_unlock_irq(&b->lock);
123
124 r = b->commit_op(b->commit_context);
125
126 list_for_each_entry_safe(ws, tmp, &work_items, entry) {
127 k = container_of(ws, struct continuation, ws);
128 k->input = r;
129 INIT_LIST_HEAD(&ws->entry); /* to avoid a WARN_ON */
130 queue_work(b->wq, ws);
131 }
132
133 while ((bio = bio_list_pop(&bios))) {
134 if (r) {
135 bio->bi_status = r;
136 bio_endio(bio);
137 } else
138 b->issue_op(bio, b->issue_context);
139 }
140}
141
142static void batcher_init(struct batcher *b,
143 blk_status_t (*commit_op)(void *),
144 void *commit_context,
145 void (*issue_op)(struct bio *bio, void *),
146 void *issue_context,
147 struct workqueue_struct *wq)
148{
149 b->commit_op = commit_op;
150 b->commit_context = commit_context;
151 b->issue_op = issue_op;
152 b->issue_context = issue_context;
153 b->wq = wq;
154
155 spin_lock_init(&b->lock);
156 INIT_LIST_HEAD(&b->work_items);
157 bio_list_init(&b->bios);
158 INIT_WORK(&b->commit_work, __commit);
159 b->commit_scheduled = false;
160}
161
162static void async_commit(struct batcher *b)
163{
164 queue_work(b->wq, &b->commit_work);
165}
166
167static void continue_after_commit(struct batcher *b, struct continuation *k)
168{
169 bool commit_scheduled;
170
171 spin_lock_irq(&b->lock);
172 commit_scheduled = b->commit_scheduled;
173 list_add_tail(&k->ws.entry, &b->work_items);
174 spin_unlock_irq(&b->lock);
175
176 if (commit_scheduled)
177 async_commit(b);
178}
179
180/*
181 * Bios are errored if commit failed.
182 */
183static void issue_after_commit(struct batcher *b, struct bio *bio)
184{
185 bool commit_scheduled;
186
187 spin_lock_irq(&b->lock);
188 commit_scheduled = b->commit_scheduled;
189 bio_list_add(&b->bios, bio);
190 spin_unlock_irq(&b->lock);
191
192 if (commit_scheduled)
193 async_commit(b);
194}
195
196/*
197 * Call this if some urgent work is waiting for the commit to complete.
198 */
199static void schedule_commit(struct batcher *b)
200{
201 bool immediate;
202
203 spin_lock_irq(&b->lock);
204 immediate = !list_empty(&b->work_items) || !bio_list_empty(&b->bios);
205 b->commit_scheduled = true;
206 spin_unlock_irq(&b->lock);
207
208 if (immediate)
209 async_commit(b);
210}
211
212/*
213 * There are a couple of places where we let a bio run, but want to do some
214 * work before calling its endio function. We do this by temporarily
215 * changing the endio fn.
216 */
217struct dm_hook_info {
218 bio_end_io_t *bi_end_io;
219};
220
221static void dm_hook_bio(struct dm_hook_info *h, struct bio *bio,
222 bio_end_io_t *bi_end_io, void *bi_private)
223{
224 h->bi_end_io = bio->bi_end_io;
225
226 bio->bi_end_io = bi_end_io;
227 bio->bi_private = bi_private;
228}
229
230static void dm_unhook_bio(struct dm_hook_info *h, struct bio *bio)
231{
232 bio->bi_end_io = h->bi_end_io;
233}
234
235/*----------------------------------------------------------------*/
236
237#define MIGRATION_POOL_SIZE 128
238#define COMMIT_PERIOD HZ
239#define MIGRATION_COUNT_WINDOW 10
240
241/*
242 * The block size of the device holding cache data must be
243 * between 32KB and 1GB.
244 */
245#define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (32 * 1024 >> SECTOR_SHIFT)
246#define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
247
248enum cache_metadata_mode {
249 CM_WRITE, /* metadata may be changed */
250 CM_READ_ONLY, /* metadata may not be changed */
251 CM_FAIL
252};
253
254enum cache_io_mode {
255 /*
256 * Data is written to cached blocks only. These blocks are marked
257 * dirty. If you lose the cache device you will lose data.
258 * Potential performance increase for both reads and writes.
259 */
260 CM_IO_WRITEBACK,
261
262 /*
263 * Data is written to both cache and origin. Blocks are never
264 * dirty. Potential performance benfit for reads only.
265 */
266 CM_IO_WRITETHROUGH,
267
268 /*
269 * A degraded mode useful for various cache coherency situations
270 * (eg, rolling back snapshots). Reads and writes always go to the
271 * origin. If a write goes to a cached oblock, then the cache
272 * block is invalidated.
273 */
274 CM_IO_PASSTHROUGH
275};
276
277struct cache_features {
278 enum cache_metadata_mode mode;
279 enum cache_io_mode io_mode;
280 unsigned int metadata_version;
281 bool discard_passdown:1;
282};
283
284struct cache_stats {
285 atomic_t read_hit;
286 atomic_t read_miss;
287 atomic_t write_hit;
288 atomic_t write_miss;
289 atomic_t demotion;
290 atomic_t promotion;
291 atomic_t writeback;
292 atomic_t copies_avoided;
293 atomic_t cache_cell_clash;
294 atomic_t commit_count;
295 atomic_t discard_count;
296};
297
298struct cache {
299 struct dm_target *ti;
300 spinlock_t lock;
301
302 /*
303 * Fields for converting from sectors to blocks.
304 */
305 int sectors_per_block_shift;
306 sector_t sectors_per_block;
307
308 struct dm_cache_metadata *cmd;
309
310 /*
311 * Metadata is written to this device.
312 */
313 struct dm_dev *metadata_dev;
314
315 /*
316 * The slower of the two data devices. Typically a spindle.
317 */
318 struct dm_dev *origin_dev;
319
320 /*
321 * The faster of the two data devices. Typically an SSD.
322 */
323 struct dm_dev *cache_dev;
324
325 /*
326 * Size of the origin device in _complete_ blocks and native sectors.
327 */
328 dm_oblock_t origin_blocks;
329 sector_t origin_sectors;
330
331 /*
332 * Size of the cache device in blocks.
333 */
334 dm_cblock_t cache_size;
335
336 /*
337 * Invalidation fields.
338 */
339 spinlock_t invalidation_lock;
340 struct list_head invalidation_requests;
341
342 sector_t migration_threshold;
343
344 /*
345 * The number of in flight migrations that are performing
346 * background io. eg, promotion, writeback.
347 */
348 atomic_t nr_io_migrations;
349
350 struct bio_list deferred_bios;
351
352 struct rw_semaphore quiesce_lock;
353
354 /*
355 * origin_blocks entries, discarded if set.
356 */
357 dm_dblock_t discard_nr_blocks;
358 unsigned long *discard_bitset;
359 uint32_t discard_block_size; /* a power of 2 times sectors per block */
360
361 /*
362 * Rather than reconstructing the table line for the status we just
363 * save it and regurgitate.
364 */
365 unsigned int nr_ctr_args;
366 const char **ctr_args;
367
368 struct dm_kcopyd_client *copier;
369 struct work_struct deferred_bio_worker;
370 struct work_struct migration_worker;
371 struct workqueue_struct *wq;
372 struct delayed_work waker;
373 struct dm_bio_prison_v2 *prison;
374
375 /*
376 * cache_size entries, dirty if set
377 */
378 unsigned long *dirty_bitset;
379 atomic_t nr_dirty;
380
381 unsigned int policy_nr_args;
382 struct dm_cache_policy *policy;
383
384 /*
385 * Cache features such as write-through.
386 */
387 struct cache_features features;
388
389 struct cache_stats stats;
390
391 bool need_tick_bio:1;
392 bool sized:1;
393 bool invalidate:1;
394 bool commit_requested:1;
395 bool loaded_mappings:1;
396 bool loaded_discards:1;
397
398 /* background work management */
399 bool background_work_allowed;
400 unsigned background_work_nr;
401 spinlock_t background_work_lock;
402 wait_queue_head_t background_work_wait;
403
404 struct batcher committer;
405 struct work_struct commit_ws;
406
407 struct dm_io_tracker tracker;
408
409 mempool_t migration_pool;
410
411 struct bio_set bs;
412
413 /*
414 * Cache_size entries. Set bits indicate blocks mapped beyond the
415 * target length, which are marked for invalidation.
416 */
417 unsigned long *invalid_bitset;
418};
419
420struct per_bio_data {
421 bool tick:1;
422 unsigned int req_nr:2;
423 struct dm_bio_prison_cell_v2 *cell;
424 struct dm_hook_info hook_info;
425 sector_t len;
426};
427
428struct dm_cache_migration {
429 struct continuation k;
430 struct cache *cache;
431
432 struct policy_work *op;
433 struct bio *overwrite_bio;
434 struct dm_bio_prison_cell_v2 *cell;
435
436 dm_cblock_t invalidate_cblock;
437 dm_oblock_t invalidate_oblock;
438};
439
440/*----------------------------------------------------------------*/
441
442static bool writethrough_mode(struct cache *cache)
443{
444 return cache->features.io_mode == CM_IO_WRITETHROUGH;
445}
446
447static bool writeback_mode(struct cache *cache)
448{
449 return cache->features.io_mode == CM_IO_WRITEBACK;
450}
451
452static inline bool passthrough_mode(struct cache *cache)
453{
454 return unlikely(cache->features.io_mode == CM_IO_PASSTHROUGH);
455}
456
457/*----------------------------------------------------------------*/
458
459static void wake_deferred_bio_worker(struct cache *cache)
460{
461 queue_work(cache->wq, &cache->deferred_bio_worker);
462}
463
464static void wake_migration_worker(struct cache *cache)
465{
466 if (passthrough_mode(cache))
467 return;
468
469 queue_work(cache->wq, &cache->migration_worker);
470}
471
472/*----------------------------------------------------------------*/
473
474static struct dm_bio_prison_cell_v2 *alloc_prison_cell(struct cache *cache)
475{
476 return dm_bio_prison_alloc_cell_v2(cache->prison, GFP_NOIO);
477}
478
479static void free_prison_cell(struct cache *cache, struct dm_bio_prison_cell_v2 *cell)
480{
481 dm_bio_prison_free_cell_v2(cache->prison, cell);
482}
483
484static struct dm_cache_migration *alloc_migration(struct cache *cache)
485{
486 struct dm_cache_migration *mg;
487
488 mg = mempool_alloc(&cache->migration_pool, GFP_NOIO);
489
490 memset(mg, 0, sizeof(*mg));
491
492 mg->cache = cache;
493
494 return mg;
495}
496
497static void free_migration(struct dm_cache_migration *mg)
498{
499 mempool_free(mg, &mg->cache->migration_pool);
500}
501
502/*----------------------------------------------------------------*/
503
504static inline dm_oblock_t oblock_succ(dm_oblock_t b)
505{
506 return to_oblock(from_oblock(b) + 1ull);
507}
508
509static void build_key(dm_oblock_t begin, dm_oblock_t end, struct dm_cell_key_v2 *key)
510{
511 key->virtual = 0;
512 key->dev = 0;
513 key->block_begin = from_oblock(begin);
514 key->block_end = from_oblock(end);
515}
516
517/*
518 * We have two lock levels. Level 0, which is used to prevent WRITEs, and
519 * level 1 which prevents *both* READs and WRITEs.
520 */
521#define WRITE_LOCK_LEVEL 0
522#define READ_WRITE_LOCK_LEVEL 1
523
524static unsigned int lock_level(struct bio *bio)
525{
526 return bio_data_dir(bio) == WRITE ?
527 WRITE_LOCK_LEVEL :
528 READ_WRITE_LOCK_LEVEL;
529}
530
531/*
532 *--------------------------------------------------------------
533 * Per bio data
534 *--------------------------------------------------------------
535 */
536
537static struct per_bio_data *get_per_bio_data(struct bio *bio)
538{
539 struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data));
540
541 BUG_ON(!pb);
542 return pb;
543}
544
545static struct per_bio_data *init_per_bio_data(struct bio *bio)
546{
547 struct per_bio_data *pb = get_per_bio_data(bio);
548
549 pb->tick = false;
550 pb->req_nr = dm_bio_get_target_bio_nr(bio);
551 pb->cell = NULL;
552 pb->len = 0;
553
554 return pb;
555}
556
557/*----------------------------------------------------------------*/
558
559static void defer_bio(struct cache *cache, struct bio *bio)
560{
561 spin_lock_irq(&cache->lock);
562 bio_list_add(&cache->deferred_bios, bio);
563 spin_unlock_irq(&cache->lock);
564
565 wake_deferred_bio_worker(cache);
566}
567
568static void defer_bios(struct cache *cache, struct bio_list *bios)
569{
570 spin_lock_irq(&cache->lock);
571 bio_list_merge_init(&cache->deferred_bios, bios);
572 spin_unlock_irq(&cache->lock);
573
574 wake_deferred_bio_worker(cache);
575}
576
577/*----------------------------------------------------------------*/
578
579static bool bio_detain_shared(struct cache *cache, dm_oblock_t oblock, struct bio *bio)
580{
581 bool r;
582 struct per_bio_data *pb;
583 struct dm_cell_key_v2 key;
584 dm_oblock_t end = to_oblock(from_oblock(oblock) + 1ULL);
585 struct dm_bio_prison_cell_v2 *cell_prealloc, *cell;
586
587 cell_prealloc = alloc_prison_cell(cache); /* FIXME: allow wait if calling from worker */
588
589 build_key(oblock, end, &key);
590 r = dm_cell_get_v2(cache->prison, &key, lock_level(bio), bio, cell_prealloc, &cell);
591 if (!r) {
592 /*
593 * Failed to get the lock.
594 */
595 free_prison_cell(cache, cell_prealloc);
596 return r;
597 }
598
599 if (cell != cell_prealloc)
600 free_prison_cell(cache, cell_prealloc);
601
602 pb = get_per_bio_data(bio);
603 pb->cell = cell;
604
605 return r;
606}
607
608/*----------------------------------------------------------------*/
609
610static bool is_dirty(struct cache *cache, dm_cblock_t b)
611{
612 return test_bit(from_cblock(b), cache->dirty_bitset);
613}
614
615static void set_dirty(struct cache *cache, dm_cblock_t cblock)
616{
617 if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset)) {
618 atomic_inc(&cache->nr_dirty);
619 policy_set_dirty(cache->policy, cblock);
620 }
621}
622
623/*
624 * These two are called when setting after migrations to force the policy
625 * and dirty bitset to be in sync.
626 */
627static void force_set_dirty(struct cache *cache, dm_cblock_t cblock)
628{
629 if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset))
630 atomic_inc(&cache->nr_dirty);
631 policy_set_dirty(cache->policy, cblock);
632}
633
634static void force_clear_dirty(struct cache *cache, dm_cblock_t cblock)
635{
636 if (test_and_clear_bit(from_cblock(cblock), cache->dirty_bitset)) {
637 if (atomic_dec_return(&cache->nr_dirty) == 0)
638 dm_table_event(cache->ti->table);
639 }
640
641 policy_clear_dirty(cache->policy, cblock);
642}
643
644/*----------------------------------------------------------------*/
645
646static bool block_size_is_power_of_two(struct cache *cache)
647{
648 return cache->sectors_per_block_shift >= 0;
649}
650
651static dm_block_t block_div(dm_block_t b, uint32_t n)
652{
653 do_div(b, n);
654
655 return b;
656}
657
658static dm_block_t oblocks_per_dblock(struct cache *cache)
659{
660 dm_block_t oblocks = cache->discard_block_size;
661
662 if (block_size_is_power_of_two(cache))
663 oblocks >>= cache->sectors_per_block_shift;
664 else
665 oblocks = block_div(oblocks, cache->sectors_per_block);
666
667 return oblocks;
668}
669
670static dm_dblock_t oblock_to_dblock(struct cache *cache, dm_oblock_t oblock)
671{
672 return to_dblock(block_div(from_oblock(oblock),
673 oblocks_per_dblock(cache)));
674}
675
676static void set_discard(struct cache *cache, dm_dblock_t b)
677{
678 BUG_ON(from_dblock(b) >= from_dblock(cache->discard_nr_blocks));
679 atomic_inc(&cache->stats.discard_count);
680
681 spin_lock_irq(&cache->lock);
682 set_bit(from_dblock(b), cache->discard_bitset);
683 spin_unlock_irq(&cache->lock);
684}
685
686static void clear_discard(struct cache *cache, dm_dblock_t b)
687{
688 spin_lock_irq(&cache->lock);
689 clear_bit(from_dblock(b), cache->discard_bitset);
690 spin_unlock_irq(&cache->lock);
691}
692
693static bool is_discarded(struct cache *cache, dm_dblock_t b)
694{
695 int r;
696
697 spin_lock_irq(&cache->lock);
698 r = test_bit(from_dblock(b), cache->discard_bitset);
699 spin_unlock_irq(&cache->lock);
700
701 return r;
702}
703
704static bool is_discarded_oblock(struct cache *cache, dm_oblock_t b)
705{
706 int r;
707
708 spin_lock_irq(&cache->lock);
709 r = test_bit(from_dblock(oblock_to_dblock(cache, b)),
710 cache->discard_bitset);
711 spin_unlock_irq(&cache->lock);
712
713 return r;
714}
715
716/*
717 * -------------------------------------------------------------
718 * Remapping
719 *--------------------------------------------------------------
720 */
721static void remap_to_origin(struct cache *cache, struct bio *bio)
722{
723 bio_set_dev(bio, cache->origin_dev->bdev);
724}
725
726static void remap_to_cache(struct cache *cache, struct bio *bio,
727 dm_cblock_t cblock)
728{
729 sector_t bi_sector = bio->bi_iter.bi_sector;
730 sector_t block = from_cblock(cblock);
731
732 bio_set_dev(bio, cache->cache_dev->bdev);
733 if (!block_size_is_power_of_two(cache))
734 bio->bi_iter.bi_sector =
735 (block * cache->sectors_per_block) +
736 sector_div(bi_sector, cache->sectors_per_block);
737 else
738 bio->bi_iter.bi_sector =
739 (block << cache->sectors_per_block_shift) |
740 (bi_sector & (cache->sectors_per_block - 1));
741}
742
743static void check_if_tick_bio_needed(struct cache *cache, struct bio *bio)
744{
745 struct per_bio_data *pb;
746
747 spin_lock_irq(&cache->lock);
748 if (cache->need_tick_bio && !op_is_flush(bio->bi_opf) &&
749 bio_op(bio) != REQ_OP_DISCARD) {
750 pb = get_per_bio_data(bio);
751 pb->tick = true;
752 cache->need_tick_bio = false;
753 }
754 spin_unlock_irq(&cache->lock);
755}
756
757static void remap_to_origin_clear_discard(struct cache *cache, struct bio *bio,
758 dm_oblock_t oblock)
759{
760 // FIXME: check_if_tick_bio_needed() is called way too much through this interface
761 check_if_tick_bio_needed(cache, bio);
762 remap_to_origin(cache, bio);
763 if (bio_data_dir(bio) == WRITE)
764 clear_discard(cache, oblock_to_dblock(cache, oblock));
765}
766
767static void remap_to_cache_dirty(struct cache *cache, struct bio *bio,
768 dm_oblock_t oblock, dm_cblock_t cblock)
769{
770 check_if_tick_bio_needed(cache, bio);
771 remap_to_cache(cache, bio, cblock);
772 if (bio_data_dir(bio) == WRITE) {
773 set_dirty(cache, cblock);
774 clear_discard(cache, oblock_to_dblock(cache, oblock));
775 }
776}
777
778static dm_oblock_t get_bio_block(struct cache *cache, struct bio *bio)
779{
780 sector_t block_nr = bio->bi_iter.bi_sector;
781
782 if (!block_size_is_power_of_two(cache))
783 (void) sector_div(block_nr, cache->sectors_per_block);
784 else
785 block_nr >>= cache->sectors_per_block_shift;
786
787 return to_oblock(block_nr);
788}
789
790static bool accountable_bio(struct cache *cache, struct bio *bio)
791{
792 return bio_op(bio) != REQ_OP_DISCARD;
793}
794
795static void accounted_begin(struct cache *cache, struct bio *bio)
796{
797 struct per_bio_data *pb;
798
799 if (accountable_bio(cache, bio)) {
800 pb = get_per_bio_data(bio);
801 pb->len = bio_sectors(bio);
802 dm_iot_io_begin(&cache->tracker, pb->len);
803 }
804}
805
806static void accounted_complete(struct cache *cache, struct bio *bio)
807{
808 struct per_bio_data *pb = get_per_bio_data(bio);
809
810 dm_iot_io_end(&cache->tracker, pb->len);
811}
812
813static void accounted_request(struct cache *cache, struct bio *bio)
814{
815 accounted_begin(cache, bio);
816 dm_submit_bio_remap(bio, NULL);
817}
818
819static void issue_op(struct bio *bio, void *context)
820{
821 struct cache *cache = context;
822
823 accounted_request(cache, bio);
824}
825
826/*
827 * When running in writethrough mode we need to send writes to clean blocks
828 * to both the cache and origin devices. Clone the bio and send them in parallel.
829 */
830static void remap_to_origin_and_cache(struct cache *cache, struct bio *bio,
831 dm_oblock_t oblock, dm_cblock_t cblock)
832{
833 struct bio *origin_bio = bio_alloc_clone(cache->origin_dev->bdev, bio,
834 GFP_NOIO, &cache->bs);
835
836 BUG_ON(!origin_bio);
837
838 bio_chain(origin_bio, bio);
839
840 if (bio_data_dir(origin_bio) == WRITE)
841 clear_discard(cache, oblock_to_dblock(cache, oblock));
842 submit_bio(origin_bio);
843
844 remap_to_cache(cache, bio, cblock);
845}
846
847/*
848 *--------------------------------------------------------------
849 * Failure modes
850 *--------------------------------------------------------------
851 */
852static enum cache_metadata_mode get_cache_mode(struct cache *cache)
853{
854 return cache->features.mode;
855}
856
857static const char *cache_device_name(struct cache *cache)
858{
859 return dm_table_device_name(cache->ti->table);
860}
861
862static void notify_mode_switch(struct cache *cache, enum cache_metadata_mode mode)
863{
864 static const char *descs[] = {
865 "write",
866 "read-only",
867 "fail"
868 };
869
870 dm_table_event(cache->ti->table);
871 DMINFO("%s: switching cache to %s mode",
872 cache_device_name(cache), descs[(int)mode]);
873}
874
875static void set_cache_mode(struct cache *cache, enum cache_metadata_mode new_mode)
876{
877 bool needs_check;
878 enum cache_metadata_mode old_mode = get_cache_mode(cache);
879
880 if (dm_cache_metadata_needs_check(cache->cmd, &needs_check)) {
881 DMERR("%s: unable to read needs_check flag, setting failure mode.",
882 cache_device_name(cache));
883 new_mode = CM_FAIL;
884 }
885
886 if (new_mode == CM_WRITE && needs_check) {
887 DMERR("%s: unable to switch cache to write mode until repaired.",
888 cache_device_name(cache));
889 if (old_mode != new_mode)
890 new_mode = old_mode;
891 else
892 new_mode = CM_READ_ONLY;
893 }
894
895 /* Never move out of fail mode */
896 if (old_mode == CM_FAIL)
897 new_mode = CM_FAIL;
898
899 switch (new_mode) {
900 case CM_FAIL:
901 case CM_READ_ONLY:
902 dm_cache_metadata_set_read_only(cache->cmd);
903 break;
904
905 case CM_WRITE:
906 dm_cache_metadata_set_read_write(cache->cmd);
907 break;
908 }
909
910 cache->features.mode = new_mode;
911
912 if (new_mode != old_mode)
913 notify_mode_switch(cache, new_mode);
914}
915
916static void abort_transaction(struct cache *cache)
917{
918 const char *dev_name = cache_device_name(cache);
919
920 if (get_cache_mode(cache) >= CM_READ_ONLY)
921 return;
922
923 DMERR_LIMIT("%s: aborting current metadata transaction", dev_name);
924 if (dm_cache_metadata_abort(cache->cmd)) {
925 DMERR("%s: failed to abort metadata transaction", dev_name);
926 set_cache_mode(cache, CM_FAIL);
927 }
928
929 if (dm_cache_metadata_set_needs_check(cache->cmd)) {
930 DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name);
931 set_cache_mode(cache, CM_FAIL);
932 }
933}
934
935static void metadata_operation_failed(struct cache *cache, const char *op, int r)
936{
937 DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d",
938 cache_device_name(cache), op, r);
939 abort_transaction(cache);
940 set_cache_mode(cache, CM_READ_ONLY);
941}
942
943/*----------------------------------------------------------------*/
944
945static void load_stats(struct cache *cache)
946{
947 struct dm_cache_statistics stats;
948
949 dm_cache_metadata_get_stats(cache->cmd, &stats);
950 atomic_set(&cache->stats.read_hit, stats.read_hits);
951 atomic_set(&cache->stats.read_miss, stats.read_misses);
952 atomic_set(&cache->stats.write_hit, stats.write_hits);
953 atomic_set(&cache->stats.write_miss, stats.write_misses);
954}
955
956static void save_stats(struct cache *cache)
957{
958 struct dm_cache_statistics stats;
959
960 if (get_cache_mode(cache) >= CM_READ_ONLY)
961 return;
962
963 stats.read_hits = atomic_read(&cache->stats.read_hit);
964 stats.read_misses = atomic_read(&cache->stats.read_miss);
965 stats.write_hits = atomic_read(&cache->stats.write_hit);
966 stats.write_misses = atomic_read(&cache->stats.write_miss);
967
968 dm_cache_metadata_set_stats(cache->cmd, &stats);
969}
970
971static void update_stats(struct cache_stats *stats, enum policy_operation op)
972{
973 switch (op) {
974 case POLICY_PROMOTE:
975 atomic_inc(&stats->promotion);
976 break;
977
978 case POLICY_DEMOTE:
979 atomic_inc(&stats->demotion);
980 break;
981
982 case POLICY_WRITEBACK:
983 atomic_inc(&stats->writeback);
984 break;
985 }
986}
987
988/*
989 *---------------------------------------------------------------------
990 * Migration processing
991 *
992 * Migration covers moving data from the origin device to the cache, or
993 * vice versa.
994 *---------------------------------------------------------------------
995 */
996static void inc_io_migrations(struct cache *cache)
997{
998 atomic_inc(&cache->nr_io_migrations);
999}
1000
1001static void dec_io_migrations(struct cache *cache)
1002{
1003 atomic_dec(&cache->nr_io_migrations);
1004}
1005
1006static bool discard_or_flush(struct bio *bio)
1007{
1008 return bio_op(bio) == REQ_OP_DISCARD || op_is_flush(bio->bi_opf);
1009}
1010
1011static void calc_discard_block_range(struct cache *cache, struct bio *bio,
1012 dm_dblock_t *b, dm_dblock_t *e)
1013{
1014 sector_t sb = bio->bi_iter.bi_sector;
1015 sector_t se = bio_end_sector(bio);
1016
1017 *b = to_dblock(dm_sector_div_up(sb, cache->discard_block_size));
1018
1019 if (se - sb < cache->discard_block_size)
1020 *e = *b;
1021 else
1022 *e = to_dblock(block_div(se, cache->discard_block_size));
1023}
1024
1025/*----------------------------------------------------------------*/
1026
1027static void prevent_background_work(struct cache *cache)
1028{
1029 spin_lock_irq(&cache->background_work_lock);
1030 cache->background_work_allowed = false;
1031 wait_event_lock_irq(cache->background_work_wait,
1032 cache->background_work_nr == 0,
1033 cache->background_work_lock);
1034 spin_unlock_irq(&cache->background_work_lock);
1035}
1036
1037static void allow_background_work(struct cache *cache)
1038{
1039 spin_lock_irq(&cache->background_work_lock);
1040 cache->background_work_allowed = true;
1041 spin_unlock_irq(&cache->background_work_lock);
1042}
1043
1044static bool background_work_begin(struct cache *cache)
1045{
1046 bool r;
1047
1048 spin_lock_irq(&cache->background_work_lock);
1049 r = cache->background_work_allowed;
1050 if (r)
1051 cache->background_work_nr++;
1052 spin_unlock_irq(&cache->background_work_lock);
1053 return r;
1054}
1055
1056static void background_work_end(struct cache *cache)
1057{
1058 spin_lock_irq(&cache->background_work_lock);
1059 if (--cache->background_work_nr == 0)
1060 wake_up(&cache->background_work_wait);
1061 spin_unlock_irq(&cache->background_work_lock);
1062}
1063
1064/*----------------------------------------------------------------*/
1065
1066static bool bio_writes_complete_block(struct cache *cache, struct bio *bio)
1067{
1068 return (bio_data_dir(bio) == WRITE) &&
1069 (bio->bi_iter.bi_size == (cache->sectors_per_block << SECTOR_SHIFT));
1070}
1071
1072static bool optimisable_bio(struct cache *cache, struct bio *bio, dm_oblock_t block)
1073{
1074 return writeback_mode(cache) &&
1075 (is_discarded_oblock(cache, block) || bio_writes_complete_block(cache, bio));
1076}
1077
1078static void quiesce(struct dm_cache_migration *mg,
1079 void (*continuation)(struct work_struct *))
1080{
1081 init_continuation(&mg->k, continuation);
1082 dm_cell_quiesce_v2(mg->cache->prison, mg->cell, &mg->k.ws);
1083}
1084
1085static struct dm_cache_migration *ws_to_mg(struct work_struct *ws)
1086{
1087 struct continuation *k = container_of(ws, struct continuation, ws);
1088
1089 return container_of(k, struct dm_cache_migration, k);
1090}
1091
1092static void copy_complete(int read_err, unsigned long write_err, void *context)
1093{
1094 struct dm_cache_migration *mg = container_of(context, struct dm_cache_migration, k);
1095
1096 if (read_err || write_err)
1097 mg->k.input = BLK_STS_IOERR;
1098
1099 queue_continuation(mg->cache->wq, &mg->k);
1100}
1101
1102static void copy(struct dm_cache_migration *mg, bool promote)
1103{
1104 struct dm_io_region o_region, c_region;
1105 struct cache *cache = mg->cache;
1106
1107 o_region.bdev = cache->origin_dev->bdev;
1108 o_region.sector = from_oblock(mg->op->oblock) * cache->sectors_per_block;
1109 o_region.count = cache->sectors_per_block;
1110
1111 c_region.bdev = cache->cache_dev->bdev;
1112 c_region.sector = from_cblock(mg->op->cblock) * cache->sectors_per_block;
1113 c_region.count = cache->sectors_per_block;
1114
1115 if (promote)
1116 dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, &mg->k);
1117 else
1118 dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, &mg->k);
1119}
1120
1121static void bio_drop_shared_lock(struct cache *cache, struct bio *bio)
1122{
1123 struct per_bio_data *pb = get_per_bio_data(bio);
1124
1125 if (pb->cell && dm_cell_put_v2(cache->prison, pb->cell))
1126 free_prison_cell(cache, pb->cell);
1127 pb->cell = NULL;
1128}
1129
1130static void overwrite_endio(struct bio *bio)
1131{
1132 struct dm_cache_migration *mg = bio->bi_private;
1133 struct cache *cache = mg->cache;
1134 struct per_bio_data *pb = get_per_bio_data(bio);
1135
1136 dm_unhook_bio(&pb->hook_info, bio);
1137
1138 if (bio->bi_status)
1139 mg->k.input = bio->bi_status;
1140
1141 queue_continuation(cache->wq, &mg->k);
1142}
1143
1144static void overwrite(struct dm_cache_migration *mg,
1145 void (*continuation)(struct work_struct *))
1146{
1147 struct bio *bio = mg->overwrite_bio;
1148 struct per_bio_data *pb = get_per_bio_data(bio);
1149
1150 dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg);
1151
1152 /*
1153 * The overwrite bio is part of the copy operation, as such it does
1154 * not set/clear discard or dirty flags.
1155 */
1156 if (mg->op->op == POLICY_PROMOTE)
1157 remap_to_cache(mg->cache, bio, mg->op->cblock);
1158 else
1159 remap_to_origin(mg->cache, bio);
1160
1161 init_continuation(&mg->k, continuation);
1162 accounted_request(mg->cache, bio);
1163}
1164
1165/*
1166 * Migration steps:
1167 *
1168 * 1) exclusive lock preventing WRITEs
1169 * 2) quiesce
1170 * 3) copy or issue overwrite bio
1171 * 4) upgrade to exclusive lock preventing READs and WRITEs
1172 * 5) quiesce
1173 * 6) update metadata and commit
1174 * 7) unlock
1175 */
1176static void mg_complete(struct dm_cache_migration *mg, bool success)
1177{
1178 struct bio_list bios;
1179 struct cache *cache = mg->cache;
1180 struct policy_work *op = mg->op;
1181 dm_cblock_t cblock = op->cblock;
1182
1183 if (success)
1184 update_stats(&cache->stats, op->op);
1185
1186 switch (op->op) {
1187 case POLICY_PROMOTE:
1188 clear_discard(cache, oblock_to_dblock(cache, op->oblock));
1189 policy_complete_background_work(cache->policy, op, success);
1190
1191 if (mg->overwrite_bio) {
1192 if (success)
1193 force_set_dirty(cache, cblock);
1194 else if (mg->k.input)
1195 mg->overwrite_bio->bi_status = mg->k.input;
1196 else
1197 mg->overwrite_bio->bi_status = BLK_STS_IOERR;
1198 bio_endio(mg->overwrite_bio);
1199 } else {
1200 if (success)
1201 force_clear_dirty(cache, cblock);
1202 dec_io_migrations(cache);
1203 }
1204 break;
1205
1206 case POLICY_DEMOTE:
1207 /*
1208 * We clear dirty here to update the nr_dirty counter.
1209 */
1210 if (success)
1211 force_clear_dirty(cache, cblock);
1212 policy_complete_background_work(cache->policy, op, success);
1213 dec_io_migrations(cache);
1214 break;
1215
1216 case POLICY_WRITEBACK:
1217 if (success)
1218 force_clear_dirty(cache, cblock);
1219 policy_complete_background_work(cache->policy, op, success);
1220 dec_io_migrations(cache);
1221 break;
1222 }
1223
1224 bio_list_init(&bios);
1225 if (mg->cell) {
1226 if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios))
1227 free_prison_cell(cache, mg->cell);
1228 }
1229
1230 free_migration(mg);
1231 defer_bios(cache, &bios);
1232 wake_migration_worker(cache);
1233
1234 background_work_end(cache);
1235}
1236
1237static void mg_success(struct work_struct *ws)
1238{
1239 struct dm_cache_migration *mg = ws_to_mg(ws);
1240
1241 mg_complete(mg, mg->k.input == 0);
1242}
1243
1244static void mg_update_metadata(struct work_struct *ws)
1245{
1246 int r;
1247 struct dm_cache_migration *mg = ws_to_mg(ws);
1248 struct cache *cache = mg->cache;
1249 struct policy_work *op = mg->op;
1250
1251 switch (op->op) {
1252 case POLICY_PROMOTE:
1253 r = dm_cache_insert_mapping(cache->cmd, op->cblock, op->oblock);
1254 if (r) {
1255 DMERR_LIMIT("%s: migration failed; couldn't insert mapping",
1256 cache_device_name(cache));
1257 metadata_operation_failed(cache, "dm_cache_insert_mapping", r);
1258
1259 mg_complete(mg, false);
1260 return;
1261 }
1262 mg_complete(mg, true);
1263 break;
1264
1265 case POLICY_DEMOTE:
1266 r = dm_cache_remove_mapping(cache->cmd, op->cblock);
1267 if (r) {
1268 DMERR_LIMIT("%s: migration failed; couldn't update on disk metadata",
1269 cache_device_name(cache));
1270 metadata_operation_failed(cache, "dm_cache_remove_mapping", r);
1271
1272 mg_complete(mg, false);
1273 return;
1274 }
1275
1276 /*
1277 * It would be nice if we only had to commit when a REQ_FLUSH
1278 * comes through. But there's one scenario that we have to
1279 * look out for:
1280 *
1281 * - vblock x in a cache block
1282 * - domotion occurs
1283 * - cache block gets reallocated and over written
1284 * - crash
1285 *
1286 * When we recover, because there was no commit the cache will
1287 * rollback to having the data for vblock x in the cache block.
1288 * But the cache block has since been overwritten, so it'll end
1289 * up pointing to data that was never in 'x' during the history
1290 * of the device.
1291 *
1292 * To avoid this issue we require a commit as part of the
1293 * demotion operation.
1294 */
1295 init_continuation(&mg->k, mg_success);
1296 continue_after_commit(&cache->committer, &mg->k);
1297 schedule_commit(&cache->committer);
1298 break;
1299
1300 case POLICY_WRITEBACK:
1301 mg_complete(mg, true);
1302 break;
1303 }
1304}
1305
1306static void mg_update_metadata_after_copy(struct work_struct *ws)
1307{
1308 struct dm_cache_migration *mg = ws_to_mg(ws);
1309
1310 /*
1311 * Did the copy succeed?
1312 */
1313 if (mg->k.input)
1314 mg_complete(mg, false);
1315 else
1316 mg_update_metadata(ws);
1317}
1318
1319static void mg_upgrade_lock(struct work_struct *ws)
1320{
1321 int r;
1322 struct dm_cache_migration *mg = ws_to_mg(ws);
1323
1324 /*
1325 * Did the copy succeed?
1326 */
1327 if (mg->k.input)
1328 mg_complete(mg, false);
1329
1330 else {
1331 /*
1332 * Now we want the lock to prevent both reads and writes.
1333 */
1334 r = dm_cell_lock_promote_v2(mg->cache->prison, mg->cell,
1335 READ_WRITE_LOCK_LEVEL);
1336 if (r < 0)
1337 mg_complete(mg, false);
1338
1339 else if (r)
1340 quiesce(mg, mg_update_metadata);
1341
1342 else
1343 mg_update_metadata(ws);
1344 }
1345}
1346
1347static void mg_full_copy(struct work_struct *ws)
1348{
1349 struct dm_cache_migration *mg = ws_to_mg(ws);
1350 struct cache *cache = mg->cache;
1351 struct policy_work *op = mg->op;
1352 bool is_policy_promote = (op->op == POLICY_PROMOTE);
1353
1354 if ((!is_policy_promote && !is_dirty(cache, op->cblock)) ||
1355 is_discarded_oblock(cache, op->oblock)) {
1356 mg_upgrade_lock(ws);
1357 return;
1358 }
1359
1360 init_continuation(&mg->k, mg_upgrade_lock);
1361 copy(mg, is_policy_promote);
1362}
1363
1364static void mg_copy(struct work_struct *ws)
1365{
1366 struct dm_cache_migration *mg = ws_to_mg(ws);
1367
1368 if (mg->overwrite_bio) {
1369 /*
1370 * No exclusive lock was held when we last checked if the bio
1371 * was optimisable. So we have to check again in case things
1372 * have changed (eg, the block may no longer be discarded).
1373 */
1374 if (!optimisable_bio(mg->cache, mg->overwrite_bio, mg->op->oblock)) {
1375 /*
1376 * Fallback to a real full copy after doing some tidying up.
1377 */
1378 bool rb = bio_detain_shared(mg->cache, mg->op->oblock, mg->overwrite_bio);
1379
1380 BUG_ON(rb); /* An exclusive lock must _not_ be held for this block */
1381 mg->overwrite_bio = NULL;
1382 inc_io_migrations(mg->cache);
1383 mg_full_copy(ws);
1384 return;
1385 }
1386
1387 /*
1388 * It's safe to do this here, even though it's new data
1389 * because all IO has been locked out of the block.
1390 *
1391 * mg_lock_writes() already took READ_WRITE_LOCK_LEVEL
1392 * so _not_ using mg_upgrade_lock() as continutation.
1393 */
1394 overwrite(mg, mg_update_metadata_after_copy);
1395
1396 } else
1397 mg_full_copy(ws);
1398}
1399
1400static int mg_lock_writes(struct dm_cache_migration *mg)
1401{
1402 int r;
1403 struct dm_cell_key_v2 key;
1404 struct cache *cache = mg->cache;
1405 struct dm_bio_prison_cell_v2 *prealloc;
1406
1407 prealloc = alloc_prison_cell(cache);
1408
1409 /*
1410 * Prevent writes to the block, but allow reads to continue.
1411 * Unless we're using an overwrite bio, in which case we lock
1412 * everything.
1413 */
1414 build_key(mg->op->oblock, oblock_succ(mg->op->oblock), &key);
1415 r = dm_cell_lock_v2(cache->prison, &key,
1416 mg->overwrite_bio ? READ_WRITE_LOCK_LEVEL : WRITE_LOCK_LEVEL,
1417 prealloc, &mg->cell);
1418 if (r < 0) {
1419 free_prison_cell(cache, prealloc);
1420 mg_complete(mg, false);
1421 return r;
1422 }
1423
1424 if (mg->cell != prealloc)
1425 free_prison_cell(cache, prealloc);
1426
1427 if (r == 0)
1428 mg_copy(&mg->k.ws);
1429 else
1430 quiesce(mg, mg_copy);
1431
1432 return 0;
1433}
1434
1435static int mg_start(struct cache *cache, struct policy_work *op, struct bio *bio)
1436{
1437 struct dm_cache_migration *mg;
1438
1439 if (!background_work_begin(cache)) {
1440 policy_complete_background_work(cache->policy, op, false);
1441 return -EPERM;
1442 }
1443
1444 mg = alloc_migration(cache);
1445
1446 mg->op = op;
1447 mg->overwrite_bio = bio;
1448
1449 if (!bio)
1450 inc_io_migrations(cache);
1451
1452 return mg_lock_writes(mg);
1453}
1454
1455/*
1456 *--------------------------------------------------------------
1457 * invalidation processing
1458 *--------------------------------------------------------------
1459 */
1460
1461static void invalidate_complete(struct dm_cache_migration *mg, bool success)
1462{
1463 struct bio_list bios;
1464 struct cache *cache = mg->cache;
1465
1466 if (success)
1467 atomic_inc(&cache->stats.demotion);
1468
1469 bio_list_init(&bios);
1470 if (mg->cell) {
1471 if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios))
1472 free_prison_cell(cache, mg->cell);
1473 }
1474
1475 if (mg->overwrite_bio) {
1476 // Set generic error if the bio hasn't been issued yet,
1477 // e.g., invalidation or metadata commit failed before bio
1478 // submission. Otherwise preserve the bio's own error status.
1479 if (!success && !mg->overwrite_bio->bi_status)
1480 mg->overwrite_bio->bi_status = BLK_STS_IOERR;
1481 bio_endio(mg->overwrite_bio);
1482 }
1483
1484 free_migration(mg);
1485 defer_bios(cache, &bios);
1486
1487 background_work_end(cache);
1488}
1489
1490static void invalidate_completed(struct work_struct *ws)
1491{
1492 struct dm_cache_migration *mg = ws_to_mg(ws);
1493
1494 invalidate_complete(mg, !mg->k.input);
1495}
1496
1497static int invalidate_cblock(struct cache *cache, dm_cblock_t cblock)
1498{
1499 int r;
1500
1501 r = policy_invalidate_mapping(cache->policy, cblock);
1502 if (!r) {
1503 r = dm_cache_remove_mapping(cache->cmd, cblock);
1504 if (r) {
1505 DMERR_LIMIT("%s: invalidation failed; couldn't update on disk metadata",
1506 cache_device_name(cache));
1507 metadata_operation_failed(cache, "dm_cache_remove_mapping", r);
1508 }
1509
1510 } else if (r == -ENODATA) {
1511 /*
1512 * Harmless, already unmapped.
1513 */
1514 r = 0;
1515
1516 } else
1517 DMERR("%s: policy_invalidate_mapping failed", cache_device_name(cache));
1518
1519 return r;
1520}
1521
1522static void invalidate_committed(struct work_struct *ws)
1523{
1524 struct dm_cache_migration *mg = ws_to_mg(ws);
1525 struct cache *cache = mg->cache;
1526 struct bio *bio = mg->overwrite_bio;
1527 struct per_bio_data *pb = get_per_bio_data(bio);
1528
1529 if (mg->k.input) {
1530 invalidate_complete(mg, false);
1531 return;
1532 }
1533
1534 init_continuation(&mg->k, invalidate_completed);
1535 remap_to_origin_clear_discard(cache, bio, mg->invalidate_oblock);
1536 dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg);
1537 dm_submit_bio_remap(bio, NULL);
1538}
1539
1540static void invalidate_remove(struct work_struct *ws)
1541{
1542 int r;
1543 struct dm_cache_migration *mg = ws_to_mg(ws);
1544 struct cache *cache = mg->cache;
1545
1546 r = invalidate_cblock(cache, mg->invalidate_cblock);
1547 if (r) {
1548 invalidate_complete(mg, false);
1549 return;
1550 }
1551
1552 init_continuation(&mg->k, invalidate_committed);
1553 continue_after_commit(&cache->committer, &mg->k);
1554 schedule_commit(&cache->committer);
1555}
1556
1557static int invalidate_lock(struct dm_cache_migration *mg)
1558{
1559 int r;
1560 struct dm_cell_key_v2 key;
1561 struct cache *cache = mg->cache;
1562 struct dm_bio_prison_cell_v2 *prealloc;
1563
1564 prealloc = alloc_prison_cell(cache);
1565
1566 build_key(mg->invalidate_oblock, oblock_succ(mg->invalidate_oblock), &key);
1567 r = dm_cell_lock_v2(cache->prison, &key,
1568 READ_WRITE_LOCK_LEVEL, prealloc, &mg->cell);
1569 if (r < 0) {
1570 free_prison_cell(cache, prealloc);
1571
1572 /* Defer the bio for retrying the cell lock */
1573 if (mg->overwrite_bio) {
1574 struct bio *bio = mg->overwrite_bio;
1575
1576 mg->overwrite_bio = NULL;
1577 defer_bio(cache, bio);
1578 }
1579
1580 invalidate_complete(mg, false);
1581 return r;
1582 }
1583
1584 if (mg->cell != prealloc)
1585 free_prison_cell(cache, prealloc);
1586
1587 if (r)
1588 quiesce(mg, invalidate_remove);
1589
1590 else {
1591 /*
1592 * We can't call invalidate_remove() directly here because we
1593 * might still be in request context.
1594 */
1595 init_continuation(&mg->k, invalidate_remove);
1596 queue_work(cache->wq, &mg->k.ws);
1597 }
1598
1599 return 0;
1600}
1601
1602static int invalidate_start(struct cache *cache, dm_cblock_t cblock,
1603 dm_oblock_t oblock, struct bio *bio)
1604{
1605 struct dm_cache_migration *mg;
1606
1607 if (!background_work_begin(cache))
1608 return -EPERM;
1609
1610 mg = alloc_migration(cache);
1611
1612 mg->overwrite_bio = bio;
1613 mg->invalidate_cblock = cblock;
1614 mg->invalidate_oblock = oblock;
1615
1616 return invalidate_lock(mg);
1617}
1618
1619/*
1620 *--------------------------------------------------------------
1621 * bio processing
1622 *--------------------------------------------------------------
1623 */
1624
1625enum busy {
1626 IDLE,
1627 BUSY
1628};
1629
1630static enum busy spare_migration_bandwidth(struct cache *cache)
1631{
1632 bool idle = dm_iot_idle_for(&cache->tracker, HZ);
1633 sector_t current_volume = (atomic_read(&cache->nr_io_migrations) + 1) *
1634 cache->sectors_per_block;
1635
1636 if (idle && current_volume <= cache->migration_threshold)
1637 return IDLE;
1638 else
1639 return BUSY;
1640}
1641
1642static void inc_hit_counter(struct cache *cache, struct bio *bio)
1643{
1644 atomic_inc(bio_data_dir(bio) == READ ?
1645 &cache->stats.read_hit : &cache->stats.write_hit);
1646}
1647
1648static void inc_miss_counter(struct cache *cache, struct bio *bio)
1649{
1650 atomic_inc(bio_data_dir(bio) == READ ?
1651 &cache->stats.read_miss : &cache->stats.write_miss);
1652}
1653
1654/*----------------------------------------------------------------*/
1655
1656static int map_bio(struct cache *cache, struct bio *bio, dm_oblock_t block,
1657 bool *commit_needed)
1658{
1659 int r, data_dir;
1660 bool rb, background_queued;
1661 dm_cblock_t cblock;
1662
1663 *commit_needed = false;
1664
1665 rb = bio_detain_shared(cache, block, bio);
1666 if (!rb) {
1667 /*
1668 * An exclusive lock is held for this block, so we have to
1669 * wait. We set the commit_needed flag so the current
1670 * transaction will be committed asap, allowing this lock
1671 * to be dropped.
1672 */
1673 *commit_needed = true;
1674 return DM_MAPIO_SUBMITTED;
1675 }
1676
1677 data_dir = bio_data_dir(bio);
1678
1679 if (optimisable_bio(cache, bio, block)) {
1680 struct policy_work *op = NULL;
1681
1682 r = policy_lookup_with_work(cache->policy, block, &cblock, data_dir, true, &op);
1683 if (unlikely(r && r != -ENOENT)) {
1684 DMERR_LIMIT("%s: policy_lookup_with_work() failed with r = %d",
1685 cache_device_name(cache), r);
1686 bio_io_error(bio);
1687 return DM_MAPIO_SUBMITTED;
1688 }
1689
1690 if (r == -ENOENT && op) {
1691 bio_drop_shared_lock(cache, bio);
1692 BUG_ON(op->op != POLICY_PROMOTE);
1693 mg_start(cache, op, bio);
1694 return DM_MAPIO_SUBMITTED;
1695 }
1696 } else {
1697 r = policy_lookup(cache->policy, block, &cblock, data_dir, false, &background_queued);
1698 if (unlikely(r && r != -ENOENT)) {
1699 DMERR_LIMIT("%s: policy_lookup() failed with r = %d",
1700 cache_device_name(cache), r);
1701 bio_io_error(bio);
1702 return DM_MAPIO_SUBMITTED;
1703 }
1704
1705 if (background_queued)
1706 wake_migration_worker(cache);
1707 }
1708
1709 if (r == -ENOENT) {
1710 struct per_bio_data *pb = get_per_bio_data(bio);
1711
1712 /*
1713 * Miss.
1714 */
1715 inc_miss_counter(cache, bio);
1716 if (pb->req_nr == 0) {
1717 accounted_begin(cache, bio);
1718 remap_to_origin_clear_discard(cache, bio, block);
1719 } else {
1720 /*
1721 * This is a duplicate writethrough io that is no
1722 * longer needed because the block has been demoted.
1723 */
1724 bio_endio(bio);
1725 return DM_MAPIO_SUBMITTED;
1726 }
1727 } else {
1728 /*
1729 * Hit.
1730 */
1731 inc_hit_counter(cache, bio);
1732
1733 /*
1734 * Passthrough always maps to the origin, invalidating any
1735 * cache blocks that are written to.
1736 */
1737 if (passthrough_mode(cache)) {
1738 if (bio_data_dir(bio) == WRITE) {
1739 bio_drop_shared_lock(cache, bio);
1740 invalidate_start(cache, cblock, block, bio);
1741 return DM_MAPIO_SUBMITTED;
1742 } else
1743 remap_to_origin_clear_discard(cache, bio, block);
1744 } else {
1745 if (bio_data_dir(bio) == WRITE && writethrough_mode(cache) &&
1746 !is_dirty(cache, cblock)) {
1747 remap_to_origin_and_cache(cache, bio, block, cblock);
1748 accounted_begin(cache, bio);
1749 } else
1750 remap_to_cache_dirty(cache, bio, block, cblock);
1751 }
1752 }
1753
1754 /*
1755 * dm core turns FUA requests into a separate payload and FLUSH req.
1756 */
1757 if (bio->bi_opf & REQ_FUA) {
1758 /*
1759 * issue_after_commit will call accounted_begin a second time. So
1760 * we call accounted_complete() to avoid double accounting.
1761 */
1762 accounted_complete(cache, bio);
1763 issue_after_commit(&cache->committer, bio);
1764 *commit_needed = true;
1765 return DM_MAPIO_SUBMITTED;
1766 }
1767
1768 return DM_MAPIO_REMAPPED;
1769}
1770
1771static bool process_bio(struct cache *cache, struct bio *bio)
1772{
1773 bool commit_needed;
1774
1775 if (map_bio(cache, bio, get_bio_block(cache, bio), &commit_needed) == DM_MAPIO_REMAPPED)
1776 dm_submit_bio_remap(bio, NULL);
1777
1778 return commit_needed;
1779}
1780
1781/*
1782 * A non-zero return indicates read_only or fail_io mode.
1783 */
1784static int commit(struct cache *cache, bool clean_shutdown)
1785{
1786 int r;
1787
1788 if (get_cache_mode(cache) >= CM_READ_ONLY)
1789 return -EINVAL;
1790
1791 atomic_inc(&cache->stats.commit_count);
1792 r = dm_cache_commit(cache->cmd, clean_shutdown);
1793 if (r)
1794 metadata_operation_failed(cache, "dm_cache_commit", r);
1795
1796 return r;
1797}
1798
1799/*
1800 * Used by the batcher.
1801 */
1802static blk_status_t commit_op(void *context)
1803{
1804 struct cache *cache = context;
1805
1806 if (dm_cache_changed_this_transaction(cache->cmd))
1807 return errno_to_blk_status(commit(cache, false));
1808
1809 return 0;
1810}
1811
1812/*----------------------------------------------------------------*/
1813
1814static bool process_flush_bio(struct cache *cache, struct bio *bio)
1815{
1816 struct per_bio_data *pb = get_per_bio_data(bio);
1817
1818 if (!pb->req_nr)
1819 remap_to_origin(cache, bio);
1820 else
1821 remap_to_cache(cache, bio, 0);
1822
1823 issue_after_commit(&cache->committer, bio);
1824 return true;
1825}
1826
1827static bool process_discard_bio(struct cache *cache, struct bio *bio)
1828{
1829 dm_dblock_t b, e;
1830
1831 /*
1832 * FIXME: do we need to lock the region? Or can we just assume the
1833 * user wont be so foolish as to issue discard concurrently with
1834 * other IO?
1835 */
1836 calc_discard_block_range(cache, bio, &b, &e);
1837 while (b != e) {
1838 set_discard(cache, b);
1839 b = to_dblock(from_dblock(b) + 1);
1840 }
1841
1842 if (cache->features.discard_passdown) {
1843 remap_to_origin(cache, bio);
1844 dm_submit_bio_remap(bio, NULL);
1845 } else
1846 bio_endio(bio);
1847
1848 return false;
1849}
1850
1851static void process_deferred_bios(struct work_struct *ws)
1852{
1853 struct cache *cache = container_of(ws, struct cache, deferred_bio_worker);
1854
1855 bool commit_needed = false;
1856 struct bio_list bios;
1857 struct bio *bio;
1858
1859 bio_list_init(&bios);
1860
1861 spin_lock_irq(&cache->lock);
1862 bio_list_merge_init(&bios, &cache->deferred_bios);
1863 spin_unlock_irq(&cache->lock);
1864
1865 while ((bio = bio_list_pop(&bios))) {
1866 if (bio->bi_opf & REQ_PREFLUSH)
1867 commit_needed = process_flush_bio(cache, bio) || commit_needed;
1868
1869 else if (bio_op(bio) == REQ_OP_DISCARD)
1870 commit_needed = process_discard_bio(cache, bio) || commit_needed;
1871
1872 else
1873 commit_needed = process_bio(cache, bio) || commit_needed;
1874 cond_resched();
1875 }
1876
1877 if (commit_needed)
1878 schedule_commit(&cache->committer);
1879}
1880
1881/*
1882 *--------------------------------------------------------------
1883 * Main worker loop
1884 *--------------------------------------------------------------
1885 */
1886static void requeue_deferred_bios(struct cache *cache)
1887{
1888 struct bio *bio;
1889 struct bio_list bios;
1890
1891 bio_list_init(&bios);
1892 bio_list_merge_init(&bios, &cache->deferred_bios);
1893
1894 while ((bio = bio_list_pop(&bios))) {
1895 bio->bi_status = BLK_STS_DM_REQUEUE;
1896 bio_endio(bio);
1897 cond_resched();
1898 }
1899}
1900
1901/*
1902 * We want to commit periodically so that not too much
1903 * unwritten metadata builds up.
1904 */
1905static void do_waker(struct work_struct *ws)
1906{
1907 struct cache *cache = container_of(to_delayed_work(ws), struct cache, waker);
1908
1909 policy_tick(cache->policy, true);
1910 wake_migration_worker(cache);
1911 schedule_commit(&cache->committer);
1912 queue_delayed_work(cache->wq, &cache->waker, COMMIT_PERIOD);
1913}
1914
1915static void check_migrations(struct work_struct *ws)
1916{
1917 int r;
1918 struct policy_work *op;
1919 struct cache *cache = container_of(ws, struct cache, migration_worker);
1920 enum busy b;
1921
1922 for (;;) {
1923 b = spare_migration_bandwidth(cache);
1924
1925 r = policy_get_background_work(cache->policy, b == IDLE, &op);
1926 if (r == -ENODATA)
1927 break;
1928
1929 if (r) {
1930 DMERR_LIMIT("%s: policy_background_work failed",
1931 cache_device_name(cache));
1932 break;
1933 }
1934
1935 r = mg_start(cache, op, NULL);
1936 if (r)
1937 break;
1938
1939 cond_resched();
1940 }
1941}
1942
1943/*
1944 *--------------------------------------------------------------
1945 * Target methods
1946 *--------------------------------------------------------------
1947 */
1948
1949/*
1950 * This function gets called on the error paths of the constructor, so we
1951 * have to cope with a partially initialised struct.
1952 */
1953static void __destroy(struct cache *cache)
1954{
1955 mempool_exit(&cache->migration_pool);
1956
1957 if (cache->prison)
1958 dm_bio_prison_destroy_v2(cache->prison);
1959
1960 if (cache->wq)
1961 destroy_workqueue(cache->wq);
1962
1963 if (cache->dirty_bitset)
1964 free_bitset(cache->dirty_bitset);
1965
1966 if (cache->discard_bitset)
1967 free_bitset(cache->discard_bitset);
1968
1969 if (cache->invalid_bitset)
1970 free_bitset(cache->invalid_bitset);
1971
1972 if (cache->copier)
1973 dm_kcopyd_client_destroy(cache->copier);
1974
1975 if (cache->cmd)
1976 dm_cache_metadata_close(cache->cmd);
1977
1978 if (cache->metadata_dev)
1979 dm_put_device(cache->ti, cache->metadata_dev);
1980
1981 if (cache->origin_dev)
1982 dm_put_device(cache->ti, cache->origin_dev);
1983
1984 if (cache->cache_dev)
1985 dm_put_device(cache->ti, cache->cache_dev);
1986
1987 if (cache->policy)
1988 dm_cache_policy_destroy(cache->policy);
1989
1990 bioset_exit(&cache->bs);
1991
1992 kfree(cache);
1993}
1994
1995static void destroy(struct cache *cache)
1996{
1997 unsigned int i;
1998
1999 cancel_delayed_work_sync(&cache->waker);
2000
2001 for (i = 0; i < cache->nr_ctr_args ; i++)
2002 kfree(cache->ctr_args[i]);
2003 kfree(cache->ctr_args);
2004
2005 __destroy(cache);
2006}
2007
2008static void cache_dtr(struct dm_target *ti)
2009{
2010 struct cache *cache = ti->private;
2011
2012 destroy(cache);
2013}
2014
2015static sector_t get_dev_size(struct dm_dev *dev)
2016{
2017 return bdev_nr_sectors(dev->bdev);
2018}
2019
2020/*----------------------------------------------------------------*/
2021
2022/*
2023 * Construct a cache device mapping.
2024 *
2025 * cache <metadata dev> <cache dev> <origin dev> <block size>
2026 * <#feature args> [<feature arg>]*
2027 * <policy> <#policy args> [<policy arg>]*
2028 *
2029 * metadata dev : fast device holding the persistent metadata
2030 * cache dev : fast device holding cached data blocks
2031 * origin dev : slow device holding original data blocks
2032 * block size : cache unit size in sectors
2033 *
2034 * #feature args : number of feature arguments passed
2035 * feature args : writethrough. (The default is writeback.)
2036 *
2037 * policy : the replacement policy to use
2038 * #policy args : an even number of policy arguments corresponding
2039 * to key/value pairs passed to the policy
2040 * policy args : key/value pairs passed to the policy
2041 * E.g. 'sequential_threshold 1024'
2042 * See cache-policies.txt for details.
2043 *
2044 * Optional feature arguments are:
2045 * writethrough : write through caching that prohibits cache block
2046 * content from being different from origin block content.
2047 * Without this argument, the default behaviour is to write
2048 * back cache block contents later for performance reasons,
2049 * so they may differ from the corresponding origin blocks.
2050 */
2051struct cache_args {
2052 struct dm_target *ti;
2053
2054 struct dm_dev *metadata_dev;
2055
2056 struct dm_dev *cache_dev;
2057 sector_t cache_sectors;
2058
2059 struct dm_dev *origin_dev;
2060
2061 uint32_t block_size;
2062
2063 const char *policy_name;
2064 int policy_argc;
2065 const char **policy_argv;
2066
2067 struct cache_features features;
2068};
2069
2070static void destroy_cache_args(struct cache_args *ca)
2071{
2072 if (ca->metadata_dev)
2073 dm_put_device(ca->ti, ca->metadata_dev);
2074
2075 if (ca->cache_dev)
2076 dm_put_device(ca->ti, ca->cache_dev);
2077
2078 if (ca->origin_dev)
2079 dm_put_device(ca->ti, ca->origin_dev);
2080
2081 kfree(ca);
2082}
2083
2084static bool at_least_one_arg(struct dm_arg_set *as, char **error)
2085{
2086 if (!as->argc) {
2087 *error = "Insufficient args";
2088 return false;
2089 }
2090
2091 return true;
2092}
2093
2094static int parse_metadata_dev(struct cache_args *ca, struct dm_arg_set *as,
2095 char **error)
2096{
2097 int r;
2098 sector_t metadata_dev_size;
2099
2100 if (!at_least_one_arg(as, error))
2101 return -EINVAL;
2102
2103 r = dm_get_device(ca->ti, dm_shift_arg(as),
2104 BLK_OPEN_READ | BLK_OPEN_WRITE, &ca->metadata_dev);
2105 if (r) {
2106 *error = "Error opening metadata device";
2107 return r;
2108 }
2109
2110 metadata_dev_size = get_dev_size(ca->metadata_dev);
2111 if (metadata_dev_size > DM_CACHE_METADATA_MAX_SECTORS_WARNING)
2112 DMWARN("Metadata device %pg is larger than %u sectors: excess space will not be used.",
2113 ca->metadata_dev->bdev, THIN_METADATA_MAX_SECTORS);
2114
2115 return 0;
2116}
2117
2118static int parse_cache_dev(struct cache_args *ca, struct dm_arg_set *as,
2119 char **error)
2120{
2121 int r;
2122
2123 if (!at_least_one_arg(as, error))
2124 return -EINVAL;
2125
2126 r = dm_get_device(ca->ti, dm_shift_arg(as),
2127 BLK_OPEN_READ | BLK_OPEN_WRITE, &ca->cache_dev);
2128 if (r) {
2129 *error = "Error opening cache device";
2130 return r;
2131 }
2132 ca->cache_sectors = get_dev_size(ca->cache_dev);
2133
2134 return 0;
2135}
2136
2137static int parse_origin_dev(struct cache_args *ca, struct dm_arg_set *as,
2138 char **error)
2139{
2140 int r;
2141
2142 if (!at_least_one_arg(as, error))
2143 return -EINVAL;
2144
2145 r = dm_get_device(ca->ti, dm_shift_arg(as),
2146 BLK_OPEN_READ | BLK_OPEN_WRITE, &ca->origin_dev);
2147 if (r) {
2148 *error = "Error opening origin device";
2149 return r;
2150 }
2151
2152 return 0;
2153}
2154
2155static int parse_block_size(struct cache_args *ca, struct dm_arg_set *as,
2156 char **error)
2157{
2158 unsigned long block_size;
2159
2160 if (!at_least_one_arg(as, error))
2161 return -EINVAL;
2162
2163 if (kstrtoul(dm_shift_arg(as), 10, &block_size) || !block_size ||
2164 block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
2165 block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||
2166 block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) {
2167 *error = "Invalid data block size";
2168 return -EINVAL;
2169 }
2170
2171 if (block_size > ca->cache_sectors) {
2172 *error = "Data block size is larger than the cache device";
2173 return -EINVAL;
2174 }
2175
2176 ca->block_size = block_size;
2177
2178 return 0;
2179}
2180
2181static void init_features(struct cache_features *cf)
2182{
2183 cf->mode = CM_WRITE;
2184 cf->io_mode = CM_IO_WRITEBACK;
2185 cf->metadata_version = 1;
2186 cf->discard_passdown = true;
2187}
2188
2189static int parse_features(struct cache_args *ca, struct dm_arg_set *as,
2190 char **error)
2191{
2192 static const struct dm_arg _args[] = {
2193 {0, 3, "Invalid number of cache feature arguments"},
2194 };
2195
2196 int r, mode_ctr = 0;
2197 unsigned int argc;
2198 const char *arg;
2199 struct cache_features *cf = &ca->features;
2200
2201 init_features(cf);
2202
2203 r = dm_read_arg_group(_args, as, &argc, error);
2204 if (r)
2205 return -EINVAL;
2206
2207 while (argc--) {
2208 arg = dm_shift_arg(as);
2209
2210 if (!strcasecmp(arg, "writeback")) {
2211 cf->io_mode = CM_IO_WRITEBACK;
2212 mode_ctr++;
2213 }
2214
2215 else if (!strcasecmp(arg, "writethrough")) {
2216 cf->io_mode = CM_IO_WRITETHROUGH;
2217 mode_ctr++;
2218 }
2219
2220 else if (!strcasecmp(arg, "passthrough")) {
2221 cf->io_mode = CM_IO_PASSTHROUGH;
2222 mode_ctr++;
2223 }
2224
2225 else if (!strcasecmp(arg, "metadata2"))
2226 cf->metadata_version = 2;
2227
2228 else if (!strcasecmp(arg, "no_discard_passdown"))
2229 cf->discard_passdown = false;
2230
2231 else {
2232 *error = "Unrecognised cache feature requested";
2233 return -EINVAL;
2234 }
2235 }
2236
2237 if (mode_ctr > 1) {
2238 *error = "Duplicate cache io_mode features requested";
2239 return -EINVAL;
2240 }
2241
2242 return 0;
2243}
2244
2245static int parse_policy(struct cache_args *ca, struct dm_arg_set *as,
2246 char **error)
2247{
2248 static const struct dm_arg _args[] = {
2249 {0, 1024, "Invalid number of policy arguments"},
2250 };
2251
2252 int r;
2253
2254 if (!at_least_one_arg(as, error))
2255 return -EINVAL;
2256
2257 ca->policy_name = dm_shift_arg(as);
2258
2259 r = dm_read_arg_group(_args, as, &ca->policy_argc, error);
2260 if (r)
2261 return -EINVAL;
2262
2263 ca->policy_argv = (const char **)as->argv;
2264 dm_consume_args(as, ca->policy_argc);
2265
2266 return 0;
2267}
2268
2269static int parse_cache_args(struct cache_args *ca, int argc, char **argv,
2270 char **error)
2271{
2272 int r;
2273 struct dm_arg_set as;
2274
2275 as.argc = argc;
2276 as.argv = argv;
2277
2278 r = parse_metadata_dev(ca, &as, error);
2279 if (r)
2280 return r;
2281
2282 r = parse_cache_dev(ca, &as, error);
2283 if (r)
2284 return r;
2285
2286 r = parse_origin_dev(ca, &as, error);
2287 if (r)
2288 return r;
2289
2290 r = parse_block_size(ca, &as, error);
2291 if (r)
2292 return r;
2293
2294 r = parse_features(ca, &as, error);
2295 if (r)
2296 return r;
2297
2298 r = parse_policy(ca, &as, error);
2299 if (r)
2300 return r;
2301
2302 return 0;
2303}
2304
2305/*----------------------------------------------------------------*/
2306
2307static struct kmem_cache *migration_cache = NULL;
2308
2309#define NOT_CORE_OPTION 1
2310
2311static int process_config_option(struct cache *cache, const char *key, const char *value)
2312{
2313 unsigned long tmp;
2314
2315 if (!strcasecmp(key, "migration_threshold")) {
2316 if (kstrtoul(value, 10, &tmp))
2317 return -EINVAL;
2318
2319 cache->migration_threshold = tmp;
2320 return 0;
2321 }
2322
2323 return NOT_CORE_OPTION;
2324}
2325
2326static int set_config_value(struct cache *cache, const char *key, const char *value)
2327{
2328 int r = process_config_option(cache, key, value);
2329
2330 if (r == NOT_CORE_OPTION)
2331 r = policy_set_config_value(cache->policy, key, value);
2332
2333 if (r)
2334 DMWARN("bad config value for %s: %s", key, value);
2335
2336 return r;
2337}
2338
2339static int set_config_values(struct cache *cache, int argc, const char **argv)
2340{
2341 int r = 0;
2342
2343 if (argc & 1) {
2344 DMWARN("Odd number of policy arguments given but they should be <key> <value> pairs.");
2345 return -EINVAL;
2346 }
2347
2348 while (argc) {
2349 r = set_config_value(cache, argv[0], argv[1]);
2350 if (r)
2351 break;
2352
2353 argc -= 2;
2354 argv += 2;
2355 }
2356
2357 return r;
2358}
2359
2360static int create_cache_policy(struct cache *cache, struct cache_args *ca,
2361 char **error)
2362{
2363 struct dm_cache_policy *p = dm_cache_policy_create(ca->policy_name,
2364 cache->cache_size,
2365 cache->origin_sectors,
2366 cache->sectors_per_block);
2367 if (IS_ERR(p)) {
2368 *error = "Error creating cache's policy";
2369 return PTR_ERR(p);
2370 }
2371 cache->policy = p;
2372 BUG_ON(!cache->policy);
2373
2374 return 0;
2375}
2376
2377/*
2378 * We want the discard block size to be at least the size of the cache
2379 * block size and have no more than 2^14 discard blocks across the origin.
2380 */
2381#define MAX_DISCARD_BLOCKS (1 << 14)
2382
2383static bool too_many_discard_blocks(sector_t discard_block_size,
2384 sector_t origin_size)
2385{
2386 (void) sector_div(origin_size, discard_block_size);
2387
2388 return origin_size > MAX_DISCARD_BLOCKS;
2389}
2390
2391static sector_t calculate_discard_block_size(sector_t cache_block_size,
2392 sector_t origin_size)
2393{
2394 sector_t discard_block_size = cache_block_size;
2395
2396 if (origin_size)
2397 while (too_many_discard_blocks(discard_block_size, origin_size))
2398 discard_block_size *= 2;
2399
2400 return discard_block_size;
2401}
2402
2403static void set_cache_size(struct cache *cache, dm_cblock_t size)
2404{
2405 dm_block_t nr_blocks = from_cblock(size);
2406
2407 if (nr_blocks > (1 << 20) && cache->cache_size != size)
2408 DMWARN_LIMIT("You have created a cache device with a lot of individual cache blocks (%llu)\n"
2409 "All these mappings can consume a lot of kernel memory, and take some time to read/write.\n"
2410 "Please consider increasing the cache block size to reduce the overall cache block count.",
2411 (unsigned long long) nr_blocks);
2412
2413 cache->cache_size = size;
2414}
2415
2416#define DEFAULT_MIGRATION_THRESHOLD 2048
2417
2418static int cache_create(struct cache_args *ca, struct cache **result)
2419{
2420 int r = 0;
2421 char **error = &ca->ti->error;
2422 struct cache *cache;
2423 struct dm_target *ti = ca->ti;
2424 dm_block_t origin_blocks;
2425 struct dm_cache_metadata *cmd;
2426 bool may_format = ca->features.mode == CM_WRITE;
2427
2428 cache = kzalloc_obj(*cache);
2429 if (!cache)
2430 return -ENOMEM;
2431
2432 cache->ti = ca->ti;
2433 ti->private = cache;
2434 ti->accounts_remapped_io = true;
2435 ti->num_flush_bios = 2;
2436 ti->flush_supported = true;
2437
2438 ti->num_discard_bios = 1;
2439 ti->discards_supported = true;
2440
2441 ti->per_io_data_size = sizeof(struct per_bio_data);
2442
2443 cache->features = ca->features;
2444 if (writethrough_mode(cache)) {
2445 /* Create bioset for writethrough bios issued to origin */
2446 r = bioset_init(&cache->bs, BIO_POOL_SIZE, 0, 0);
2447 if (r)
2448 goto bad;
2449 }
2450
2451 cache->metadata_dev = ca->metadata_dev;
2452 cache->origin_dev = ca->origin_dev;
2453 cache->cache_dev = ca->cache_dev;
2454
2455 ca->metadata_dev = ca->origin_dev = ca->cache_dev = NULL;
2456
2457 origin_blocks = cache->origin_sectors = ti->len;
2458 origin_blocks = block_div(origin_blocks, ca->block_size);
2459 cache->origin_blocks = to_oblock(origin_blocks);
2460
2461 cache->sectors_per_block = ca->block_size;
2462 if (dm_set_target_max_io_len(ti, cache->sectors_per_block)) {
2463 r = -EINVAL;
2464 goto bad;
2465 }
2466
2467 if (ca->block_size & (ca->block_size - 1)) {
2468 dm_block_t cache_size = ca->cache_sectors;
2469
2470 cache->sectors_per_block_shift = -1;
2471 cache_size = block_div(cache_size, ca->block_size);
2472 set_cache_size(cache, to_cblock(cache_size));
2473 } else {
2474 cache->sectors_per_block_shift = __ffs(ca->block_size);
2475 set_cache_size(cache, to_cblock(ca->cache_sectors >> cache->sectors_per_block_shift));
2476 }
2477
2478 r = create_cache_policy(cache, ca, error);
2479 if (r)
2480 goto bad;
2481
2482 cache->policy_nr_args = ca->policy_argc;
2483 cache->migration_threshold = DEFAULT_MIGRATION_THRESHOLD;
2484
2485 r = set_config_values(cache, ca->policy_argc, ca->policy_argv);
2486 if (r) {
2487 *error = "Error setting cache policy's config values";
2488 goto bad;
2489 }
2490
2491 cmd = dm_cache_metadata_open(cache->metadata_dev->bdev,
2492 ca->block_size, may_format,
2493 dm_cache_policy_get_hint_size(cache->policy),
2494 ca->features.metadata_version);
2495 if (IS_ERR(cmd)) {
2496 *error = "Error creating metadata object";
2497 r = PTR_ERR(cmd);
2498 goto bad;
2499 }
2500 cache->cmd = cmd;
2501 set_cache_mode(cache, CM_WRITE);
2502 if (get_cache_mode(cache) != CM_WRITE) {
2503 *error = "Unable to get write access to metadata, please check/repair metadata.";
2504 r = -EINVAL;
2505 goto bad;
2506 }
2507
2508 if (passthrough_mode(cache))
2509 policy_allow_migrations(cache->policy, false);
2510
2511 spin_lock_init(&cache->lock);
2512 bio_list_init(&cache->deferred_bios);
2513 atomic_set(&cache->nr_io_migrations, 0);
2514
2515 r = -ENOMEM;
2516 atomic_set(&cache->nr_dirty, 0);
2517 cache->dirty_bitset = alloc_bitset(from_cblock(cache->cache_size));
2518 if (!cache->dirty_bitset) {
2519 *error = "could not allocate dirty bitset";
2520 goto bad;
2521 }
2522 clear_bitset(cache->dirty_bitset, from_cblock(cache->cache_size));
2523
2524 cache->discard_block_size =
2525 calculate_discard_block_size(cache->sectors_per_block,
2526 cache->origin_sectors);
2527 cache->discard_nr_blocks = to_dblock(dm_sector_div_up(cache->origin_sectors,
2528 cache->discard_block_size));
2529 cache->discard_bitset = alloc_bitset(from_dblock(cache->discard_nr_blocks));
2530 if (!cache->discard_bitset) {
2531 *error = "could not allocate discard bitset";
2532 goto bad;
2533 }
2534 clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks));
2535
2536 cache->invalid_bitset = alloc_bitset(from_cblock(cache->cache_size));
2537 if (!cache->invalid_bitset) {
2538 *error = "could not allocate bitset for invalid blocks";
2539 goto bad;
2540 }
2541 clear_bitset(cache->invalid_bitset, from_cblock(cache->cache_size));
2542
2543 cache->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle);
2544 if (IS_ERR(cache->copier)) {
2545 *error = "could not create kcopyd client";
2546 r = PTR_ERR(cache->copier);
2547 goto bad;
2548 }
2549
2550 cache->wq = alloc_workqueue("dm-" DM_MSG_PREFIX,
2551 WQ_MEM_RECLAIM | WQ_PERCPU, 0);
2552 if (!cache->wq) {
2553 *error = "could not create workqueue for metadata object";
2554 goto bad;
2555 }
2556 INIT_WORK(&cache->deferred_bio_worker, process_deferred_bios);
2557 INIT_WORK(&cache->migration_worker, check_migrations);
2558 INIT_DELAYED_WORK(&cache->waker, do_waker);
2559
2560 cache->prison = dm_bio_prison_create_v2(cache->wq);
2561 if (!cache->prison) {
2562 *error = "could not create bio prison";
2563 goto bad;
2564 }
2565
2566 r = mempool_init_slab_pool(&cache->migration_pool, MIGRATION_POOL_SIZE,
2567 migration_cache);
2568 if (r) {
2569 *error = "Error creating cache's migration mempool";
2570 goto bad;
2571 }
2572
2573 cache->need_tick_bio = true;
2574 cache->sized = false;
2575 cache->invalidate = false;
2576 cache->commit_requested = false;
2577 cache->loaded_mappings = false;
2578 cache->loaded_discards = false;
2579
2580 load_stats(cache);
2581
2582 atomic_set(&cache->stats.demotion, 0);
2583 atomic_set(&cache->stats.promotion, 0);
2584 atomic_set(&cache->stats.copies_avoided, 0);
2585 atomic_set(&cache->stats.cache_cell_clash, 0);
2586 atomic_set(&cache->stats.commit_count, 0);
2587 atomic_set(&cache->stats.discard_count, 0);
2588
2589 spin_lock_init(&cache->invalidation_lock);
2590 INIT_LIST_HEAD(&cache->invalidation_requests);
2591
2592 batcher_init(&cache->committer, commit_op, cache,
2593 issue_op, cache, cache->wq);
2594 dm_iot_init(&cache->tracker);
2595
2596 init_waitqueue_head(&cache->background_work_wait);
2597 spin_lock_init(&cache->background_work_lock);
2598 cache->background_work_allowed = false;
2599 cache->background_work_nr = 0;
2600
2601 *result = cache;
2602 return 0;
2603bad:
2604 __destroy(cache);
2605 return r;
2606}
2607
2608static int copy_ctr_args(struct cache *cache, int argc, const char **argv)
2609{
2610 unsigned int i;
2611 const char **copy;
2612
2613 copy = kcalloc(argc, sizeof(*copy), GFP_KERNEL);
2614 if (!copy)
2615 return -ENOMEM;
2616 for (i = 0; i < argc; i++) {
2617 copy[i] = kstrdup(argv[i], GFP_KERNEL);
2618 if (!copy[i]) {
2619 while (i--)
2620 kfree(copy[i]);
2621 kfree(copy);
2622 return -ENOMEM;
2623 }
2624 }
2625
2626 cache->nr_ctr_args = argc;
2627 cache->ctr_args = copy;
2628
2629 return 0;
2630}
2631
2632static int cache_ctr(struct dm_target *ti, unsigned int argc, char **argv)
2633{
2634 int r = -EINVAL;
2635 struct cache_args *ca;
2636 struct cache *cache = NULL;
2637
2638 ca = kzalloc_obj(*ca);
2639 if (!ca) {
2640 ti->error = "Error allocating memory for cache";
2641 return -ENOMEM;
2642 }
2643 ca->ti = ti;
2644
2645 r = parse_cache_args(ca, argc, argv, &ti->error);
2646 if (r)
2647 goto out;
2648
2649 r = cache_create(ca, &cache);
2650 if (r)
2651 goto out;
2652
2653 r = copy_ctr_args(cache, argc - 3, (const char **)argv + 3);
2654 if (r) {
2655 __destroy(cache);
2656 goto out;
2657 }
2658
2659 ti->private = cache;
2660out:
2661 destroy_cache_args(ca);
2662 return r;
2663}
2664
2665/*----------------------------------------------------------------*/
2666
2667static int cache_map(struct dm_target *ti, struct bio *bio)
2668{
2669 struct cache *cache = ti->private;
2670
2671 int r;
2672 bool commit_needed;
2673 dm_oblock_t block = get_bio_block(cache, bio);
2674
2675 init_per_bio_data(bio);
2676 if (unlikely(from_oblock(block) >= from_oblock(cache->origin_blocks))) {
2677 /*
2678 * This can only occur if the io goes to a partial block at
2679 * the end of the origin device. We don't cache these.
2680 * Just remap to the origin and carry on.
2681 */
2682 remap_to_origin(cache, bio);
2683 accounted_begin(cache, bio);
2684 return DM_MAPIO_REMAPPED;
2685 }
2686
2687 if (discard_or_flush(bio)) {
2688 defer_bio(cache, bio);
2689 return DM_MAPIO_SUBMITTED;
2690 }
2691
2692 r = map_bio(cache, bio, block, &commit_needed);
2693 if (commit_needed)
2694 schedule_commit(&cache->committer);
2695
2696 return r;
2697}
2698
2699static int cache_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *error)
2700{
2701 struct cache *cache = ti->private;
2702 unsigned long flags;
2703 struct per_bio_data *pb = get_per_bio_data(bio);
2704
2705 if (pb->tick) {
2706 policy_tick(cache->policy, false);
2707
2708 spin_lock_irqsave(&cache->lock, flags);
2709 cache->need_tick_bio = true;
2710 spin_unlock_irqrestore(&cache->lock, flags);
2711 }
2712
2713 bio_drop_shared_lock(cache, bio);
2714 accounted_complete(cache, bio);
2715
2716 return DM_ENDIO_DONE;
2717}
2718
2719static int write_dirty_bitset(struct cache *cache)
2720{
2721 int r;
2722
2723 if (get_cache_mode(cache) >= CM_READ_ONLY)
2724 return -EINVAL;
2725
2726 r = dm_cache_set_dirty_bits(cache->cmd, from_cblock(cache->cache_size), cache->dirty_bitset);
2727 if (r)
2728 metadata_operation_failed(cache, "dm_cache_set_dirty_bits", r);
2729
2730 return r;
2731}
2732
2733static int write_discard_bitset(struct cache *cache)
2734{
2735 unsigned int i, r;
2736
2737 if (get_cache_mode(cache) >= CM_READ_ONLY)
2738 return -EINVAL;
2739
2740 r = dm_cache_discard_bitset_resize(cache->cmd, cache->discard_block_size,
2741 cache->discard_nr_blocks);
2742 if (r) {
2743 DMERR("%s: could not resize on-disk discard bitset", cache_device_name(cache));
2744 metadata_operation_failed(cache, "dm_cache_discard_bitset_resize", r);
2745 return r;
2746 }
2747
2748 for (i = 0; i < from_dblock(cache->discard_nr_blocks); i++) {
2749 r = dm_cache_set_discard(cache->cmd, to_dblock(i),
2750 is_discarded(cache, to_dblock(i)));
2751 if (r) {
2752 metadata_operation_failed(cache, "dm_cache_set_discard", r);
2753 return r;
2754 }
2755 }
2756
2757 return 0;
2758}
2759
2760static int write_hints(struct cache *cache)
2761{
2762 int r;
2763
2764 if (get_cache_mode(cache) >= CM_READ_ONLY)
2765 return -EINVAL;
2766
2767 r = dm_cache_write_hints(cache->cmd, cache->policy);
2768 if (r) {
2769 metadata_operation_failed(cache, "dm_cache_write_hints", r);
2770 return r;
2771 }
2772
2773 return 0;
2774}
2775
2776/*
2777 * returns true on success
2778 */
2779static bool sync_metadata(struct cache *cache)
2780{
2781 int r1, r2, r3, r4;
2782
2783 r1 = write_dirty_bitset(cache);
2784 if (r1)
2785 DMERR("%s: could not write dirty bitset", cache_device_name(cache));
2786
2787 r2 = write_discard_bitset(cache);
2788 if (r2)
2789 DMERR("%s: could not write discard bitset", cache_device_name(cache));
2790
2791 save_stats(cache);
2792
2793 r3 = write_hints(cache);
2794 if (r3)
2795 DMERR("%s: could not write hints", cache_device_name(cache));
2796
2797 /*
2798 * If writing the above metadata failed, we still commit, but don't
2799 * set the clean shutdown flag. This will effectively force every
2800 * dirty bit to be set on reload.
2801 */
2802 r4 = commit(cache, !r1 && !r2 && !r3);
2803 if (r4)
2804 DMERR("%s: could not write cache metadata", cache_device_name(cache));
2805
2806 return !r1 && !r2 && !r3 && !r4;
2807}
2808
2809static void cache_postsuspend(struct dm_target *ti)
2810{
2811 struct cache *cache = ti->private;
2812
2813 prevent_background_work(cache);
2814 BUG_ON(atomic_read(&cache->nr_io_migrations));
2815
2816 cancel_delayed_work_sync(&cache->waker);
2817 drain_workqueue(cache->wq);
2818 WARN_ON(cache->tracker.in_flight);
2819
2820 /*
2821 * If it's a flush suspend there won't be any deferred bios, so this
2822 * call is harmless.
2823 */
2824 requeue_deferred_bios(cache);
2825
2826 if (get_cache_mode(cache) == CM_WRITE)
2827 (void) sync_metadata(cache);
2828}
2829
2830static int load_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock,
2831 bool dirty, uint32_t hint, bool hint_valid)
2832{
2833 struct cache *cache = context;
2834
2835 if (dirty) {
2836 if (passthrough_mode(cache)) {
2837 DMERR("%s: cannot enter passthrough mode unless all blocks are clean",
2838 cache_device_name(cache));
2839 return -EBUSY;
2840 }
2841
2842 set_bit(from_cblock(cblock), cache->dirty_bitset);
2843 atomic_inc(&cache->nr_dirty);
2844 } else
2845 clear_bit(from_cblock(cblock), cache->dirty_bitset);
2846
2847 return policy_load_mapping(cache->policy, oblock, cblock, dirty, hint, hint_valid);
2848}
2849
2850static int load_filtered_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock,
2851 bool dirty, uint32_t hint, bool hint_valid)
2852{
2853 struct cache *cache = context;
2854
2855 if (from_oblock(oblock) >= from_oblock(cache->origin_blocks)) {
2856 if (dirty) {
2857 DMERR("%s: unable to shrink origin; cache block %u is dirty",
2858 cache_device_name(cache), from_cblock(cblock));
2859 return -EFBIG;
2860 }
2861 set_bit(from_cblock(cblock), cache->invalid_bitset);
2862 return 0;
2863 }
2864
2865 return load_mapping(context, oblock, cblock, dirty, hint, hint_valid);
2866}
2867
2868/*
2869 * The discard block size in the on disk metadata is not
2870 * necessarily the same as we're currently using. So we have to
2871 * be careful to only set the discarded attribute if we know it
2872 * covers a complete block of the new size.
2873 */
2874struct discard_load_info {
2875 struct cache *cache;
2876
2877 /*
2878 * These blocks are sized using the on disk dblock size, rather
2879 * than the current one.
2880 */
2881 dm_block_t block_size;
2882 dm_block_t discard_begin, discard_end;
2883};
2884
2885static void discard_load_info_init(struct cache *cache,
2886 struct discard_load_info *li)
2887{
2888 li->cache = cache;
2889 li->discard_begin = li->discard_end = 0;
2890}
2891
2892static void set_discard_range(struct discard_load_info *li)
2893{
2894 sector_t b, e;
2895
2896 if (li->discard_begin == li->discard_end)
2897 return;
2898
2899 /*
2900 * Convert to sectors.
2901 */
2902 b = li->discard_begin * li->block_size;
2903 e = li->discard_end * li->block_size;
2904
2905 /*
2906 * Then convert back to the current dblock size.
2907 */
2908 b = dm_sector_div_up(b, li->cache->discard_block_size);
2909 sector_div(e, li->cache->discard_block_size);
2910
2911 /*
2912 * The origin may have shrunk, so we need to check we're still in
2913 * bounds.
2914 */
2915 if (e > from_dblock(li->cache->discard_nr_blocks))
2916 e = from_dblock(li->cache->discard_nr_blocks);
2917
2918 for (; b < e; b++)
2919 set_discard(li->cache, to_dblock(b));
2920}
2921
2922static int load_discard(void *context, sector_t discard_block_size,
2923 dm_dblock_t dblock, bool discard)
2924{
2925 struct discard_load_info *li = context;
2926
2927 li->block_size = discard_block_size;
2928
2929 if (discard) {
2930 if (from_dblock(dblock) == li->discard_end)
2931 /*
2932 * We're already in a discard range, just extend it.
2933 */
2934 li->discard_end = li->discard_end + 1ULL;
2935
2936 else {
2937 /*
2938 * Emit the old range and start a new one.
2939 */
2940 set_discard_range(li);
2941 li->discard_begin = from_dblock(dblock);
2942 li->discard_end = li->discard_begin + 1ULL;
2943 }
2944 } else {
2945 set_discard_range(li);
2946 li->discard_begin = li->discard_end = 0;
2947 }
2948
2949 return 0;
2950}
2951
2952static dm_cblock_t get_cache_dev_size(struct cache *cache)
2953{
2954 sector_t size = get_dev_size(cache->cache_dev);
2955 (void) sector_div(size, cache->sectors_per_block);
2956 return to_cblock(size);
2957}
2958
2959static bool can_resume(struct cache *cache)
2960{
2961 bool clean_when_opened;
2962 int r;
2963
2964 /*
2965 * Disallow retrying the resume operation for devices that failed the
2966 * first resume attempt, as the failure leaves the policy object partially
2967 * initialized. Retrying could trigger BUG_ON when loading cache mappings
2968 * into the incomplete policy object.
2969 */
2970 if (cache->sized && !cache->loaded_mappings) {
2971 if (get_cache_mode(cache) != CM_WRITE)
2972 DMERR("%s: unable to resume a failed-loaded cache, please check metadata.",
2973 cache_device_name(cache));
2974 else
2975 DMERR("%s: unable to resume cache due to missing proper cache table reload",
2976 cache_device_name(cache));
2977 return false;
2978 }
2979
2980 if (passthrough_mode(cache)) {
2981 r = dm_cache_metadata_clean_when_opened(cache->cmd, &clean_when_opened);
2982 if (r) {
2983 DMERR("%s: failed to query metadata flags", cache_device_name(cache));
2984 return false;
2985 }
2986
2987 if (!clean_when_opened) {
2988 DMERR("%s: unable to resume into passthrough mode after unclean shutdown",
2989 cache_device_name(cache));
2990 return false;
2991 }
2992 }
2993
2994 return true;
2995}
2996
2997static bool can_resize(struct cache *cache, dm_cblock_t new_size)
2998{
2999 if (from_cblock(new_size) > from_cblock(cache->cache_size)) {
3000 DMERR("%s: unable to extend cache due to missing cache table reload",
3001 cache_device_name(cache));
3002 return false;
3003 }
3004
3005 /*
3006 * We can't drop a dirty block when shrinking the cache.
3007 */
3008 if (cache->loaded_mappings) {
3009 new_size = to_cblock(find_next_bit(cache->dirty_bitset,
3010 from_cblock(cache->cache_size),
3011 from_cblock(new_size)));
3012 if (new_size != cache->cache_size) {
3013 DMERR("%s: unable to shrink cache; cache block %llu is dirty",
3014 cache_device_name(cache),
3015 (unsigned long long) from_cblock(new_size));
3016 return false;
3017 }
3018 }
3019
3020 return true;
3021}
3022
3023static int resize_cache_dev(struct cache *cache, dm_cblock_t new_size)
3024{
3025 int r;
3026
3027 r = dm_cache_resize(cache->cmd, new_size);
3028 if (r) {
3029 DMERR("%s: could not resize cache metadata", cache_device_name(cache));
3030 metadata_operation_failed(cache, "dm_cache_resize", r);
3031 return r;
3032 }
3033
3034 set_cache_size(cache, new_size);
3035
3036 return 0;
3037}
3038
3039static int truncate_oblocks(struct cache *cache)
3040{
3041 uint32_t nr_blocks = from_cblock(cache->cache_size);
3042 uint32_t i;
3043 int r;
3044
3045 for_each_set_bit(i, cache->invalid_bitset, nr_blocks) {
3046 r = dm_cache_remove_mapping(cache->cmd, to_cblock(i));
3047 if (r) {
3048 DMERR_LIMIT("%s: invalidation failed; couldn't update on disk metadata",
3049 cache_device_name(cache));
3050 return r;
3051 }
3052 }
3053
3054 return 0;
3055}
3056
3057static int cache_preresume(struct dm_target *ti)
3058{
3059 int r = 0;
3060 struct cache *cache = ti->private;
3061 dm_cblock_t csize = get_cache_dev_size(cache);
3062
3063 if (!can_resume(cache))
3064 return -EINVAL;
3065
3066 /*
3067 * Check to see if the cache has resized.
3068 */
3069 if (!cache->sized || csize != cache->cache_size) {
3070 if (!can_resize(cache, csize))
3071 return -EINVAL;
3072
3073 r = resize_cache_dev(cache, csize);
3074 if (r)
3075 return r;
3076
3077 cache->sized = true;
3078 }
3079
3080 if (!cache->loaded_mappings) {
3081 /*
3082 * The fast device could have been resized since the last
3083 * failed preresume attempt. To be safe we start by a blank
3084 * bitset for cache blocks.
3085 */
3086 clear_bitset(cache->invalid_bitset, from_cblock(cache->cache_size));
3087
3088 r = dm_cache_load_mappings(cache->cmd, cache->policy,
3089 load_filtered_mapping, cache);
3090 if (r) {
3091 DMERR("%s: could not load cache mappings", cache_device_name(cache));
3092 if (r != -EFBIG && r != -EBUSY)
3093 metadata_operation_failed(cache, "dm_cache_load_mappings", r);
3094 return r;
3095 }
3096
3097 r = truncate_oblocks(cache);
3098 if (r) {
3099 metadata_operation_failed(cache, "dm_cache_remove_mapping", r);
3100 return r;
3101 }
3102
3103 cache->loaded_mappings = true;
3104 }
3105
3106 if (!cache->loaded_discards) {
3107 struct discard_load_info li;
3108
3109 /*
3110 * The discard bitset could have been resized, or the
3111 * discard block size changed. To be safe we start by
3112 * setting every dblock to not discarded.
3113 */
3114 clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks));
3115
3116 discard_load_info_init(cache, &li);
3117 r = dm_cache_load_discards(cache->cmd, load_discard, &li);
3118 if (r) {
3119 DMERR("%s: could not load origin discards", cache_device_name(cache));
3120 metadata_operation_failed(cache, "dm_cache_load_discards", r);
3121 return r;
3122 }
3123 set_discard_range(&li);
3124
3125 cache->loaded_discards = true;
3126 }
3127
3128 return r;
3129}
3130
3131static void cache_resume(struct dm_target *ti)
3132{
3133 struct cache *cache = ti->private;
3134
3135 cache->need_tick_bio = true;
3136 allow_background_work(cache);
3137 do_waker(&cache->waker.work);
3138}
3139
3140static void emit_flags(struct cache *cache, char *result,
3141 unsigned int maxlen, ssize_t *sz_ptr)
3142{
3143 ssize_t sz = *sz_ptr;
3144 struct cache_features *cf = &cache->features;
3145 unsigned int count = (cf->metadata_version == 2) + !cf->discard_passdown + 1;
3146
3147 DMEMIT("%u ", count);
3148
3149 if (cf->metadata_version == 2)
3150 DMEMIT("metadata2 ");
3151
3152 if (writethrough_mode(cache))
3153 DMEMIT("writethrough ");
3154
3155 else if (passthrough_mode(cache))
3156 DMEMIT("passthrough ");
3157
3158 else if (writeback_mode(cache))
3159 DMEMIT("writeback ");
3160
3161 else {
3162 DMEMIT("unknown ");
3163 DMERR("%s: internal error: unknown io mode: %d",
3164 cache_device_name(cache), (int) cf->io_mode);
3165 }
3166
3167 if (!cf->discard_passdown)
3168 DMEMIT("no_discard_passdown ");
3169
3170 *sz_ptr = sz;
3171}
3172
3173/*
3174 * Status format:
3175 *
3176 * <metadata block size> <#used metadata blocks>/<#total metadata blocks>
3177 * <cache block size> <#used cache blocks>/<#total cache blocks>
3178 * <#read hits> <#read misses> <#write hits> <#write misses>
3179 * <#demotions> <#promotions> <#dirty>
3180 * <#features> <features>*
3181 * <#core args> <core args>
3182 * <policy name> <#policy args> <policy args>* <cache metadata mode> <needs_check>
3183 */
3184static void cache_status(struct dm_target *ti, status_type_t type,
3185 unsigned int status_flags, char *result, unsigned int maxlen)
3186{
3187 int r = 0;
3188 unsigned int i;
3189 ssize_t sz = 0;
3190 dm_block_t nr_free_blocks_metadata = 0;
3191 dm_block_t nr_blocks_metadata = 0;
3192 char buf[BDEVNAME_SIZE];
3193 struct cache *cache = ti->private;
3194 dm_cblock_t residency;
3195 bool needs_check;
3196
3197 switch (type) {
3198 case STATUSTYPE_INFO:
3199 if (get_cache_mode(cache) == CM_FAIL) {
3200 DMEMIT("Fail");
3201 break;
3202 }
3203
3204 /* Commit to ensure statistics aren't out-of-date */
3205 if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))
3206 (void) commit(cache, false);
3207
3208 r = dm_cache_get_free_metadata_block_count(cache->cmd, &nr_free_blocks_metadata);
3209 if (r) {
3210 DMERR("%s: dm_cache_get_free_metadata_block_count returned %d",
3211 cache_device_name(cache), r);
3212 goto err;
3213 }
3214
3215 r = dm_cache_get_metadata_dev_size(cache->cmd, &nr_blocks_metadata);
3216 if (r) {
3217 DMERR("%s: dm_cache_get_metadata_dev_size returned %d",
3218 cache_device_name(cache), r);
3219 goto err;
3220 }
3221
3222 residency = policy_residency(cache->policy);
3223
3224 DMEMIT("%u %llu/%llu %llu %llu/%llu %u %u %u %u %u %u %lu ",
3225 (unsigned int)DM_CACHE_METADATA_BLOCK_SIZE,
3226 (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
3227 (unsigned long long)nr_blocks_metadata,
3228 (unsigned long long)cache->sectors_per_block,
3229 (unsigned long long) from_cblock(residency),
3230 (unsigned long long) from_cblock(cache->cache_size),
3231 (unsigned int) atomic_read(&cache->stats.read_hit),
3232 (unsigned int) atomic_read(&cache->stats.read_miss),
3233 (unsigned int) atomic_read(&cache->stats.write_hit),
3234 (unsigned int) atomic_read(&cache->stats.write_miss),
3235 (unsigned int) atomic_read(&cache->stats.demotion),
3236 (unsigned int) atomic_read(&cache->stats.promotion),
3237 (unsigned long) atomic_read(&cache->nr_dirty));
3238
3239 emit_flags(cache, result, maxlen, &sz);
3240
3241 DMEMIT("2 migration_threshold %llu ", (unsigned long long) cache->migration_threshold);
3242
3243 DMEMIT("%s ", dm_cache_policy_get_name(cache->policy));
3244 if (sz < maxlen) {
3245 r = policy_emit_config_values(cache->policy, result, maxlen, &sz);
3246 if (r)
3247 DMERR("%s: policy_emit_config_values returned %d",
3248 cache_device_name(cache), r);
3249 }
3250
3251 if (get_cache_mode(cache) == CM_READ_ONLY)
3252 DMEMIT("ro ");
3253 else
3254 DMEMIT("rw ");
3255
3256 r = dm_cache_metadata_needs_check(cache->cmd, &needs_check);
3257
3258 if (r || needs_check)
3259 DMEMIT("needs_check ");
3260 else
3261 DMEMIT("- ");
3262
3263 break;
3264
3265 case STATUSTYPE_TABLE:
3266 format_dev_t(buf, cache->metadata_dev->bdev->bd_dev);
3267 DMEMIT("%s ", buf);
3268 format_dev_t(buf, cache->cache_dev->bdev->bd_dev);
3269 DMEMIT("%s ", buf);
3270 format_dev_t(buf, cache->origin_dev->bdev->bd_dev);
3271 DMEMIT("%s", buf);
3272
3273 for (i = 0; i < cache->nr_ctr_args - 1; i++)
3274 DMEMIT(" %s", cache->ctr_args[i]);
3275 if (cache->nr_ctr_args)
3276 DMEMIT(" %s", cache->ctr_args[cache->nr_ctr_args - 1]);
3277 break;
3278
3279 case STATUSTYPE_IMA:
3280 DMEMIT_TARGET_NAME_VERSION(ti->type);
3281 if (get_cache_mode(cache) == CM_FAIL)
3282 DMEMIT(",metadata_mode=fail");
3283 else if (get_cache_mode(cache) == CM_READ_ONLY)
3284 DMEMIT(",metadata_mode=ro");
3285 else
3286 DMEMIT(",metadata_mode=rw");
3287
3288 format_dev_t(buf, cache->metadata_dev->bdev->bd_dev);
3289 DMEMIT(",cache_metadata_device=%s", buf);
3290 format_dev_t(buf, cache->cache_dev->bdev->bd_dev);
3291 DMEMIT(",cache_device=%s", buf);
3292 format_dev_t(buf, cache->origin_dev->bdev->bd_dev);
3293 DMEMIT(",cache_origin_device=%s", buf);
3294 DMEMIT(",writethrough=%c", writethrough_mode(cache) ? 'y' : 'n');
3295 DMEMIT(",writeback=%c", writeback_mode(cache) ? 'y' : 'n');
3296 DMEMIT(",passthrough=%c", passthrough_mode(cache) ? 'y' : 'n');
3297 DMEMIT(",metadata2=%c", cache->features.metadata_version == 2 ? 'y' : 'n');
3298 DMEMIT(",no_discard_passdown=%c", cache->features.discard_passdown ? 'n' : 'y');
3299 DMEMIT(";");
3300 break;
3301 }
3302
3303 return;
3304
3305err:
3306 DMEMIT("Error");
3307}
3308
3309/*
3310 * Defines a range of cblocks, begin to (end - 1) are in the range. end is
3311 * the one-past-the-end value.
3312 */
3313struct cblock_range {
3314 dm_cblock_t begin;
3315 dm_cblock_t end;
3316};
3317
3318static inline dm_cblock_t cblock_succ(dm_cblock_t b)
3319{
3320 return to_cblock(from_cblock(b) + 1);
3321}
3322
3323/*
3324 * A cache block range can take two forms:
3325 *
3326 * i) A single cblock, eg. '3456'
3327 * ii) A begin and end cblock with a dash between, eg. 123-234
3328 */
3329static int parse_cblock_range(struct cache *cache, char *str,
3330 struct cblock_range *result)
3331{
3332 char *blocknr = strsep(&str, "-");
3333 unsigned int b, e;
3334 int r;
3335
3336 r = kstrtouint(blocknr, 10, &b);
3337 if (r)
3338 goto bad;
3339
3340 result->begin = to_cblock(b);
3341
3342 if (str) {
3343 blocknr = str;
3344
3345 r = kstrtouint(blocknr, 10, &e);
3346 if (r)
3347 goto bad;
3348
3349 result->end = to_cblock(e);
3350 } else {
3351 result->end = cblock_succ(result->begin);
3352 }
3353
3354 return 0;
3355
3356bad:
3357 DMERR("%s: invalid cblock range '%s'", cache_device_name(cache), blocknr);
3358 return -EINVAL;
3359}
3360
3361static int validate_cblock_range(struct cache *cache, struct cblock_range *range)
3362{
3363 uint64_t b = from_cblock(range->begin);
3364 uint64_t e = from_cblock(range->end);
3365 uint64_t n = from_cblock(cache->cache_size);
3366
3367 if (b >= n) {
3368 DMERR("%s: begin cblock out of range: %llu >= %llu",
3369 cache_device_name(cache), b, n);
3370 return -EINVAL;
3371 }
3372
3373 if (e > n) {
3374 DMERR("%s: end cblock out of range: %llu > %llu",
3375 cache_device_name(cache), e, n);
3376 return -EINVAL;
3377 }
3378
3379 if (b >= e) {
3380 DMERR("%s: invalid cblock range: %llu >= %llu",
3381 cache_device_name(cache), b, e);
3382 return -EINVAL;
3383 }
3384
3385 return 0;
3386}
3387
3388static int request_invalidation(struct cache *cache, struct cblock_range *range)
3389{
3390 int r = 0;
3391
3392 /*
3393 * We don't need to do any locking here because we know we're in
3394 * passthrough mode. There's is potential for a race between an
3395 * invalidation triggered by an io and an invalidation message. This
3396 * is harmless, we must not worry if the policy call fails.
3397 */
3398 while (range->begin != range->end) {
3399 r = invalidate_cblock(cache, range->begin);
3400 if (r)
3401 return r;
3402
3403 range->begin = cblock_succ(range->begin);
3404 }
3405
3406 cache->commit_requested = true;
3407 return r;
3408}
3409
3410static int process_invalidate_cblocks_message(struct cache *cache, unsigned int count,
3411 char **cblock_ranges)
3412{
3413 int r = 0;
3414 unsigned int i;
3415 struct cblock_range range;
3416
3417 if (!passthrough_mode(cache)) {
3418 DMERR("%s: cache has to be in passthrough mode for invalidation",
3419 cache_device_name(cache));
3420 return -EPERM;
3421 }
3422
3423 for (i = 0; i < count; i++) {
3424 r = parse_cblock_range(cache, cblock_ranges[i], &range);
3425 if (r)
3426 break;
3427
3428 r = validate_cblock_range(cache, &range);
3429 if (r)
3430 break;
3431
3432 /*
3433 * Pass begin and end origin blocks to the worker and wake it.
3434 */
3435 r = request_invalidation(cache, &range);
3436 if (r)
3437 break;
3438 }
3439
3440 return r;
3441}
3442
3443/*
3444 * Supports
3445 * "<key> <value>"
3446 * and
3447 * "invalidate_cblocks [(<begin>)|(<begin>-<end>)]*
3448 *
3449 * The key migration_threshold is supported by the cache target core.
3450 */
3451static int cache_message(struct dm_target *ti, unsigned int argc, char **argv,
3452 char *result, unsigned int maxlen)
3453{
3454 struct cache *cache = ti->private;
3455
3456 if (!argc)
3457 return -EINVAL;
3458
3459 if (get_cache_mode(cache) >= CM_READ_ONLY) {
3460 DMERR("%s: unable to service cache target messages in READ_ONLY or FAIL mode",
3461 cache_device_name(cache));
3462 return -EOPNOTSUPP;
3463 }
3464
3465 if (!strcasecmp(argv[0], "invalidate_cblocks"))
3466 return process_invalidate_cblocks_message(cache, argc - 1, argv + 1);
3467
3468 if (argc != 2)
3469 return -EINVAL;
3470
3471 return set_config_value(cache, argv[0], argv[1]);
3472}
3473
3474static int cache_iterate_devices(struct dm_target *ti,
3475 iterate_devices_callout_fn fn, void *data)
3476{
3477 int r = 0;
3478 struct cache *cache = ti->private;
3479
3480 r = fn(ti, cache->cache_dev, 0, get_dev_size(cache->cache_dev), data);
3481 if (!r)
3482 r = fn(ti, cache->origin_dev, 0, ti->len, data);
3483
3484 return r;
3485}
3486
3487/*
3488 * If discard_passdown was enabled verify that the origin device
3489 * supports discards. Disable discard_passdown if not.
3490 */
3491static void disable_passdown_if_not_supported(struct cache *cache)
3492{
3493 struct block_device *origin_bdev = cache->origin_dev->bdev;
3494 struct queue_limits *origin_limits = bdev_limits(origin_bdev);
3495 const char *reason = NULL;
3496
3497 if (!cache->features.discard_passdown)
3498 return;
3499
3500 if (!bdev_max_discard_sectors(origin_bdev))
3501 reason = "discard unsupported";
3502
3503 else if (origin_limits->max_discard_sectors < cache->sectors_per_block)
3504 reason = "max discard sectors smaller than a block";
3505
3506 if (reason) {
3507 DMWARN("Origin device (%pg) %s: Disabling discard passdown.",
3508 origin_bdev, reason);
3509 cache->features.discard_passdown = false;
3510 }
3511}
3512
3513static void set_discard_limits(struct cache *cache, struct queue_limits *limits)
3514{
3515 struct block_device *origin_bdev = cache->origin_dev->bdev;
3516 struct queue_limits *origin_limits = bdev_limits(origin_bdev);
3517
3518 if (!cache->features.discard_passdown) {
3519 /* No passdown is done so setting own virtual limits */
3520 limits->max_hw_discard_sectors = min_t(sector_t, cache->discard_block_size * 1024,
3521 cache->origin_sectors);
3522 limits->discard_granularity = cache->discard_block_size << SECTOR_SHIFT;
3523 return;
3524 }
3525
3526 /*
3527 * cache_iterate_devices() is stacking both origin and fast device limits
3528 * but discards aren't passed to fast device, so inherit origin's limits.
3529 */
3530 limits->max_hw_discard_sectors = origin_limits->max_hw_discard_sectors;
3531 limits->discard_granularity = origin_limits->discard_granularity;
3532 limits->discard_alignment = origin_limits->discard_alignment;
3533}
3534
3535static void cache_io_hints(struct dm_target *ti, struct queue_limits *limits)
3536{
3537 struct cache *cache = ti->private;
3538 uint64_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
3539
3540 /*
3541 * If the system-determined stacked limits are compatible with the
3542 * cache's blocksize (io_opt is a factor) do not override them.
3543 */
3544 if (io_opt_sectors < cache->sectors_per_block ||
3545 do_div(io_opt_sectors, cache->sectors_per_block)) {
3546 limits->io_min = cache->sectors_per_block << SECTOR_SHIFT;
3547 limits->io_opt = cache->sectors_per_block << SECTOR_SHIFT;
3548 }
3549
3550 disable_passdown_if_not_supported(cache);
3551 set_discard_limits(cache, limits);
3552}
3553
3554/*----------------------------------------------------------------*/
3555
3556static struct target_type cache_target = {
3557 .name = "cache",
3558 .version = {2, 4, 0},
3559 .module = THIS_MODULE,
3560 .ctr = cache_ctr,
3561 .dtr = cache_dtr,
3562 .map = cache_map,
3563 .end_io = cache_end_io,
3564 .postsuspend = cache_postsuspend,
3565 .preresume = cache_preresume,
3566 .resume = cache_resume,
3567 .status = cache_status,
3568 .message = cache_message,
3569 .iterate_devices = cache_iterate_devices,
3570 .io_hints = cache_io_hints,
3571};
3572
3573static int __init dm_cache_init(void)
3574{
3575 int r;
3576
3577 migration_cache = KMEM_CACHE(dm_cache_migration, 0);
3578 if (!migration_cache) {
3579 r = -ENOMEM;
3580 goto err;
3581 }
3582
3583 btracker_work_cache = kmem_cache_create("dm_cache_bt_work",
3584 sizeof(struct bt_work), __alignof__(struct bt_work), 0, NULL);
3585 if (!btracker_work_cache) {
3586 r = -ENOMEM;
3587 goto err;
3588 }
3589
3590 r = dm_register_target(&cache_target);
3591 if (r) {
3592 goto err;
3593 }
3594
3595 return 0;
3596
3597err:
3598 kmem_cache_destroy(migration_cache);
3599 kmem_cache_destroy(btracker_work_cache);
3600 return r;
3601}
3602
3603static void __exit dm_cache_exit(void)
3604{
3605 dm_unregister_target(&cache_target);
3606 kmem_cache_destroy(migration_cache);
3607 kmem_cache_destroy(btracker_work_cache);
3608}
3609
3610module_init(dm_cache_init);
3611module_exit(dm_cache_exit);
3612
3613MODULE_DESCRIPTION(DM_NAME " cache target");
3614MODULE_AUTHOR("Joe Thornber <[email protected]>");
3615MODULE_LICENSE("GPL");