projects/linux·blob master · refs

2135 lines · 47.8 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2011-2012 Red Hat, Inc.
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm-thin-metadata.h"
9#include "persistent-data/dm-btree.h"
10#include "persistent-data/dm-space-map.h"
11#include "persistent-data/dm-space-map-disk.h"
12#include "persistent-data/dm-transaction-manager.h"
13
14#include <linux/list.h>
15#include <linux/device-mapper.h>
16#include <linux/workqueue.h>
17
18/*
19 *--------------------------------------------------------------------------
20 * As far as the metadata goes, there is:
21 *
22 * - A superblock in block zero, taking up fewer than 512 bytes for
23 * atomic writes.
24 *
25 * - A space map managing the metadata blocks.
26 *
27 * - A space map managing the data blocks.
28 *
29 * - A btree mapping our internal thin dev ids onto struct disk_device_details.
30 *
31 * - A hierarchical btree, with 2 levels which effectively maps (thin
32 * dev id, virtual block) -> block_time. Block time is a 64-bit
33 * field holding the time in the low 24 bits, and block in the top 40
34 * bits.
35 *
36 * BTrees consist solely of btree_nodes, that fill a block. Some are
37 * internal nodes, as such their values are a __le64 pointing to other
38 * nodes. Leaf nodes can store data of any reasonable size (ie. much
39 * smaller than the block size). The nodes consist of the header,
40 * followed by an array of keys, followed by an array of values. We have
41 * to binary search on the keys so they're all held together to help the
42 * cpu cache.
43 *
44 * Space maps have 2 btrees:
45 *
46 * - One maps a uint64_t onto a struct index_entry. Which points to a
47 * bitmap block, and has some details about how many free entries there
48 * are etc.
49 *
50 * - The bitmap blocks have a header (for the checksum). Then the rest
51 * of the block is pairs of bits. With the meaning being:
52 *
53 * 0 - ref count is 0
54 * 1 - ref count is 1
55 * 2 - ref count is 2
56 * 3 - ref count is higher than 2
57 *
58 * - If the count is higher than 2 then the ref count is entered in a
59 * second btree that directly maps the block_address to a uint32_t ref
60 * count.
61 *
62 * The space map metadata variant doesn't have a bitmaps btree. Instead
63 * it has one single blocks worth of index_entries. This avoids
64 * recursive issues with the bitmap btree needing to allocate space in
65 * order to insert. With a small data block size such as 64k the
66 * metadata support data devices that are hundreds of terrabytes.
67 *
68 * The space maps allocate space linearly from front to back. Space that
69 * is freed in a transaction is never recycled within that transaction.
70 * To try and avoid fragmenting _free_ space the allocator always goes
71 * back and fills in gaps.
72 *
73 * All metadata io is in THIN_METADATA_BLOCK_SIZE sized/aligned chunks
74 * from the block manager.
75 *--------------------------------------------------------------------------
76 */
77
78#define DM_MSG_PREFIX "thin metadata"
79
80#define THIN_SUPERBLOCK_MAGIC 27022010
81#define THIN_SUPERBLOCK_LOCATION 0
82#define THIN_VERSION 2
83#define SECTOR_TO_BLOCK_SHIFT 3
84
85/*
86 * For btree insert:
87 * 3 for btree insert +
88 * 2 for btree lookup used within space map
89 * For btree remove:
90 * 2 for shadow spine +
91 * 4 for rebalance 3 child node
92 */
93#define THIN_MAX_CONCURRENT_LOCKS 6
94
95/* This should be plenty */
96#define SPACE_MAP_ROOT_SIZE 128
97
98/*
99 * Little endian on-disk superblock and device details.
100 */
101struct thin_disk_superblock {
102 __le32 csum; /* Checksum of superblock except for this field. */
103 __le32 flags;
104 __le64 blocknr; /* This block number, dm_block_t. */
105
106 __u8 uuid[16];
107 __le64 magic;
108 __le32 version;
109 __le32 time;
110
111 __le64 trans_id;
112
113 /*
114 * Root held by userspace transactions.
115 */
116 __le64 held_root;
117
118 __u8 data_space_map_root[SPACE_MAP_ROOT_SIZE];
119 __u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
120
121 /*
122 * 2-level btree mapping (dev_id, (dev block, time)) -> data block
123 */
124 __le64 data_mapping_root;
125
126 /*
127 * Device detail root mapping dev_id -> device_details
128 */
129 __le64 device_details_root;
130
131 __le32 data_block_size; /* In 512-byte sectors. */
132
133 __le32 metadata_block_size; /* In 512-byte sectors. */
134 __le64 metadata_nr_blocks;
135
136 __le32 compat_flags;
137 __le32 compat_ro_flags;
138 __le32 incompat_flags;
139} __packed;
140
141struct disk_device_details {
142 __le64 mapped_blocks;
143 __le64 transaction_id; /* When created. */
144 __le32 creation_time;
145 __le32 snapshotted_time;
146} __packed;
147
148struct dm_pool_metadata {
149 struct hlist_node hash;
150
151 struct block_device *bdev;
152 struct dm_block_manager *bm;
153 struct dm_space_map *metadata_sm;
154 struct dm_space_map *data_sm;
155 struct dm_transaction_manager *tm;
156 struct dm_transaction_manager *nb_tm;
157
158 /*
159 * Two-level btree.
160 * First level holds thin_dev_t.
161 * Second level holds mappings.
162 */
163 struct dm_btree_info info;
164
165 /*
166 * Non-blocking version of the above.
167 */
168 struct dm_btree_info nb_info;
169
170 /*
171 * Just the top level for deleting whole devices.
172 */
173 struct dm_btree_info tl_info;
174
175 /*
176 * Just the bottom level for creating new devices.
177 */
178 struct dm_btree_info bl_info;
179
180 /*
181 * Describes the device details btree.
182 */
183 struct dm_btree_info details_info;
184
185 struct rw_semaphore root_lock;
186 uint32_t time;
187 dm_block_t root;
188 dm_block_t details_root;
189 dm_block_t held_root;
190 struct list_head thin_devices;
191 uint64_t trans_id;
192 unsigned long flags;
193 sector_t data_block_size;
194
195 /*
196 * Pre-commit callback.
197 *
198 * This allows the thin provisioning target to run a callback before
199 * the metadata are committed.
200 */
201 dm_pool_pre_commit_fn pre_commit_fn;
202 void *pre_commit_context;
203
204 /*
205 * We reserve a section of the metadata for commit overhead.
206 * All reported space does *not* include this.
207 */
208 dm_block_t metadata_reserve;
209
210 /*
211 * Set if a transaction has to be aborted but the attempt to roll back
212 * to the previous (good) transaction failed. The only pool metadata
213 * operation possible in this state is the closing of the device.
214 */
215 bool fail_io:1;
216
217 /*
218 * Set once a thin-pool has been accessed through one of the interfaces
219 * that imply the pool is in-service (e.g. thin devices created/deleted,
220 * thin-pool message, metadata snapshots, etc).
221 */
222 bool in_service:1;
223
224 /*
225 * Reading the space map roots can fail, so we read it into these
226 * buffers before the superblock is locked and updated.
227 */
228 __u8 data_space_map_root[SPACE_MAP_ROOT_SIZE];
229 __u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
230};
231
232struct dm_thin_device {
233 struct list_head list;
234 struct dm_pool_metadata *pmd;
235 dm_thin_id id;
236
237 int open_count;
238 bool changed:1;
239 bool aborted_with_changes:1;
240 uint64_t mapped_blocks;
241 uint64_t transaction_id;
242 uint32_t creation_time;
243 uint32_t snapshotted_time;
244};
245
246/*
247 *--------------------------------------------------------------
248 * superblock validator
249 *--------------------------------------------------------------
250 */
251#define SUPERBLOCK_CSUM_XOR 160774
252
253static void sb_prepare_for_write(const struct dm_block_validator *v,
254 struct dm_block *b,
255 size_t block_size)
256{
257 struct thin_disk_superblock *disk_super = dm_block_data(b);
258
259 disk_super->blocknr = cpu_to_le64(dm_block_location(b));
260 disk_super->csum = cpu_to_le32(dm_bm_checksum(&disk_super->flags,
261 block_size - sizeof(__le32),
262 SUPERBLOCK_CSUM_XOR));
263}
264
265static int sb_check(const struct dm_block_validator *v,
266 struct dm_block *b,
267 size_t block_size)
268{
269 struct thin_disk_superblock *disk_super = dm_block_data(b);
270 __le32 csum_le;
271
272 if (dm_block_location(b) != le64_to_cpu(disk_super->blocknr)) {
273 DMERR("%s failed: blocknr %llu: wanted %llu",
274 __func__, le64_to_cpu(disk_super->blocknr),
275 (unsigned long long)dm_block_location(b));
276 return -ENOTBLK;
277 }
278
279 if (le64_to_cpu(disk_super->magic) != THIN_SUPERBLOCK_MAGIC) {
280 DMERR("%s failed: magic %llu: wanted %llu",
281 __func__, le64_to_cpu(disk_super->magic),
282 (unsigned long long)THIN_SUPERBLOCK_MAGIC);
283 return -EILSEQ;
284 }
285
286 csum_le = cpu_to_le32(dm_bm_checksum(&disk_super->flags,
287 block_size - sizeof(__le32),
288 SUPERBLOCK_CSUM_XOR));
289 if (csum_le != disk_super->csum) {
290 DMERR("%s failed: csum %u: wanted %u",
291 __func__, le32_to_cpu(csum_le), le32_to_cpu(disk_super->csum));
292 return -EILSEQ;
293 }
294
295 return 0;
296}
297
298static const struct dm_block_validator sb_validator = {
299 .name = "superblock",
300 .prepare_for_write = sb_prepare_for_write,
301 .check = sb_check
302};
303
304/*
305 *--------------------------------------------------------------
306 * Methods for the btree value types
307 *--------------------------------------------------------------
308 */
309static uint64_t pack_block_time(dm_block_t b, uint32_t t)
310{
311 return (b << 24) | t;
312}
313
314static void unpack_block_time(uint64_t v, dm_block_t *b, uint32_t *t)
315{
316 *b = v >> 24;
317 *t = v & ((1 << 24) - 1);
318}
319
320/*
321 * It's more efficient to call dm_sm_{inc,dec}_blocks as few times as
322 * possible. 'with_runs' reads contiguous runs of blocks, and calls the
323 * given sm function.
324 */
325typedef int (*run_fn)(struct dm_space_map *, dm_block_t, dm_block_t);
326
327static void with_runs(struct dm_space_map *sm, const __le64 *value_le, unsigned int count, run_fn fn)
328{
329 uint64_t b, begin, end;
330 uint32_t t;
331 bool in_run = false;
332 unsigned int i;
333
334 for (i = 0; i < count; i++, value_le++) {
335 /* We know value_le is 8 byte aligned */
336 unpack_block_time(le64_to_cpu(*value_le), &b, &t);
337
338 if (in_run) {
339 if (b == end) {
340 end++;
341 } else {
342 fn(sm, begin, end);
343 begin = b;
344 end = b + 1;
345 }
346 } else {
347 in_run = true;
348 begin = b;
349 end = b + 1;
350 }
351 }
352
353 if (in_run)
354 fn(sm, begin, end);
355}
356
357static void data_block_inc(void *context, const void *value_le, unsigned int count)
358{
359 with_runs((struct dm_space_map *) context,
360 (const __le64 *) value_le, count, dm_sm_inc_blocks);
361}
362
363static void data_block_dec(void *context, const void *value_le, unsigned int count)
364{
365 with_runs((struct dm_space_map *) context,
366 (const __le64 *) value_le, count, dm_sm_dec_blocks);
367}
368
369static int data_block_equal(void *context, const void *value1_le, const void *value2_le)
370{
371 __le64 v1_le, v2_le;
372 uint64_t b1, b2;
373 uint32_t t;
374
375 memcpy(&v1_le, value1_le, sizeof(v1_le));
376 memcpy(&v2_le, value2_le, sizeof(v2_le));
377 unpack_block_time(le64_to_cpu(v1_le), &b1, &t);
378 unpack_block_time(le64_to_cpu(v2_le), &b2, &t);
379
380 return b1 == b2;
381}
382
383static void subtree_inc(void *context, const void *value, unsigned int count)
384{
385 struct dm_btree_info *info = context;
386 const __le64 *root_le = value;
387 unsigned int i;
388
389 for (i = 0; i < count; i++, root_le++)
390 dm_tm_inc(info->tm, le64_to_cpu(*root_le));
391}
392
393static void subtree_dec(void *context, const void *value, unsigned int count)
394{
395 struct dm_btree_info *info = context;
396 const __le64 *root_le = value;
397 unsigned int i;
398
399 for (i = 0; i < count; i++, root_le++)
400 if (dm_btree_del(info, le64_to_cpu(*root_le)))
401 DMERR("btree delete failed");
402}
403
404static int subtree_equal(void *context, const void *value1_le, const void *value2_le)
405{
406 __le64 v1_le, v2_le;
407
408 memcpy(&v1_le, value1_le, sizeof(v1_le));
409 memcpy(&v2_le, value2_le, sizeof(v2_le));
410
411 return v1_le == v2_le;
412}
413
414/*----------------------------------------------------------------*/
415
416/*
417 * Variant that is used for in-core only changes or code that
418 * shouldn't put the pool in service on its own (e.g. commit).
419 */
420static inline void pmd_write_lock_in_core(struct dm_pool_metadata *pmd)
421 __acquires(pmd->root_lock)
422{
423 down_write(&pmd->root_lock);
424}
425
426static inline void pmd_write_lock(struct dm_pool_metadata *pmd)
427{
428 pmd_write_lock_in_core(pmd);
429 if (unlikely(!pmd->in_service))
430 pmd->in_service = true;
431}
432
433static inline void pmd_write_unlock(struct dm_pool_metadata *pmd)
434 __releases(pmd->root_lock)
435{
436 up_write(&pmd->root_lock);
437}
438
439/*----------------------------------------------------------------*/
440
441static int superblock_lock_zero(struct dm_pool_metadata *pmd,
442 struct dm_block **sblock)
443{
444 return dm_bm_write_lock_zero(pmd->bm, THIN_SUPERBLOCK_LOCATION,
445 &sb_validator, sblock);
446}
447
448static int superblock_lock(struct dm_pool_metadata *pmd,
449 struct dm_block **sblock)
450{
451 return dm_bm_write_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION,
452 &sb_validator, sblock);
453}
454
455static int __superblock_all_zeroes(struct dm_block_manager *bm, int *result)
456{
457 int r;
458 unsigned int i;
459 struct dm_block *b;
460 __le64 *data_le, zero = cpu_to_le64(0);
461 unsigned int block_size = dm_bm_block_size(bm) / sizeof(__le64);
462
463 /*
464 * We can't use a validator here - it may be all zeroes.
465 */
466 r = dm_bm_read_lock(bm, THIN_SUPERBLOCK_LOCATION, NULL, &b);
467 if (r)
468 return r;
469
470 data_le = dm_block_data(b);
471 *result = 1;
472 for (i = 0; i < block_size; i++) {
473 if (data_le[i] != zero) {
474 *result = 0;
475 break;
476 }
477 }
478
479 dm_bm_unlock(b);
480
481 return 0;
482}
483
484static void __setup_btree_details(struct dm_pool_metadata *pmd)
485{
486 pmd->info.tm = pmd->tm;
487 pmd->info.levels = 2;
488 pmd->info.value_type.context = pmd->data_sm;
489 pmd->info.value_type.size = sizeof(__le64);
490 pmd->info.value_type.inc = data_block_inc;
491 pmd->info.value_type.dec = data_block_dec;
492 pmd->info.value_type.equal = data_block_equal;
493
494 memcpy(&pmd->nb_info, &pmd->info, sizeof(pmd->nb_info));
495 pmd->nb_info.tm = pmd->nb_tm;
496
497 pmd->tl_info.tm = pmd->tm;
498 pmd->tl_info.levels = 1;
499 pmd->tl_info.value_type.context = &pmd->bl_info;
500 pmd->tl_info.value_type.size = sizeof(__le64);
501 pmd->tl_info.value_type.inc = subtree_inc;
502 pmd->tl_info.value_type.dec = subtree_dec;
503 pmd->tl_info.value_type.equal = subtree_equal;
504
505 pmd->bl_info.tm = pmd->tm;
506 pmd->bl_info.levels = 1;
507 pmd->bl_info.value_type.context = pmd->data_sm;
508 pmd->bl_info.value_type.size = sizeof(__le64);
509 pmd->bl_info.value_type.inc = data_block_inc;
510 pmd->bl_info.value_type.dec = data_block_dec;
511 pmd->bl_info.value_type.equal = data_block_equal;
512
513 pmd->details_info.tm = pmd->tm;
514 pmd->details_info.levels = 1;
515 pmd->details_info.value_type.context = NULL;
516 pmd->details_info.value_type.size = sizeof(struct disk_device_details);
517 pmd->details_info.value_type.inc = NULL;
518 pmd->details_info.value_type.dec = NULL;
519 pmd->details_info.value_type.equal = NULL;
520}
521
522static int save_sm_roots(struct dm_pool_metadata *pmd)
523{
524 int r;
525 size_t len;
526
527 r = dm_sm_root_size(pmd->metadata_sm, &len);
528 if (r < 0)
529 return r;
530
531 r = dm_sm_copy_root(pmd->metadata_sm, &pmd->metadata_space_map_root, len);
532 if (r < 0)
533 return r;
534
535 r = dm_sm_root_size(pmd->data_sm, &len);
536 if (r < 0)
537 return r;
538
539 return dm_sm_copy_root(pmd->data_sm, &pmd->data_space_map_root, len);
540}
541
542static void copy_sm_roots(struct dm_pool_metadata *pmd,
543 struct thin_disk_superblock *disk)
544{
545 memcpy(&disk->metadata_space_map_root,
546 &pmd->metadata_space_map_root,
547 sizeof(pmd->metadata_space_map_root));
548
549 memcpy(&disk->data_space_map_root,
550 &pmd->data_space_map_root,
551 sizeof(pmd->data_space_map_root));
552}
553
554static int __write_initial_superblock(struct dm_pool_metadata *pmd)
555{
556 int r;
557 struct dm_block *sblock;
558 struct thin_disk_superblock *disk_super;
559 sector_t bdev_size = bdev_nr_sectors(pmd->bdev);
560
561 if (bdev_size > THIN_METADATA_MAX_SECTORS)
562 bdev_size = THIN_METADATA_MAX_SECTORS;
563
564 r = dm_sm_commit(pmd->data_sm);
565 if (r < 0)
566 return r;
567
568 r = dm_tm_pre_commit(pmd->tm);
569 if (r < 0)
570 return r;
571
572 r = save_sm_roots(pmd);
573 if (r < 0)
574 return r;
575
576 r = superblock_lock_zero(pmd, &sblock);
577 if (r)
578 return r;
579
580 disk_super = dm_block_data(sblock);
581 disk_super->flags = 0;
582 memset(disk_super->uuid, 0, sizeof(disk_super->uuid));
583 disk_super->magic = cpu_to_le64(THIN_SUPERBLOCK_MAGIC);
584 disk_super->version = cpu_to_le32(THIN_VERSION);
585 disk_super->time = 0;
586 disk_super->trans_id = 0;
587 disk_super->held_root = 0;
588
589 copy_sm_roots(pmd, disk_super);
590
591 disk_super->data_mapping_root = cpu_to_le64(pmd->root);
592 disk_super->device_details_root = cpu_to_le64(pmd->details_root);
593 disk_super->metadata_block_size = cpu_to_le32(THIN_METADATA_BLOCK_SIZE);
594 disk_super->metadata_nr_blocks = cpu_to_le64(bdev_size >> SECTOR_TO_BLOCK_SHIFT);
595 disk_super->data_block_size = cpu_to_le32(pmd->data_block_size);
596
597 return dm_tm_commit(pmd->tm, sblock);
598}
599
600static int __format_metadata(struct dm_pool_metadata *pmd)
601{
602 int r;
603
604 r = dm_tm_create_with_sm(pmd->bm, THIN_SUPERBLOCK_LOCATION,
605 &pmd->tm, &pmd->metadata_sm);
606 if (r < 0) {
607 pmd->tm = NULL;
608 pmd->metadata_sm = NULL;
609 DMERR("tm_create_with_sm failed");
610 return r;
611 }
612
613 pmd->data_sm = dm_sm_disk_create(pmd->tm, 0);
614 if (IS_ERR(pmd->data_sm)) {
615 DMERR("sm_disk_create failed");
616 r = PTR_ERR(pmd->data_sm);
617 pmd->data_sm = NULL;
618 goto bad_cleanup_tm;
619 }
620
621 pmd->nb_tm = dm_tm_create_non_blocking_clone(pmd->tm);
622 if (!pmd->nb_tm) {
623 DMERR("could not create non-blocking clone tm");
624 r = -ENOMEM;
625 goto bad_cleanup_data_sm;
626 }
627
628 __setup_btree_details(pmd);
629
630 r = dm_btree_empty(&pmd->info, &pmd->root);
631 if (r < 0)
632 goto bad_cleanup_nb_tm;
633
634 r = dm_btree_empty(&pmd->details_info, &pmd->details_root);
635 if (r < 0) {
636 DMERR("couldn't create devices root");
637 goto bad_cleanup_nb_tm;
638 }
639
640 r = __write_initial_superblock(pmd);
641 if (r)
642 goto bad_cleanup_nb_tm;
643
644 return 0;
645
646bad_cleanup_nb_tm:
647 dm_tm_destroy(pmd->nb_tm);
648 pmd->nb_tm = NULL;
649bad_cleanup_data_sm:
650 dm_sm_destroy(pmd->data_sm);
651 pmd->data_sm = NULL;
652bad_cleanup_tm:
653 dm_tm_destroy(pmd->tm);
654 pmd->tm = NULL;
655 dm_sm_destroy(pmd->metadata_sm);
656 pmd->metadata_sm = NULL;
657
658 return r;
659}
660
661static int __check_incompat_features(struct thin_disk_superblock *disk_super,
662 struct dm_pool_metadata *pmd)
663{
664 uint32_t features;
665
666 features = le32_to_cpu(disk_super->incompat_flags) & ~THIN_FEATURE_INCOMPAT_SUPP;
667 if (features) {
668 DMERR("could not access metadata due to unsupported optional features (%lx).",
669 (unsigned long)features);
670 return -EINVAL;
671 }
672
673 /*
674 * Check for read-only metadata to skip the following RDWR checks.
675 */
676 if (bdev_read_only(pmd->bdev))
677 return 0;
678
679 features = le32_to_cpu(disk_super->compat_ro_flags) & ~THIN_FEATURE_COMPAT_RO_SUPP;
680 if (features) {
681 DMERR("could not access metadata RDWR due to unsupported optional features (%lx).",
682 (unsigned long)features);
683 return -EINVAL;
684 }
685
686 return 0;
687}
688
689static int __open_metadata(struct dm_pool_metadata *pmd)
690{
691 int r;
692 struct dm_block *sblock;
693 struct thin_disk_superblock *disk_super;
694
695 r = dm_bm_read_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION,
696 &sb_validator, &sblock);
697 if (r < 0) {
698 DMERR("couldn't read superblock");
699 return r;
700 }
701
702 disk_super = dm_block_data(sblock);
703
704 /* Verify the data block size hasn't changed */
705 if (le32_to_cpu(disk_super->data_block_size) != pmd->data_block_size) {
706 DMERR("changing the data block size (from %u to %llu) is not supported",
707 le32_to_cpu(disk_super->data_block_size),
708 (unsigned long long)pmd->data_block_size);
709 r = -EINVAL;
710 goto bad_unlock_sblock;
711 }
712
713 r = __check_incompat_features(disk_super, pmd);
714 if (r < 0)
715 goto bad_unlock_sblock;
716
717 r = dm_tm_open_with_sm(pmd->bm, THIN_SUPERBLOCK_LOCATION,
718 disk_super->metadata_space_map_root,
719 sizeof(disk_super->metadata_space_map_root),
720 &pmd->tm, &pmd->metadata_sm);
721 if (r < 0) {
722 pmd->tm = NULL;
723 pmd->metadata_sm = NULL;
724 DMERR("tm_open_with_sm failed");
725 goto bad_unlock_sblock;
726 }
727
728 pmd->data_sm = dm_sm_disk_open(pmd->tm, disk_super->data_space_map_root,
729 sizeof(disk_super->data_space_map_root));
730 if (IS_ERR(pmd->data_sm)) {
731 DMERR("sm_disk_open failed");
732 r = PTR_ERR(pmd->data_sm);
733 pmd->data_sm = NULL;
734 goto bad_cleanup_tm;
735 }
736
737 pmd->nb_tm = dm_tm_create_non_blocking_clone(pmd->tm);
738 if (!pmd->nb_tm) {
739 DMERR("could not create non-blocking clone tm");
740 r = -ENOMEM;
741 goto bad_cleanup_data_sm;
742 }
743
744 /*
745 * For pool metadata opening process, root setting is redundant
746 * because it will be set again in __begin_transaction(). But dm
747 * pool aborting process really needs to get last transaction's
748 * root to avoid accessing broken btree.
749 */
750 pmd->root = le64_to_cpu(disk_super->data_mapping_root);
751 pmd->details_root = le64_to_cpu(disk_super->device_details_root);
752 pmd->held_root = le64_to_cpu(disk_super->held_root);
753
754 __setup_btree_details(pmd);
755 dm_bm_unlock(sblock);
756
757 return 0;
758
759bad_cleanup_data_sm:
760 dm_sm_destroy(pmd->data_sm);
761 pmd->data_sm = NULL;
762bad_cleanup_tm:
763 dm_tm_destroy(pmd->tm);
764 pmd->tm = NULL;
765 dm_sm_destroy(pmd->metadata_sm);
766 pmd->metadata_sm = NULL;
767bad_unlock_sblock:
768 dm_bm_unlock(sblock);
769
770 return r;
771}
772
773static int __open_or_format_metadata(struct dm_pool_metadata *pmd, bool format_device)
774{
775 int r, unformatted;
776
777 r = __superblock_all_zeroes(pmd->bm, &unformatted);
778 if (r)
779 return r;
780
781 if (unformatted)
782 return format_device ? __format_metadata(pmd) : -EPERM;
783
784 return __open_metadata(pmd);
785}
786
787static int __create_persistent_data_objects(struct dm_pool_metadata *pmd, bool format_device)
788{
789 int r;
790
791 pmd->bm = dm_block_manager_create(pmd->bdev, THIN_METADATA_BLOCK_SIZE << SECTOR_SHIFT,
792 THIN_MAX_CONCURRENT_LOCKS);
793 if (IS_ERR(pmd->bm)) {
794 DMERR("could not create block manager");
795 r = PTR_ERR(pmd->bm);
796 pmd->bm = NULL;
797 return r;
798 }
799
800 r = __open_or_format_metadata(pmd, format_device);
801 if (r) {
802 dm_block_manager_destroy(pmd->bm);
803 pmd->bm = NULL;
804 }
805
806 return r;
807}
808
809static void __destroy_persistent_data_objects(struct dm_pool_metadata *pmd,
810 bool destroy_bm)
811{
812 dm_sm_destroy(pmd->data_sm);
813 pmd->data_sm = NULL;
814 dm_sm_destroy(pmd->metadata_sm);
815 pmd->metadata_sm = NULL;
816 dm_tm_destroy(pmd->nb_tm);
817 pmd->nb_tm = NULL;
818 dm_tm_destroy(pmd->tm);
819 pmd->tm = NULL;
820 if (destroy_bm)
821 dm_block_manager_destroy(pmd->bm);
822}
823
824static int __begin_transaction(struct dm_pool_metadata *pmd)
825{
826 int r;
827 struct thin_disk_superblock *disk_super;
828 struct dm_block *sblock;
829
830 /*
831 * We re-read the superblock every time. Shouldn't need to do this
832 * really.
833 */
834 r = dm_bm_read_lock(pmd->bm, THIN_SUPERBLOCK_LOCATION,
835 &sb_validator, &sblock);
836 if (r)
837 return r;
838
839 disk_super = dm_block_data(sblock);
840 pmd->time = le32_to_cpu(disk_super->time);
841 pmd->root = le64_to_cpu(disk_super->data_mapping_root);
842 pmd->details_root = le64_to_cpu(disk_super->device_details_root);
843 pmd->held_root = le64_to_cpu(disk_super->held_root);
844 pmd->trans_id = le64_to_cpu(disk_super->trans_id);
845 pmd->flags = le32_to_cpu(disk_super->flags);
846 pmd->data_block_size = le32_to_cpu(disk_super->data_block_size);
847
848 dm_bm_unlock(sblock);
849 return 0;
850}
851
852static int __write_changed_details(struct dm_pool_metadata *pmd)
853{
854 int r;
855 struct dm_thin_device *td, *tmp;
856 struct disk_device_details details;
857 uint64_t key;
858
859 list_for_each_entry_safe(td, tmp, &pmd->thin_devices, list) {
860 if (!td->changed)
861 continue;
862
863 key = td->id;
864
865 details.mapped_blocks = cpu_to_le64(td->mapped_blocks);
866 details.transaction_id = cpu_to_le64(td->transaction_id);
867 details.creation_time = cpu_to_le32(td->creation_time);
868 details.snapshotted_time = cpu_to_le32(td->snapshotted_time);
869 __dm_bless_for_disk(&details);
870
871 r = dm_btree_insert(&pmd->details_info, pmd->details_root,
872 &key, &details, &pmd->details_root);
873 if (r)
874 return r;
875
876 if (td->open_count)
877 td->changed = false;
878 else {
879 list_del(&td->list);
880 kfree(td);
881 }
882 }
883
884 return 0;
885}
886
887static int __commit_transaction(struct dm_pool_metadata *pmd)
888{
889 int r;
890 struct thin_disk_superblock *disk_super;
891 struct dm_block *sblock;
892
893 /*
894 * We need to know if the thin_disk_superblock exceeds a 512-byte sector.
895 */
896 BUILD_BUG_ON(sizeof(struct thin_disk_superblock) > 512);
897 BUG_ON(!rwsem_is_locked(&pmd->root_lock));
898
899 if (unlikely(!pmd->in_service))
900 return 0;
901
902 if (pmd->pre_commit_fn) {
903 r = pmd->pre_commit_fn(pmd->pre_commit_context);
904 if (r < 0) {
905 DMERR("pre-commit callback failed");
906 return r;
907 }
908 }
909
910 r = __write_changed_details(pmd);
911 if (r < 0)
912 return r;
913
914 r = dm_sm_commit(pmd->data_sm);
915 if (r < 0)
916 return r;
917
918 r = dm_tm_pre_commit(pmd->tm);
919 if (r < 0)
920 return r;
921
922 r = save_sm_roots(pmd);
923 if (r < 0)
924 return r;
925
926 r = superblock_lock(pmd, &sblock);
927 if (r)
928 return r;
929
930 disk_super = dm_block_data(sblock);
931 disk_super->time = cpu_to_le32(pmd->time);
932 disk_super->data_mapping_root = cpu_to_le64(pmd->root);
933 disk_super->device_details_root = cpu_to_le64(pmd->details_root);
934 disk_super->held_root = cpu_to_le64(pmd->held_root);
935 disk_super->trans_id = cpu_to_le64(pmd->trans_id);
936 disk_super->flags = cpu_to_le32(pmd->flags);
937
938 copy_sm_roots(pmd, disk_super);
939
940 return dm_tm_commit(pmd->tm, sblock);
941}
942
943static void __set_metadata_reserve(struct dm_pool_metadata *pmd)
944{
945 int r;
946 dm_block_t total;
947 dm_block_t max_blocks = 4096; /* 16M */
948
949 r = dm_sm_get_nr_blocks(pmd->metadata_sm, &total);
950 if (r) {
951 DMERR("could not get size of metadata device");
952 pmd->metadata_reserve = max_blocks;
953 } else
954 pmd->metadata_reserve = min(max_blocks, div_u64(total, 10));
955}
956
957struct dm_pool_metadata *dm_pool_metadata_open(struct block_device *bdev,
958 sector_t data_block_size,
959 bool format_device)
960{
961 int r;
962 struct dm_pool_metadata *pmd;
963
964 pmd = kmalloc_obj(*pmd);
965 if (!pmd) {
966 DMERR("could not allocate metadata struct");
967 return ERR_PTR(-ENOMEM);
968 }
969
970 init_rwsem(&pmd->root_lock);
971 pmd->time = 0;
972 INIT_LIST_HEAD(&pmd->thin_devices);
973 pmd->fail_io = false;
974 pmd->in_service = false;
975 pmd->bdev = bdev;
976 pmd->data_block_size = data_block_size;
977 pmd->pre_commit_fn = NULL;
978 pmd->pre_commit_context = NULL;
979
980 r = __create_persistent_data_objects(pmd, format_device);
981 if (r) {
982 kfree(pmd);
983 return ERR_PTR(r);
984 }
985
986 r = __begin_transaction(pmd);
987 if (r < 0) {
988 if (dm_pool_metadata_close(pmd) < 0)
989 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
990 return ERR_PTR(r);
991 }
992
993 __set_metadata_reserve(pmd);
994
995 return pmd;
996}
997
998int dm_pool_metadata_close(struct dm_pool_metadata *pmd)
999{
1000 int r;
1001 unsigned int open_devices = 0;
1002 struct dm_thin_device *td, *tmp;
1003
1004 down_read(&pmd->root_lock);
1005 list_for_each_entry_safe(td, tmp, &pmd->thin_devices, list) {
1006 if (td->open_count)
1007 open_devices++;
1008 else {
1009 list_del(&td->list);
1010 kfree(td);
1011 }
1012 }
1013 up_read(&pmd->root_lock);
1014
1015 if (open_devices) {
1016 DMERR("attempt to close pmd when %u device(s) are still open",
1017 open_devices);
1018 return -EBUSY;
1019 }
1020
1021 pmd_write_lock_in_core(pmd);
1022 if (!pmd->fail_io && !dm_bm_is_read_only(pmd->bm)) {
1023 r = __commit_transaction(pmd);
1024 if (r < 0)
1025 DMWARN("%s: __commit_transaction() failed, error = %d",
1026 __func__, r);
1027 }
1028 pmd_write_unlock(pmd);
1029 __destroy_persistent_data_objects(pmd, true);
1030
1031 kfree(pmd);
1032 return 0;
1033}
1034
1035/*
1036 * __open_device: Returns @td corresponding to device with id @dev,
1037 * creating it if @create is set and incrementing @td->open_count.
1038 * On failure, @td is undefined.
1039 */
1040static int __open_device(struct dm_pool_metadata *pmd,
1041 dm_thin_id dev, int create,
1042 struct dm_thin_device **td)
1043{
1044 int r, changed = 0;
1045 struct dm_thin_device *td2;
1046 uint64_t key = dev;
1047 struct disk_device_details details_le;
1048
1049 /*
1050 * If the device is already open, return it.
1051 */
1052 list_for_each_entry(td2, &pmd->thin_devices, list)
1053 if (td2->id == dev) {
1054 /*
1055 * May not create an already-open device.
1056 */
1057 if (create)
1058 return -EEXIST;
1059
1060 td2->open_count++;
1061 *td = td2;
1062 return 0;
1063 }
1064
1065 /*
1066 * Check the device exists.
1067 */
1068 r = dm_btree_lookup(&pmd->details_info, pmd->details_root,
1069 &key, &details_le);
1070 if (r) {
1071 if (r != -ENODATA || !create)
1072 return r;
1073
1074 /*
1075 * Create new device.
1076 */
1077 changed = 1;
1078 details_le.mapped_blocks = 0;
1079 details_le.transaction_id = cpu_to_le64(pmd->trans_id);
1080 details_le.creation_time = cpu_to_le32(pmd->time);
1081 details_le.snapshotted_time = cpu_to_le32(pmd->time);
1082 }
1083
1084 *td = kmalloc_obj(**td, GFP_NOIO);
1085 if (!*td)
1086 return -ENOMEM;
1087
1088 (*td)->pmd = pmd;
1089 (*td)->id = dev;
1090 (*td)->open_count = 1;
1091 (*td)->changed = changed;
1092 (*td)->aborted_with_changes = false;
1093 (*td)->mapped_blocks = le64_to_cpu(details_le.mapped_blocks);
1094 (*td)->transaction_id = le64_to_cpu(details_le.transaction_id);
1095 (*td)->creation_time = le32_to_cpu(details_le.creation_time);
1096 (*td)->snapshotted_time = le32_to_cpu(details_le.snapshotted_time);
1097
1098 list_add(&(*td)->list, &pmd->thin_devices);
1099
1100 return 0;
1101}
1102
1103static void __close_device(struct dm_thin_device *td)
1104{
1105 --td->open_count;
1106}
1107
1108static int __create_thin(struct dm_pool_metadata *pmd,
1109 dm_thin_id dev)
1110{
1111 int r;
1112 dm_block_t dev_root;
1113 uint64_t key = dev;
1114 struct dm_thin_device *td;
1115 __le64 value;
1116
1117 r = dm_btree_lookup(&pmd->details_info, pmd->details_root,
1118 &key, NULL);
1119 if (!r)
1120 return -EEXIST;
1121
1122 /*
1123 * Create an empty btree for the mappings.
1124 */
1125 r = dm_btree_empty(&pmd->bl_info, &dev_root);
1126 if (r)
1127 return r;
1128
1129 /*
1130 * Insert it into the main mapping tree.
1131 */
1132 value = cpu_to_le64(dev_root);
1133 __dm_bless_for_disk(&value);
1134 r = dm_btree_insert(&pmd->tl_info, pmd->root, &key, &value, &pmd->root);
1135 if (r) {
1136 dm_btree_del(&pmd->bl_info, dev_root);
1137 return r;
1138 }
1139
1140 r = __open_device(pmd, dev, 1, &td);
1141 if (r) {
1142 dm_btree_remove(&pmd->tl_info, pmd->root, &key, &pmd->root);
1143 dm_btree_del(&pmd->bl_info, dev_root);
1144 return r;
1145 }
1146 __close_device(td);
1147
1148 return r;
1149}
1150
1151int dm_pool_create_thin(struct dm_pool_metadata *pmd, dm_thin_id dev)
1152{
1153 int r = -EINVAL;
1154
1155 pmd_write_lock(pmd);
1156 if (!pmd->fail_io)
1157 r = __create_thin(pmd, dev);
1158 pmd_write_unlock(pmd);
1159
1160 return r;
1161}
1162
1163static int __set_snapshot_details(struct dm_pool_metadata *pmd,
1164 struct dm_thin_device *snap,
1165 dm_thin_id origin, uint32_t time)
1166{
1167 int r;
1168 struct dm_thin_device *td;
1169
1170 r = __open_device(pmd, origin, 0, &td);
1171 if (r)
1172 return r;
1173
1174 td->changed = true;
1175 td->snapshotted_time = time;
1176
1177 snap->mapped_blocks = td->mapped_blocks;
1178 snap->snapshotted_time = time;
1179 __close_device(td);
1180
1181 return 0;
1182}
1183
1184static int __create_snap(struct dm_pool_metadata *pmd,
1185 dm_thin_id dev, dm_thin_id origin)
1186{
1187 int r;
1188 dm_block_t origin_root;
1189 uint64_t key = origin, dev_key = dev;
1190 struct dm_thin_device *td;
1191 __le64 value;
1192
1193 /* check this device is unused */
1194 r = dm_btree_lookup(&pmd->details_info, pmd->details_root,
1195 &dev_key, NULL);
1196 if (!r)
1197 return -EEXIST;
1198
1199 /* find the mapping tree for the origin */
1200 r = dm_btree_lookup(&pmd->tl_info, pmd->root, &key, &value);
1201 if (r)
1202 return r;
1203 origin_root = le64_to_cpu(value);
1204
1205 /* clone the origin, an inc will do */
1206 dm_tm_inc(pmd->tm, origin_root);
1207
1208 /* insert into the main mapping tree */
1209 value = cpu_to_le64(origin_root);
1210 __dm_bless_for_disk(&value);
1211 key = dev;
1212 r = dm_btree_insert(&pmd->tl_info, pmd->root, &key, &value, &pmd->root);
1213 if (r) {
1214 dm_tm_dec(pmd->tm, origin_root);
1215 return r;
1216 }
1217
1218 pmd->time++;
1219
1220 r = __open_device(pmd, dev, 1, &td);
1221 if (r)
1222 goto bad;
1223
1224 r = __set_snapshot_details(pmd, td, origin, pmd->time);
1225 __close_device(td);
1226
1227 if (r)
1228 goto bad;
1229
1230 return 0;
1231
1232bad:
1233 dm_btree_remove(&pmd->tl_info, pmd->root, &key, &pmd->root);
1234 dm_btree_remove(&pmd->details_info, pmd->details_root,
1235 &key, &pmd->details_root);
1236 return r;
1237}
1238
1239int dm_pool_create_snap(struct dm_pool_metadata *pmd,
1240 dm_thin_id dev,
1241 dm_thin_id origin)
1242{
1243 int r = -EINVAL;
1244
1245 pmd_write_lock(pmd);
1246 if (!pmd->fail_io)
1247 r = __create_snap(pmd, dev, origin);
1248 pmd_write_unlock(pmd);
1249
1250 return r;
1251}
1252
1253static int __delete_device(struct dm_pool_metadata *pmd, dm_thin_id dev)
1254{
1255 int r;
1256 uint64_t key = dev;
1257 struct dm_thin_device *td;
1258
1259 /* TODO: failure should mark the transaction invalid */
1260 r = __open_device(pmd, dev, 0, &td);
1261 if (r)
1262 return r;
1263
1264 if (td->open_count > 1) {
1265 __close_device(td);
1266 return -EBUSY;
1267 }
1268
1269 list_del(&td->list);
1270 kfree(td);
1271 r = dm_btree_remove(&pmd->details_info, pmd->details_root,
1272 &key, &pmd->details_root);
1273 if (r)
1274 return r;
1275
1276 r = dm_btree_remove(&pmd->tl_info, pmd->root, &key, &pmd->root);
1277 if (r)
1278 return r;
1279
1280 return 0;
1281}
1282
1283int dm_pool_delete_thin_device(struct dm_pool_metadata *pmd,
1284 dm_thin_id dev)
1285{
1286 int r = -EINVAL;
1287
1288 pmd_write_lock(pmd);
1289 if (!pmd->fail_io)
1290 r = __delete_device(pmd, dev);
1291 pmd_write_unlock(pmd);
1292
1293 return r;
1294}
1295
1296int dm_pool_set_metadata_transaction_id(struct dm_pool_metadata *pmd,
1297 uint64_t current_id,
1298 uint64_t new_id)
1299{
1300 int r = -EINVAL;
1301
1302 pmd_write_lock(pmd);
1303
1304 if (pmd->fail_io)
1305 goto out;
1306
1307 if (pmd->trans_id != current_id) {
1308 DMERR("mismatched transaction id");
1309 goto out;
1310 }
1311
1312 pmd->trans_id = new_id;
1313 r = 0;
1314
1315out:
1316 pmd_write_unlock(pmd);
1317
1318 return r;
1319}
1320
1321int dm_pool_get_metadata_transaction_id(struct dm_pool_metadata *pmd,
1322 uint64_t *result)
1323{
1324 int r = -EINVAL;
1325
1326 down_read(&pmd->root_lock);
1327 if (!pmd->fail_io) {
1328 *result = pmd->trans_id;
1329 r = 0;
1330 }
1331 up_read(&pmd->root_lock);
1332
1333 return r;
1334}
1335
1336static int __reserve_metadata_snap(struct dm_pool_metadata *pmd)
1337{
1338 int r, inc;
1339 struct thin_disk_superblock *disk_super;
1340 struct dm_block *copy;
1341 dm_block_t held_root;
1342
1343 if (pmd->held_root) {
1344 DMWARN("Pool metadata snapshot already exists: release this before taking another.");
1345 return -EBUSY;
1346 }
1347
1348 /*
1349 * We commit to ensure the btree roots which we increment in a
1350 * moment are up to date.
1351 */
1352 r = __commit_transaction(pmd);
1353 if (r < 0) {
1354 DMWARN("%s: __commit_transaction() failed, error = %d",
1355 __func__, r);
1356 return r;
1357 }
1358
1359 /*
1360 * Copy the superblock.
1361 */
1362 dm_sm_inc_block(pmd->metadata_sm, THIN_SUPERBLOCK_LOCATION);
1363 r = dm_tm_shadow_block(pmd->tm, THIN_SUPERBLOCK_LOCATION,
1364 &sb_validator, &copy, &inc);
1365 if (r) {
1366 dm_sm_dec_block(pmd->metadata_sm, THIN_SUPERBLOCK_LOCATION);
1367 return r;
1368 }
1369
1370 BUG_ON(!inc);
1371
1372 held_root = dm_block_location(copy);
1373 disk_super = dm_block_data(copy);
1374
1375 /*
1376 * Wipe the spacemap since we're not publishing this.
1377 */
1378 memset(&disk_super->data_space_map_root, 0,
1379 sizeof(disk_super->data_space_map_root));
1380 memset(&disk_super->metadata_space_map_root, 0,
1381 sizeof(disk_super->metadata_space_map_root));
1382
1383 /*
1384 * Increment the data structures that need to be preserved.
1385 */
1386 dm_tm_inc(pmd->tm, le64_to_cpu(disk_super->data_mapping_root));
1387 dm_tm_inc(pmd->tm, le64_to_cpu(disk_super->device_details_root));
1388 dm_tm_unlock(pmd->tm, copy);
1389
1390 pmd->held_root = held_root;
1391
1392 return 0;
1393}
1394
1395int dm_pool_reserve_metadata_snap(struct dm_pool_metadata *pmd)
1396{
1397 int r = -EINVAL;
1398
1399 pmd_write_lock(pmd);
1400 if (!pmd->fail_io)
1401 r = __reserve_metadata_snap(pmd);
1402 pmd_write_unlock(pmd);
1403
1404 return r;
1405}
1406
1407static int __release_metadata_snap(struct dm_pool_metadata *pmd)
1408{
1409 int r;
1410 struct thin_disk_superblock *disk_super;
1411 struct dm_block *copy;
1412 dm_block_t held_root;
1413
1414 held_root = pmd->held_root;
1415
1416 if (!held_root) {
1417 DMWARN("No pool metadata snapshot found: nothing to release.");
1418 return -EINVAL;
1419 }
1420
1421 r = dm_tm_read_lock(pmd->tm, held_root, &sb_validator, &copy);
1422 if (r)
1423 return r;
1424
1425 pmd->held_root = 0;
1426
1427 disk_super = dm_block_data(copy);
1428 dm_btree_del(&pmd->info, le64_to_cpu(disk_super->data_mapping_root));
1429 dm_btree_del(&pmd->details_info, le64_to_cpu(disk_super->device_details_root));
1430 dm_tm_unlock(pmd->tm, copy);
1431
1432 dm_sm_dec_block(pmd->metadata_sm, held_root);
1433
1434 return 0;
1435}
1436
1437int dm_pool_release_metadata_snap(struct dm_pool_metadata *pmd)
1438{
1439 int r = -EINVAL;
1440
1441 pmd_write_lock(pmd);
1442 if (!pmd->fail_io)
1443 r = __release_metadata_snap(pmd);
1444 pmd_write_unlock(pmd);
1445
1446 return r;
1447}
1448
1449static int __get_metadata_snap(struct dm_pool_metadata *pmd,
1450 dm_block_t *result)
1451{
1452 *result = pmd->held_root;
1453
1454 return 0;
1455}
1456
1457int dm_pool_get_metadata_snap(struct dm_pool_metadata *pmd,
1458 dm_block_t *result)
1459{
1460 int r = -EINVAL;
1461
1462 down_read(&pmd->root_lock);
1463 if (!pmd->fail_io)
1464 r = __get_metadata_snap(pmd, result);
1465 up_read(&pmd->root_lock);
1466
1467 return r;
1468}
1469
1470int dm_pool_open_thin_device(struct dm_pool_metadata *pmd, dm_thin_id dev,
1471 struct dm_thin_device **td)
1472{
1473 int r = -EINVAL;
1474
1475 pmd_write_lock_in_core(pmd);
1476 if (!pmd->fail_io)
1477 r = __open_device(pmd, dev, 0, td);
1478 pmd_write_unlock(pmd);
1479
1480 return r;
1481}
1482
1483int dm_pool_close_thin_device(struct dm_thin_device *td)
1484{
1485 pmd_write_lock_in_core(td->pmd);
1486 __close_device(td);
1487 pmd_write_unlock(td->pmd);
1488
1489 return 0;
1490}
1491
1492dm_thin_id dm_thin_dev_id(struct dm_thin_device *td)
1493{
1494 return td->id;
1495}
1496
1497/*
1498 * Check whether @time (of block creation) is older than @td's last snapshot.
1499 * If so then the associated block is shared with the last snapshot device.
1500 * Any block on a device created *after* the device last got snapshotted is
1501 * necessarily not shared.
1502 */
1503static bool __snapshotted_since(struct dm_thin_device *td, uint32_t time)
1504{
1505 return td->snapshotted_time > time;
1506}
1507
1508static void unpack_lookup_result(struct dm_thin_device *td, __le64 value,
1509 struct dm_thin_lookup_result *result)
1510{
1511 uint64_t block_time = 0;
1512 dm_block_t exception_block;
1513 uint32_t exception_time;
1514
1515 block_time = le64_to_cpu(value);
1516 unpack_block_time(block_time, &exception_block, &exception_time);
1517 result->block = exception_block;
1518 result->shared = __snapshotted_since(td, exception_time);
1519}
1520
1521static int __find_block(struct dm_thin_device *td, dm_block_t block,
1522 int can_issue_io, struct dm_thin_lookup_result *result)
1523{
1524 int r;
1525 __le64 value;
1526 struct dm_pool_metadata *pmd = td->pmd;
1527 dm_block_t keys[2] = { td->id, block };
1528 struct dm_btree_info *info;
1529
1530 if (can_issue_io)
1531 info = &pmd->info;
1532 else
1533 info = &pmd->nb_info;
1534
1535 r = dm_btree_lookup(info, pmd->root, keys, &value);
1536 if (!r)
1537 unpack_lookup_result(td, value, result);
1538
1539 return r;
1540}
1541
1542int dm_thin_find_block(struct dm_thin_device *td, dm_block_t block,
1543 int can_issue_io, struct dm_thin_lookup_result *result)
1544{
1545 int r;
1546 struct dm_pool_metadata *pmd = td->pmd;
1547
1548 down_read(&pmd->root_lock);
1549 if (pmd->fail_io) {
1550 up_read(&pmd->root_lock);
1551 return -EINVAL;
1552 }
1553
1554 r = __find_block(td, block, can_issue_io, result);
1555
1556 up_read(&pmd->root_lock);
1557 return r;
1558}
1559
1560static int __find_next_mapped_block(struct dm_thin_device *td, dm_block_t block,
1561 dm_block_t *vblock,
1562 struct dm_thin_lookup_result *result)
1563{
1564 int r;
1565 __le64 value;
1566 struct dm_pool_metadata *pmd = td->pmd;
1567 dm_block_t keys[2] = { td->id, block };
1568
1569 r = dm_btree_lookup_next(&pmd->info, pmd->root, keys, vblock, &value);
1570 if (!r)
1571 unpack_lookup_result(td, value, result);
1572
1573 return r;
1574}
1575
1576static int __find_mapped_range(struct dm_thin_device *td,
1577 dm_block_t begin, dm_block_t end,
1578 dm_block_t *thin_begin, dm_block_t *thin_end,
1579 dm_block_t *pool_begin, bool *maybe_shared)
1580{
1581 int r;
1582 dm_block_t pool_end;
1583 struct dm_thin_lookup_result lookup;
1584
1585 if (end < begin)
1586 return -ENODATA;
1587
1588 r = __find_next_mapped_block(td, begin, &begin, &lookup);
1589 if (r)
1590 return r;
1591
1592 if (begin >= end)
1593 return -ENODATA;
1594
1595 *thin_begin = begin;
1596 *pool_begin = lookup.block;
1597 *maybe_shared = lookup.shared;
1598
1599 begin++;
1600 pool_end = *pool_begin + 1;
1601 while (begin != end) {
1602 r = __find_block(td, begin, true, &lookup);
1603 if (r) {
1604 if (r == -ENODATA)
1605 break;
1606
1607 return r;
1608 }
1609
1610 if ((lookup.block != pool_end) ||
1611 (lookup.shared != *maybe_shared))
1612 break;
1613
1614 pool_end++;
1615 begin++;
1616 }
1617
1618 *thin_end = begin;
1619 return 0;
1620}
1621
1622int dm_thin_find_mapped_range(struct dm_thin_device *td,
1623 dm_block_t begin, dm_block_t end,
1624 dm_block_t *thin_begin, dm_block_t *thin_end,
1625 dm_block_t *pool_begin, bool *maybe_shared)
1626{
1627 int r = -EINVAL;
1628 struct dm_pool_metadata *pmd = td->pmd;
1629
1630 down_read(&pmd->root_lock);
1631 if (!pmd->fail_io) {
1632 r = __find_mapped_range(td, begin, end, thin_begin, thin_end,
1633 pool_begin, maybe_shared);
1634 }
1635 up_read(&pmd->root_lock);
1636
1637 return r;
1638}
1639
1640static int __insert(struct dm_thin_device *td, dm_block_t block,
1641 dm_block_t data_block)
1642{
1643 int r, inserted;
1644 __le64 value;
1645 struct dm_pool_metadata *pmd = td->pmd;
1646 dm_block_t keys[2] = { td->id, block };
1647
1648 value = cpu_to_le64(pack_block_time(data_block, pmd->time));
1649 __dm_bless_for_disk(&value);
1650
1651 r = dm_btree_insert_notify(&pmd->info, pmd->root, keys, &value,
1652 &pmd->root, &inserted);
1653 if (r)
1654 return r;
1655
1656 td->changed = true;
1657 if (inserted)
1658 td->mapped_blocks++;
1659
1660 return 0;
1661}
1662
1663int dm_thin_insert_block(struct dm_thin_device *td, dm_block_t block,
1664 dm_block_t data_block)
1665{
1666 int r = -EINVAL;
1667
1668 pmd_write_lock(td->pmd);
1669 if (!td->pmd->fail_io)
1670 r = __insert(td, block, data_block);
1671 pmd_write_unlock(td->pmd);
1672
1673 return r;
1674}
1675
1676static int __remove_range(struct dm_thin_device *td, dm_block_t begin, dm_block_t end)
1677{
1678 int r;
1679 unsigned int count, total_count = 0;
1680 struct dm_pool_metadata *pmd = td->pmd;
1681 dm_block_t keys[1] = { td->id };
1682 __le64 value;
1683 dm_block_t mapping_root;
1684
1685 /*
1686 * Find the mapping tree
1687 */
1688 r = dm_btree_lookup(&pmd->tl_info, pmd->root, keys, &value);
1689 if (r)
1690 return r;
1691
1692 /*
1693 * Remove from the mapping tree, taking care to inc the
1694 * ref count so it doesn't get deleted.
1695 */
1696 mapping_root = le64_to_cpu(value);
1697 dm_tm_inc(pmd->tm, mapping_root);
1698 r = dm_btree_remove(&pmd->tl_info, pmd->root, keys, &pmd->root);
1699 if (r)
1700 return r;
1701
1702 /*
1703 * Remove leaves stops at the first unmapped entry, so we have to
1704 * loop round finding mapped ranges.
1705 */
1706 while (begin < end) {
1707 r = dm_btree_lookup_next(&pmd->bl_info, mapping_root, &begin, &begin, &value);
1708 if (r == -ENODATA)
1709 break;
1710
1711 if (r)
1712 return r;
1713
1714 if (begin >= end)
1715 break;
1716
1717 r = dm_btree_remove_leaves(&pmd->bl_info, mapping_root, &begin, end, &mapping_root, &count);
1718 if (r)
1719 return r;
1720
1721 total_count += count;
1722 }
1723
1724 td->mapped_blocks -= total_count;
1725 td->changed = true;
1726
1727 /*
1728 * Reinsert the mapping tree.
1729 */
1730 value = cpu_to_le64(mapping_root);
1731 __dm_bless_for_disk(&value);
1732 return dm_btree_insert(&pmd->tl_info, pmd->root, keys, &value, &pmd->root);
1733}
1734
1735int dm_thin_remove_range(struct dm_thin_device *td,
1736 dm_block_t begin, dm_block_t end)
1737{
1738 int r = -EINVAL;
1739
1740 pmd_write_lock(td->pmd);
1741 if (!td->pmd->fail_io)
1742 r = __remove_range(td, begin, end);
1743 pmd_write_unlock(td->pmd);
1744
1745 return r;
1746}
1747
1748int dm_pool_block_is_shared(struct dm_pool_metadata *pmd, dm_block_t b, bool *result)
1749{
1750 int r = -EINVAL;
1751 uint32_t ref_count;
1752
1753 down_read(&pmd->root_lock);
1754 if (!pmd->fail_io) {
1755 r = dm_sm_get_count(pmd->data_sm, b, &ref_count);
1756 if (!r)
1757 *result = (ref_count > 1);
1758 }
1759 up_read(&pmd->root_lock);
1760
1761 return r;
1762}
1763
1764int dm_pool_inc_data_range(struct dm_pool_metadata *pmd, dm_block_t b, dm_block_t e)
1765{
1766 int r = -EINVAL;
1767
1768 pmd_write_lock(pmd);
1769 if (!pmd->fail_io)
1770 r = dm_sm_inc_blocks(pmd->data_sm, b, e);
1771 pmd_write_unlock(pmd);
1772
1773 return r;
1774}
1775
1776int dm_pool_dec_data_range(struct dm_pool_metadata *pmd, dm_block_t b, dm_block_t e)
1777{
1778 int r = -EINVAL;
1779
1780 pmd_write_lock(pmd);
1781 if (!pmd->fail_io)
1782 r = dm_sm_dec_blocks(pmd->data_sm, b, e);
1783 pmd_write_unlock(pmd);
1784
1785 return r;
1786}
1787
1788bool dm_thin_changed_this_transaction(struct dm_thin_device *td)
1789{
1790 int r;
1791
1792 down_read(&td->pmd->root_lock);
1793 r = td->changed;
1794 up_read(&td->pmd->root_lock);
1795
1796 return r;
1797}
1798
1799bool dm_pool_changed_this_transaction(struct dm_pool_metadata *pmd)
1800{
1801 bool r = false;
1802 struct dm_thin_device *td, *tmp;
1803
1804 down_read(&pmd->root_lock);
1805 list_for_each_entry_safe(td, tmp, &pmd->thin_devices, list) {
1806 if (td->changed) {
1807 r = td->changed;
1808 break;
1809 }
1810 }
1811 up_read(&pmd->root_lock);
1812
1813 return r;
1814}
1815
1816bool dm_thin_aborted_changes(struct dm_thin_device *td)
1817{
1818 bool r;
1819
1820 down_read(&td->pmd->root_lock);
1821 r = td->aborted_with_changes;
1822 up_read(&td->pmd->root_lock);
1823
1824 return r;
1825}
1826
1827int dm_pool_alloc_data_block(struct dm_pool_metadata *pmd, dm_block_t *result)
1828{
1829 int r = -EINVAL;
1830
1831 pmd_write_lock(pmd);
1832 if (!pmd->fail_io)
1833 r = dm_sm_new_block(pmd->data_sm, result);
1834 pmd_write_unlock(pmd);
1835
1836 return r;
1837}
1838
1839int dm_pool_commit_metadata(struct dm_pool_metadata *pmd)
1840{
1841 int r = -EINVAL;
1842
1843 /*
1844 * Care is taken to not have commit be what
1845 * triggers putting the thin-pool in-service.
1846 */
1847 pmd_write_lock_in_core(pmd);
1848 if (pmd->fail_io)
1849 goto out;
1850
1851 r = __commit_transaction(pmd);
1852 if (r < 0)
1853 goto out;
1854
1855 /*
1856 * Open the next transaction.
1857 */
1858 r = __begin_transaction(pmd);
1859out:
1860 pmd_write_unlock(pmd);
1861 return r;
1862}
1863
1864static void __set_abort_with_changes_flags(struct dm_pool_metadata *pmd)
1865{
1866 struct dm_thin_device *td;
1867
1868 list_for_each_entry(td, &pmd->thin_devices, list)
1869 td->aborted_with_changes = td->changed;
1870}
1871
1872int dm_pool_abort_metadata(struct dm_pool_metadata *pmd)
1873{
1874 int r = -EINVAL;
1875
1876 /* fail_io is double-checked with pmd->root_lock held below */
1877 if (unlikely(pmd->fail_io))
1878 return r;
1879
1880 pmd_write_lock(pmd);
1881 if (pmd->fail_io) {
1882 pmd_write_unlock(pmd);
1883 return r;
1884 }
1885 __set_abort_with_changes_flags(pmd);
1886
1887 /* destroy data_sm/metadata_sm/nb_tm/tm */
1888 __destroy_persistent_data_objects(pmd, false);
1889
1890 /* reset bm */
1891 dm_block_manager_reset(pmd->bm);
1892
1893 /* rebuild data_sm/metadata_sm/nb_tm/tm */
1894 r = __open_or_format_metadata(pmd, false);
1895 if (r)
1896 pmd->fail_io = true;
1897 pmd_write_unlock(pmd);
1898 return r;
1899}
1900
1901int dm_pool_get_free_block_count(struct dm_pool_metadata *pmd, dm_block_t *result)
1902{
1903 int r = -EINVAL;
1904
1905 down_read(&pmd->root_lock);
1906 if (!pmd->fail_io)
1907 r = dm_sm_get_nr_free(pmd->data_sm, result);
1908 up_read(&pmd->root_lock);
1909
1910 return r;
1911}
1912
1913int dm_pool_get_free_metadata_block_count(struct dm_pool_metadata *pmd,
1914 dm_block_t *result)
1915{
1916 int r = -EINVAL;
1917
1918 down_read(&pmd->root_lock);
1919 if (!pmd->fail_io)
1920 r = dm_sm_get_nr_free(pmd->metadata_sm, result);
1921
1922 if (!r) {
1923 if (*result < pmd->metadata_reserve)
1924 *result = 0;
1925 else
1926 *result -= pmd->metadata_reserve;
1927 }
1928 up_read(&pmd->root_lock);
1929
1930 return r;
1931}
1932
1933int dm_pool_get_metadata_dev_size(struct dm_pool_metadata *pmd,
1934 dm_block_t *result)
1935{
1936 int r = -EINVAL;
1937
1938 down_read(&pmd->root_lock);
1939 if (!pmd->fail_io)
1940 r = dm_sm_get_nr_blocks(pmd->metadata_sm, result);
1941 up_read(&pmd->root_lock);
1942
1943 return r;
1944}
1945
1946int dm_pool_get_data_dev_size(struct dm_pool_metadata *pmd, dm_block_t *result)
1947{
1948 int r = -EINVAL;
1949
1950 down_read(&pmd->root_lock);
1951 if (!pmd->fail_io)
1952 r = dm_sm_get_nr_blocks(pmd->data_sm, result);
1953 up_read(&pmd->root_lock);
1954
1955 return r;
1956}
1957
1958int dm_thin_get_mapped_count(struct dm_thin_device *td, dm_block_t *result)
1959{
1960 int r = -EINVAL;
1961 struct dm_pool_metadata *pmd = td->pmd;
1962
1963 down_read(&pmd->root_lock);
1964 if (!pmd->fail_io) {
1965 *result = td->mapped_blocks;
1966 r = 0;
1967 }
1968 up_read(&pmd->root_lock);
1969
1970 return r;
1971}
1972
1973static int __highest_block(struct dm_thin_device *td, dm_block_t *result)
1974{
1975 int r;
1976 __le64 value_le;
1977 dm_block_t thin_root;
1978 struct dm_pool_metadata *pmd = td->pmd;
1979
1980 r = dm_btree_lookup(&pmd->tl_info, pmd->root, &td->id, &value_le);
1981 if (r)
1982 return r;
1983
1984 thin_root = le64_to_cpu(value_le);
1985
1986 return dm_btree_find_highest_key(&pmd->bl_info, thin_root, result);
1987}
1988
1989int dm_thin_get_highest_mapped_block(struct dm_thin_device *td,
1990 dm_block_t *result)
1991{
1992 int r = -EINVAL;
1993 struct dm_pool_metadata *pmd = td->pmd;
1994
1995 down_read(&pmd->root_lock);
1996 if (!pmd->fail_io)
1997 r = __highest_block(td, result);
1998 up_read(&pmd->root_lock);
1999
2000 return r;
2001}
2002
2003static int __resize_space_map(struct dm_space_map *sm, dm_block_t new_count)
2004{
2005 int r;
2006 dm_block_t old_count;
2007
2008 r = dm_sm_get_nr_blocks(sm, &old_count);
2009 if (r)
2010 return r;
2011
2012 if (new_count == old_count)
2013 return 0;
2014
2015 if (new_count < old_count) {
2016 DMERR("cannot reduce size of space map");
2017 return -EINVAL;
2018 }
2019
2020 return dm_sm_extend(sm, new_count - old_count);
2021}
2022
2023int dm_pool_resize_data_dev(struct dm_pool_metadata *pmd, dm_block_t new_count)
2024{
2025 int r = -EINVAL;
2026
2027 pmd_write_lock(pmd);
2028 if (!pmd->fail_io)
2029 r = __resize_space_map(pmd->data_sm, new_count);
2030 pmd_write_unlock(pmd);
2031
2032 return r;
2033}
2034
2035int dm_pool_resize_metadata_dev(struct dm_pool_metadata *pmd, dm_block_t new_count)
2036{
2037 int r = -EINVAL;
2038
2039 pmd_write_lock(pmd);
2040 if (!pmd->fail_io) {
2041 r = __resize_space_map(pmd->metadata_sm, new_count);
2042 if (!r)
2043 __set_metadata_reserve(pmd);
2044 }
2045 pmd_write_unlock(pmd);
2046
2047 return r;
2048}
2049
2050void dm_pool_metadata_read_only(struct dm_pool_metadata *pmd)
2051{
2052 pmd_write_lock_in_core(pmd);
2053 dm_bm_set_read_only(pmd->bm);
2054 pmd_write_unlock(pmd);
2055}
2056
2057void dm_pool_metadata_read_write(struct dm_pool_metadata *pmd)
2058{
2059 pmd_write_lock_in_core(pmd);
2060 dm_bm_set_read_write(pmd->bm);
2061 pmd_write_unlock(pmd);
2062}
2063
2064int dm_pool_register_metadata_threshold(struct dm_pool_metadata *pmd,
2065 dm_block_t threshold,
2066 dm_sm_threshold_fn fn,
2067 void *context)
2068{
2069 int r = -EINVAL;
2070
2071 pmd_write_lock_in_core(pmd);
2072 if (!pmd->fail_io) {
2073 r = dm_sm_register_threshold_callback(pmd->metadata_sm,
2074 threshold, fn, context);
2075 }
2076 pmd_write_unlock(pmd);
2077
2078 return r;
2079}
2080
2081void dm_pool_register_pre_commit_callback(struct dm_pool_metadata *pmd,
2082 dm_pool_pre_commit_fn fn,
2083 void *context)
2084{
2085 pmd_write_lock_in_core(pmd);
2086 pmd->pre_commit_fn = fn;
2087 pmd->pre_commit_context = context;
2088 pmd_write_unlock(pmd);
2089}
2090
2091int dm_pool_metadata_set_needs_check(struct dm_pool_metadata *pmd)
2092{
2093 int r = -EINVAL;
2094 struct dm_block *sblock;
2095 struct thin_disk_superblock *disk_super;
2096
2097 pmd_write_lock(pmd);
2098 if (pmd->fail_io)
2099 goto out;
2100
2101 pmd->flags |= THIN_METADATA_NEEDS_CHECK_FLAG;
2102
2103 r = superblock_lock(pmd, &sblock);
2104 if (r) {
2105 DMERR("couldn't lock superblock");
2106 goto out;
2107 }
2108
2109 disk_super = dm_block_data(sblock);
2110 disk_super->flags = cpu_to_le32(pmd->flags);
2111
2112 dm_bm_unlock(sblock);
2113out:
2114 pmd_write_unlock(pmd);
2115 return r;
2116}
2117
2118bool dm_pool_metadata_needs_check(struct dm_pool_metadata *pmd)
2119{
2120 bool needs_check;
2121
2122 down_read(&pmd->root_lock);
2123 needs_check = pmd->flags & THIN_METADATA_NEEDS_CHECK_FLAG;
2124 up_read(&pmd->root_lock);
2125
2126 return needs_check;
2127}
2128
2129void dm_pool_issue_prefetches(struct dm_pool_metadata *pmd)
2130{
2131 down_read(&pmd->root_lock);
2132 if (!pmd->fail_io)
2133 dm_tm_issue_prefetches(pmd->tm);
2134 up_read(&pmd->root_lock);
2135}