projects/linux·blob master · refs

2231 lines · 54.1 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2001 Sistina Software (UK) Limited.
4 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
5 *
6 * This file is released under the GPL.
7 */
8
9#include "dm-core.h"
10#include "dm-rq.h"
11
12#include <linux/module.h>
13#include <linux/vmalloc.h>
14#include <linux/blkdev.h>
15#include <linux/blk-integrity.h>
16#include <linux/namei.h>
17#include <linux/ctype.h>
18#include <linux/string.h>
19#include <linux/slab.h>
20#include <linux/interrupt.h>
21#include <linux/mutex.h>
22#include <linux/delay.h>
23#include <linux/atomic.h>
24#include <linux/blk-mq.h>
25#include <linux/mount.h>
26#include <linux/dax.h>
27
28#define DM_MSG_PREFIX "table"
29
30#define NODE_SIZE L1_CACHE_BYTES
31#define KEYS_PER_NODE (NODE_SIZE / sizeof(sector_t))
32#define CHILDREN_PER_NODE (KEYS_PER_NODE + 1)
33
34/*
35 * Similar to ceiling(log_size(n))
36 */
37static unsigned int int_log(unsigned int n, unsigned int base)
38{
39 int result = 0;
40
41 while (n > 1) {
42 n = dm_div_up(n, base);
43 result++;
44 }
45
46 return result;
47}
48
49/*
50 * Calculate the index of the child node of the n'th node k'th key.
51 */
52static inline unsigned int get_child(unsigned int n, unsigned int k)
53{
54 return (n * CHILDREN_PER_NODE) + k;
55}
56
57/*
58 * Return the n'th node of level l from table t.
59 */
60static inline sector_t *get_node(struct dm_table *t,
61 unsigned int l, unsigned int n)
62{
63 return t->index[l] + (n * KEYS_PER_NODE);
64}
65
66/*
67 * Return the highest key that you could lookup from the n'th
68 * node on level l of the btree.
69 */
70static sector_t high(struct dm_table *t, unsigned int l, unsigned int n)
71{
72 for (; l < t->depth - 1; l++)
73 n = get_child(n, CHILDREN_PER_NODE - 1);
74
75 if (n >= t->counts[l])
76 return (sector_t) -1;
77
78 return get_node(t, l, n)[KEYS_PER_NODE - 1];
79}
80
81/*
82 * Fills in a level of the btree based on the highs of the level
83 * below it.
84 */
85static int setup_btree_index(unsigned int l, struct dm_table *t)
86{
87 unsigned int n, k;
88 sector_t *node;
89
90 for (n = 0U; n < t->counts[l]; n++) {
91 node = get_node(t, l, n);
92
93 for (k = 0U; k < KEYS_PER_NODE; k++)
94 node[k] = high(t, l + 1, get_child(n, k));
95 }
96
97 return 0;
98}
99
100/*
101 * highs, and targets are managed as dynamic arrays during a
102 * table load.
103 */
104static int alloc_targets(struct dm_table *t, unsigned int num)
105{
106 sector_t *n_highs;
107 struct dm_target *n_targets;
108
109 /*
110 * Allocate both the target array and offset array at once.
111 */
112 n_highs = kvcalloc(num, sizeof(struct dm_target) + sizeof(sector_t),
113 GFP_KERNEL);
114 if (!n_highs)
115 return -ENOMEM;
116
117 n_targets = (struct dm_target *) (n_highs + num);
118
119 memset(n_highs, -1, sizeof(*n_highs) * num);
120
121 t->num_allocated = num;
122 t->highs = n_highs;
123 t->targets = n_targets;
124
125 return 0;
126}
127
128int dm_table_create(struct dm_table **result, blk_mode_t mode,
129 unsigned int num_targets, struct mapped_device *md)
130{
131 struct dm_table *t;
132
133 if (num_targets > DM_MAX_TARGETS)
134 return -EOVERFLOW;
135
136 t = kzalloc_obj(*t);
137
138 if (!t)
139 return -ENOMEM;
140
141 INIT_LIST_HEAD(&t->devices);
142
143 if (!num_targets)
144 num_targets = KEYS_PER_NODE;
145
146 num_targets = dm_round_up(num_targets, KEYS_PER_NODE);
147
148 if (!num_targets) {
149 kfree(t);
150 return -EOVERFLOW;
151 }
152
153 if (alloc_targets(t, num_targets)) {
154 kfree(t);
155 return -ENOMEM;
156 }
157
158 t->type = DM_TYPE_NONE;
159 t->mode = mode;
160 t->md = md;
161 t->flush_bypasses_map = true;
162 *result = t;
163 return 0;
164}
165
166static void free_devices(struct list_head *devices, struct mapped_device *md)
167{
168 struct list_head *tmp, *next;
169
170 list_for_each_safe(tmp, next, devices) {
171 struct dm_dev_internal *dd =
172 list_entry(tmp, struct dm_dev_internal, list);
173 DMWARN("%s: dm_table_destroy: dm_put_device call missing for %s",
174 dm_device_name(md), dd->dm_dev->name);
175 dm_put_table_device(md, dd->dm_dev);
176 kfree(dd);
177 }
178}
179
180static void dm_table_destroy_crypto_profile(struct dm_table *t);
181
182void dm_table_destroy(struct dm_table *t)
183{
184 if (!t)
185 return;
186
187 /* free the indexes */
188 if (t->depth >= 2)
189 kvfree(t->index[t->depth - 2]);
190
191 /* free the targets */
192 for (unsigned int i = 0; i < t->num_targets; i++) {
193 struct dm_target *ti = dm_table_get_target(t, i);
194
195 if (ti->type->dtr)
196 ti->type->dtr(ti);
197
198 dm_put_target_type(ti->type);
199 }
200
201 kvfree(t->highs);
202
203 /* free the device list */
204 free_devices(&t->devices, t->md);
205
206 dm_free_md_mempools(t->mempools);
207
208 dm_table_destroy_crypto_profile(t);
209
210 kfree(t);
211}
212
213/*
214 * See if we've already got a device in the list.
215 */
216static struct dm_dev_internal *find_device(struct list_head *l, dev_t dev)
217{
218 struct dm_dev_internal *dd;
219
220 list_for_each_entry(dd, l, list)
221 if (dd->dm_dev->bdev->bd_dev == dev)
222 return dd;
223
224 return NULL;
225}
226
227/*
228 * If possible, this checks an area of a destination device is invalid.
229 */
230static int device_area_is_invalid(struct dm_target *ti, struct dm_dev *dev,
231 sector_t start, sector_t len, void *data)
232{
233 struct queue_limits *limits = data;
234 struct block_device *bdev = dev->bdev;
235 sector_t dev_size = bdev_nr_sectors(bdev);
236 unsigned short logical_block_size_sectors =
237 limits->logical_block_size >> SECTOR_SHIFT;
238
239 if (!dev_size)
240 return 0;
241
242 if ((start >= dev_size) || (start + len > dev_size)) {
243 DMERR("%s: %pg too small for target: start=%llu, len=%llu, dev_size=%llu",
244 dm_device_name(ti->table->md), bdev,
245 (unsigned long long)start,
246 (unsigned long long)len,
247 (unsigned long long)dev_size);
248 return 1;
249 }
250
251 /*
252 * If the target is mapped to zoned block device(s), check
253 * that the zones are not partially mapped.
254 */
255 if (bdev_is_zoned(bdev)) {
256 unsigned int zone_sectors = bdev_zone_sectors(bdev);
257
258 if (!bdev_is_zone_aligned(bdev, start)) {
259 DMERR("%s: start=%llu not aligned to h/w zone size %u of %pg",
260 dm_device_name(ti->table->md),
261 (unsigned long long)start,
262 zone_sectors, bdev);
263 return 1;
264 }
265
266 /*
267 * Note: The last zone of a zoned block device may be smaller
268 * than other zones. So for a target mapping the end of a
269 * zoned block device with such a zone, len would not be zone
270 * aligned. We do not allow such last smaller zone to be part
271 * of the mapping here to ensure that mappings with multiple
272 * devices do not end up with a smaller zone in the middle of
273 * the sector range.
274 */
275 if (!bdev_is_zone_aligned(bdev, len)) {
276 DMERR("%s: len=%llu not aligned to h/w zone size %u of %pg",
277 dm_device_name(ti->table->md),
278 (unsigned long long)len,
279 zone_sectors, bdev);
280 return 1;
281 }
282 }
283
284 if (logical_block_size_sectors <= 1)
285 return 0;
286
287 if (start & (logical_block_size_sectors - 1)) {
288 DMERR("%s: start=%llu not aligned to h/w logical block size %u of %pg",
289 dm_device_name(ti->table->md),
290 (unsigned long long)start,
291 limits->logical_block_size, bdev);
292 return 1;
293 }
294
295 if (len & (logical_block_size_sectors - 1)) {
296 DMERR("%s: len=%llu not aligned to h/w logical block size %u of %pg",
297 dm_device_name(ti->table->md),
298 (unsigned long long)len,
299 limits->logical_block_size, bdev);
300 return 1;
301 }
302
303 return 0;
304}
305
306/*
307 * This upgrades the mode on an already open dm_dev, being
308 * careful to leave things as they were if we fail to reopen the
309 * device and not to touch the existing bdev field in case
310 * it is accessed concurrently.
311 */
312static int upgrade_mode(struct dm_dev_internal *dd, blk_mode_t new_mode,
313 struct mapped_device *md)
314{
315 int r;
316 struct dm_dev *old_dev, *new_dev;
317
318 old_dev = dd->dm_dev;
319
320 r = dm_get_table_device(md, dd->dm_dev->bdev->bd_dev,
321 dd->dm_dev->mode | new_mode, &new_dev);
322 if (r)
323 return r;
324
325 dd->dm_dev = new_dev;
326 dm_put_table_device(md, old_dev);
327
328 return 0;
329}
330
331/*
332 * Note: the __ref annotation is because this function can call the __init
333 * marked early_lookup_bdev when called during early boot code from dm-init.c.
334 */
335int __ref dm_devt_from_path(const char *path, dev_t *dev_p)
336{
337 int r;
338 dev_t dev;
339 unsigned int major, minor;
340 char dummy;
341
342 if (sscanf(path, "%u:%u%c", &major, &minor, &dummy) == 2) {
343 /* Extract the major/minor numbers */
344 dev = MKDEV(major, minor);
345 if (MAJOR(dev) != major || MINOR(dev) != minor)
346 return -EOVERFLOW;
347 } else {
348 r = lookup_bdev(path, &dev);
349#ifndef MODULE
350 if (r && system_state < SYSTEM_RUNNING)
351 r = early_lookup_bdev(path, &dev);
352#endif
353 if (r)
354 return r;
355 }
356 *dev_p = dev;
357 return 0;
358}
359EXPORT_SYMBOL(dm_devt_from_path);
360
361/*
362 * Add a device to the list, or just increment the usage count if
363 * it's already present.
364 */
365int dm_get_device(struct dm_target *ti, const char *path, blk_mode_t mode,
366 struct dm_dev **result)
367{
368 int r;
369 dev_t dev;
370 struct dm_dev_internal *dd;
371 struct dm_table *t = ti->table;
372
373 BUG_ON(!t);
374
375 r = dm_devt_from_path(path, &dev);
376 if (r)
377 return r;
378
379 if (dev == disk_devt(t->md->disk))
380 return -EINVAL;
381
382 dd = find_device(&t->devices, dev);
383 if (!dd) {
384 dd = kmalloc_obj(*dd);
385 if (!dd)
386 return -ENOMEM;
387
388 r = dm_get_table_device(t->md, dev, mode, &dd->dm_dev);
389 if (r) {
390 kfree(dd);
391 return r;
392 }
393
394 refcount_set(&dd->count, 1);
395 list_add(&dd->list, &t->devices);
396 goto out;
397
398 } else if (dd->dm_dev->mode != (mode | dd->dm_dev->mode)) {
399 r = upgrade_mode(dd, mode, t->md);
400 if (r)
401 return r;
402 }
403 refcount_inc(&dd->count);
404out:
405 *result = dd->dm_dev;
406 return 0;
407}
408EXPORT_SYMBOL(dm_get_device);
409
410static int dm_set_device_limits(struct dm_target *ti, struct dm_dev *dev,
411 sector_t start, sector_t len, void *data)
412{
413 struct queue_limits *limits = data;
414 struct block_device *bdev = dev->bdev;
415 struct request_queue *q = bdev_get_queue(bdev);
416
417 if (unlikely(!q)) {
418 DMWARN("%s: Cannot set limits for nonexistent device %pg",
419 dm_device_name(ti->table->md), bdev);
420 return 0;
421 }
422
423 mutex_lock(&q->limits_lock);
424 /*
425 * BLK_FEAT_ATOMIC_WRITES is not inherited from the bottom device in
426 * blk_stack_limits(), so do it manually.
427 */
428 limits->features |= (q->limits.features & BLK_FEAT_ATOMIC_WRITES);
429
430 if (blk_stack_limits(limits, &q->limits,
431 get_start_sect(bdev) + start) < 0)
432 DMWARN("%s: adding target device %pg caused an alignment inconsistency: "
433 "physical_block_size=%u, logical_block_size=%u, "
434 "alignment_offset=%u, start=%llu",
435 dm_device_name(ti->table->md), bdev,
436 q->limits.physical_block_size,
437 q->limits.logical_block_size,
438 q->limits.alignment_offset,
439 (unsigned long long) start << SECTOR_SHIFT);
440
441 /*
442 * Only stack the integrity profile if the target doesn't have native
443 * integrity support.
444 */
445 if (!dm_target_has_integrity(ti->type))
446 queue_limits_stack_integrity_bdev(limits, bdev);
447 mutex_unlock(&q->limits_lock);
448 return 0;
449}
450
451/*
452 * Decrement a device's use count and remove it if necessary.
453 */
454void dm_put_device(struct dm_target *ti, struct dm_dev *d)
455{
456 int found = 0;
457 struct list_head *devices = &ti->table->devices;
458 struct dm_dev_internal *dd;
459
460 list_for_each_entry(dd, devices, list) {
461 if (dd->dm_dev == d) {
462 found = 1;
463 break;
464 }
465 }
466 if (!found) {
467 DMERR("%s: device %s not in table devices list",
468 dm_device_name(ti->table->md), d->name);
469 return;
470 }
471 if (refcount_dec_and_test(&dd->count)) {
472 dm_put_table_device(ti->table->md, d);
473 list_del(&dd->list);
474 kfree(dd);
475 }
476}
477EXPORT_SYMBOL(dm_put_device);
478
479/*
480 * Checks to see if the target joins onto the end of the table.
481 */
482static int adjoin(struct dm_table *t, struct dm_target *ti)
483{
484 struct dm_target *prev;
485
486 if (!t->num_targets)
487 return !ti->begin;
488
489 prev = &t->targets[t->num_targets - 1];
490 return (ti->begin == (prev->begin + prev->len));
491}
492
493/*
494 * Used to dynamically allocate the arg array.
495 *
496 * We do first allocation with GFP_NOIO because dm-mpath and dm-thin must
497 * process messages even if some device is suspended. These messages have a
498 * small fixed number of arguments.
499 *
500 * On the other hand, dm-switch needs to process bulk data using messages and
501 * excessive use of GFP_NOIO could cause trouble.
502 */
503static char **realloc_argv(unsigned int *size, char **old_argv)
504{
505 char **argv;
506 unsigned int new_size;
507 gfp_t gfp;
508
509 if (*size) {
510 new_size = *size * 2;
511 gfp = GFP_KERNEL;
512 } else {
513 new_size = 8;
514 gfp = GFP_NOIO;
515 }
516 argv = kmalloc_array(new_size, sizeof(*argv), gfp);
517 if (argv) {
518 if (old_argv)
519 memcpy(argv, old_argv, *size * sizeof(*argv));
520 *size = new_size;
521 }
522
523 kfree(old_argv);
524 return argv;
525}
526
527/*
528 * Destructively splits up the argument list to pass to ctr.
529 */
530int dm_split_args(int *argc, char ***argvp, char *input)
531{
532 char *start, *end = input, *out, **argv = NULL;
533 unsigned int array_size = 0;
534
535 *argc = 0;
536
537 if (!input) {
538 *argvp = NULL;
539 return 0;
540 }
541
542 argv = realloc_argv(&array_size, argv);
543 if (!argv)
544 return -ENOMEM;
545
546 while (1) {
547 /* Skip whitespace */
548 start = skip_spaces(end);
549
550 if (!*start)
551 break; /* success, we hit the end */
552
553 /* 'out' is used to remove any back-quotes */
554 end = out = start;
555 while (*end) {
556 /* Everything apart from '\0' can be quoted */
557 if (*end == '\\' && *(end + 1)) {
558 *out++ = *(end + 1);
559 end += 2;
560 continue;
561 }
562
563 if (isspace(*end))
564 break; /* end of token */
565
566 *out++ = *end++;
567 }
568
569 /* have we already filled the array ? */
570 if ((*argc + 1) > array_size) {
571 argv = realloc_argv(&array_size, argv);
572 if (!argv)
573 return -ENOMEM;
574 }
575
576 /* we know this is whitespace */
577 if (*end)
578 end++;
579
580 /* terminate the string and put it in the array */
581 *out = '\0';
582 argv[*argc] = start;
583 (*argc)++;
584 }
585
586 *argvp = argv;
587 return 0;
588}
589
590static void dm_set_stacking_limits(struct queue_limits *limits)
591{
592 blk_set_stacking_limits(limits);
593 limits->features |= BLK_FEAT_IO_STAT | BLK_FEAT_NOWAIT | BLK_FEAT_POLL;
594}
595
596/*
597 * Impose necessary and sufficient conditions on a devices's table such
598 * that any incoming bio which respects its logical_block_size can be
599 * processed successfully. If it falls across the boundary between
600 * two or more targets, the size of each piece it gets split into must
601 * be compatible with the logical_block_size of the target processing it.
602 */
603static int validate_hardware_logical_block_alignment(struct dm_table *t,
604 struct queue_limits *limits)
605{
606 /*
607 * This function uses arithmetic modulo the logical_block_size
608 * (in units of 512-byte sectors).
609 */
610 unsigned short device_logical_block_size_sects =
611 limits->logical_block_size >> SECTOR_SHIFT;
612
613 /*
614 * Offset of the start of the next table entry, mod logical_block_size.
615 */
616 unsigned short next_target_start = 0;
617
618 /*
619 * Given an aligned bio that extends beyond the end of a
620 * target, how many sectors must the next target handle?
621 */
622 unsigned short remaining = 0;
623
624 struct dm_target *ti;
625 struct queue_limits ti_limits;
626 unsigned int i;
627
628 /*
629 * Check each entry in the table in turn.
630 */
631 for (i = 0; i < t->num_targets; i++) {
632 ti = dm_table_get_target(t, i);
633
634 dm_set_stacking_limits(&ti_limits);
635
636 /* combine all target devices' limits */
637 if (ti->type->iterate_devices)
638 ti->type->iterate_devices(ti, dm_set_device_limits,
639 &ti_limits);
640
641 /*
642 * If the remaining sectors fall entirely within this
643 * table entry are they compatible with its logical_block_size?
644 */
645 if (remaining < ti->len &&
646 remaining & ((ti_limits.logical_block_size >>
647 SECTOR_SHIFT) - 1))
648 break; /* Error */
649
650 next_target_start =
651 (unsigned short) ((next_target_start + ti->len) &
652 (device_logical_block_size_sects - 1));
653 remaining = next_target_start ?
654 device_logical_block_size_sects - next_target_start : 0;
655 }
656
657 if (remaining) {
658 DMERR("%s: table line %u (start sect %llu len %llu) "
659 "not aligned to h/w logical block size %u",
660 dm_device_name(t->md), i,
661 (unsigned long long) ti->begin,
662 (unsigned long long) ti->len,
663 limits->logical_block_size);
664 return -EINVAL;
665 }
666
667 return 0;
668}
669
670int dm_table_add_target(struct dm_table *t, const char *type,
671 sector_t start, sector_t len, char *params)
672{
673 int r = -EINVAL, argc;
674 char **argv;
675 struct dm_target *ti;
676
677 if (t->singleton) {
678 DMERR("%s: target type %s must appear alone in table",
679 dm_device_name(t->md), t->targets->type->name);
680 return -EINVAL;
681 }
682
683 BUG_ON(t->num_targets >= t->num_allocated);
684
685 ti = t->targets + t->num_targets;
686 memset(ti, 0, sizeof(*ti));
687
688 if (!len) {
689 DMERR("%s: zero-length target", dm_device_name(t->md));
690 return -EINVAL;
691 }
692 if (start + len < start || start + len > LLONG_MAX >> SECTOR_SHIFT) {
693 DMERR("%s: too large device", dm_device_name(t->md));
694 return -EINVAL;
695 }
696
697 ti->type = dm_get_target_type(type);
698 if (!ti->type) {
699 DMERR("%s: %s: unknown target type", dm_device_name(t->md), type);
700 return -EINVAL;
701 }
702
703 if (dm_target_needs_singleton(ti->type)) {
704 if (t->num_targets) {
705 ti->error = "singleton target type must appear alone in table";
706 goto bad;
707 }
708 t->singleton = true;
709 }
710
711 if (dm_target_always_writeable(ti->type) &&
712 !(t->mode & BLK_OPEN_WRITE)) {
713 ti->error = "target type may not be included in a read-only table";
714 goto bad;
715 }
716
717 if (t->immutable_target_type) {
718 if (t->immutable_target_type != ti->type) {
719 ti->error = "immutable target type cannot be mixed with other target types";
720 goto bad;
721 }
722 } else if (dm_target_is_immutable(ti->type)) {
723 if (t->num_targets) {
724 ti->error = "immutable target type cannot be mixed with other target types";
725 goto bad;
726 }
727 t->immutable_target_type = ti->type;
728 }
729
730 ti->table = t;
731 ti->begin = start;
732 ti->len = len;
733 ti->error = "Unknown error";
734
735 /*
736 * Does this target adjoin the previous one ?
737 */
738 if (!adjoin(t, ti)) {
739 ti->error = "Gap in table";
740 goto bad;
741 }
742
743 r = dm_split_args(&argc, &argv, params);
744 if (r) {
745 ti->error = "couldn't split parameters";
746 goto bad;
747 }
748
749 r = ti->type->ctr(ti, argc, argv);
750 kfree(argv);
751 if (r)
752 goto bad;
753
754 t->highs[t->num_targets++] = ti->begin + ti->len - 1;
755
756 if (!ti->num_discard_bios && ti->discards_supported)
757 DMWARN("%s: %s: ignoring discards_supported because num_discard_bios is zero.",
758 dm_device_name(t->md), type);
759
760 if (ti->limit_swap_bios && !static_key_enabled(&swap_bios_enabled.key))
761 static_branch_enable(&swap_bios_enabled);
762
763 if (!ti->flush_bypasses_map)
764 t->flush_bypasses_map = false;
765
766 return 0;
767
768 bad:
769 DMERR("%s: %s: %s (%pe)", dm_device_name(t->md), type, ti->error, ERR_PTR(r));
770 dm_put_target_type(ti->type);
771 return r;
772}
773
774/*
775 * Target argument parsing helpers.
776 */
777static int validate_next_arg(const struct dm_arg *arg, struct dm_arg_set *arg_set,
778 unsigned int *value, char **error, unsigned int grouped)
779{
780 const char *arg_str = dm_shift_arg(arg_set);
781 char dummy;
782
783 if (!arg_str ||
784 (sscanf(arg_str, "%u%c", value, &dummy) != 1) ||
785 (*value < arg->min) ||
786 (*value > arg->max) ||
787 (grouped && arg_set->argc < *value)) {
788 *error = arg->error;
789 return -EINVAL;
790 }
791
792 return 0;
793}
794
795int dm_read_arg(const struct dm_arg *arg, struct dm_arg_set *arg_set,
796 unsigned int *value, char **error)
797{
798 return validate_next_arg(arg, arg_set, value, error, 0);
799}
800EXPORT_SYMBOL(dm_read_arg);
801
802int dm_read_arg_group(const struct dm_arg *arg, struct dm_arg_set *arg_set,
803 unsigned int *value, char **error)
804{
805 return validate_next_arg(arg, arg_set, value, error, 1);
806}
807EXPORT_SYMBOL(dm_read_arg_group);
808
809const char *dm_shift_arg(struct dm_arg_set *as)
810{
811 char *r;
812
813 if (as->argc) {
814 as->argc--;
815 r = *as->argv;
816 as->argv++;
817 return r;
818 }
819
820 return NULL;
821}
822EXPORT_SYMBOL(dm_shift_arg);
823
824void dm_consume_args(struct dm_arg_set *as, unsigned int num_args)
825{
826 BUG_ON(as->argc < num_args);
827 as->argc -= num_args;
828 as->argv += num_args;
829}
830EXPORT_SYMBOL(dm_consume_args);
831
832static bool __table_type_bio_based(enum dm_queue_mode table_type)
833{
834 return (table_type == DM_TYPE_BIO_BASED ||
835 table_type == DM_TYPE_DAX_BIO_BASED);
836}
837
838static bool __table_type_request_based(enum dm_queue_mode table_type)
839{
840 return table_type == DM_TYPE_REQUEST_BASED;
841}
842
843void dm_table_set_type(struct dm_table *t, enum dm_queue_mode type)
844{
845 t->type = type;
846}
847EXPORT_SYMBOL_GPL(dm_table_set_type);
848
849/* validate the dax capability of the target device span */
850static int device_not_dax_capable(struct dm_target *ti, struct dm_dev *dev,
851 sector_t start, sector_t len, void *data)
852{
853 if (dev->dax_dev)
854 return false;
855
856 DMDEBUG("%pg: error: dax unsupported by block device", dev->bdev);
857 return true;
858}
859
860/* Check devices support synchronous DAX */
861static int device_not_dax_synchronous_capable(struct dm_target *ti, struct dm_dev *dev,
862 sector_t start, sector_t len, void *data)
863{
864 return !dev->dax_dev || !dax_synchronous(dev->dax_dev);
865}
866
867static bool dm_table_supports_dax(struct dm_table *t,
868 iterate_devices_callout_fn iterate_fn)
869{
870 /* Ensure that all targets support DAX. */
871 for (unsigned int i = 0; i < t->num_targets; i++) {
872 struct dm_target *ti = dm_table_get_target(t, i);
873
874 if (!ti->type->direct_access)
875 return false;
876
877 if (dm_target_is_wildcard(ti->type) ||
878 !ti->type->iterate_devices ||
879 ti->type->iterate_devices(ti, iterate_fn, NULL))
880 return false;
881 }
882
883 return true;
884}
885
886static int device_is_not_rq_stackable(struct dm_target *ti, struct dm_dev *dev,
887 sector_t start, sector_t len, void *data)
888{
889 struct block_device *bdev = dev->bdev;
890 struct request_queue *q = bdev_get_queue(bdev);
891
892 /* request-based cannot stack on partitions! */
893 if (bdev_is_partition(bdev))
894 return true;
895
896 return !queue_is_mq(q);
897}
898
899static int dm_table_determine_type(struct dm_table *t)
900{
901 unsigned int bio_based = 0, request_based = 0, hybrid = 0;
902 struct dm_target *ti;
903 struct list_head *devices = dm_table_get_devices(t);
904 enum dm_queue_mode live_md_type = dm_get_md_type(t->md);
905
906 if (t->type != DM_TYPE_NONE) {
907 /* target already set the table's type */
908 if (t->type == DM_TYPE_BIO_BASED) {
909 /* possibly upgrade to a variant of bio-based */
910 goto verify_bio_based;
911 }
912 BUG_ON(t->type == DM_TYPE_DAX_BIO_BASED);
913 goto verify_rq_based;
914 }
915
916 for (unsigned int i = 0; i < t->num_targets; i++) {
917 ti = dm_table_get_target(t, i);
918 if (dm_target_hybrid(ti))
919 hybrid = 1;
920 else if (dm_target_request_based(ti))
921 request_based = 1;
922 else
923 bio_based = 1;
924
925 if (bio_based && request_based) {
926 DMERR("Inconsistent table: different target types can't be mixed up");
927 return -EINVAL;
928 }
929 }
930
931 if (hybrid && !bio_based && !request_based) {
932 /*
933 * The targets can work either way.
934 * Determine the type from the live device.
935 * Default to bio-based if device is new.
936 */
937 if (__table_type_request_based(live_md_type))
938 request_based = 1;
939 else
940 bio_based = 1;
941 }
942
943 if (bio_based) {
944verify_bio_based:
945 /* We must use this table as bio-based */
946 t->type = DM_TYPE_BIO_BASED;
947 if (dm_table_supports_dax(t, device_not_dax_capable) ||
948 (list_empty(devices) && live_md_type == DM_TYPE_DAX_BIO_BASED)) {
949 t->type = DM_TYPE_DAX_BIO_BASED;
950 }
951 return 0;
952 }
953
954 BUG_ON(!request_based); /* No targets in this table */
955
956 t->type = DM_TYPE_REQUEST_BASED;
957
958verify_rq_based:
959 /*
960 * Request-based dm supports only tables that have a single target now.
961 * To support multiple targets, request splitting support is needed,
962 * and that needs lots of changes in the block-layer.
963 * (e.g. request completion process for partial completion.)
964 */
965 if (t->num_targets > 1) {
966 DMERR("request-based DM doesn't support multiple targets");
967 return -EINVAL;
968 }
969
970 if (list_empty(devices)) {
971 int srcu_idx;
972 struct dm_table *live_table = dm_get_live_table(t->md, &srcu_idx);
973
974 /* inherit live table's type */
975 if (live_table)
976 t->type = live_table->type;
977 dm_put_live_table(t->md, srcu_idx);
978 return 0;
979 }
980
981 ti = dm_table_get_immutable_target(t);
982 if (!ti) {
983 DMERR("table load rejected: immutable target is required");
984 return -EINVAL;
985 } else if (ti->max_io_len) {
986 DMERR("table load rejected: immutable target that splits IO is not supported");
987 return -EINVAL;
988 }
989
990 /* Non-request-stackable devices can't be used for request-based dm */
991 if (!ti->type->iterate_devices ||
992 ti->type->iterate_devices(ti, device_is_not_rq_stackable, NULL)) {
993 DMERR("table load rejected: including non-request-stackable devices");
994 return -EINVAL;
995 }
996
997 return 0;
998}
999
1000enum dm_queue_mode dm_table_get_type(struct dm_table *t)
1001{
1002 return t->type;
1003}
1004
1005struct target_type *dm_table_get_immutable_target_type(struct dm_table *t)
1006{
1007 return t->immutable_target_type;
1008}
1009
1010struct dm_target *dm_table_get_immutable_target(struct dm_table *t)
1011{
1012 /* Immutable target is implicitly a singleton */
1013 if (t->num_targets > 1 ||
1014 !dm_target_is_immutable(t->targets[0].type))
1015 return NULL;
1016
1017 return t->targets;
1018}
1019
1020struct dm_target *dm_table_get_wildcard_target(struct dm_table *t)
1021{
1022 for (unsigned int i = 0; i < t->num_targets; i++) {
1023 struct dm_target *ti = dm_table_get_target(t, i);
1024
1025 if (dm_target_is_wildcard(ti->type))
1026 return ti;
1027 }
1028
1029 return NULL;
1030}
1031
1032bool dm_table_request_based(struct dm_table *t)
1033{
1034 return __table_type_request_based(dm_table_get_type(t));
1035}
1036
1037static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *md)
1038{
1039 enum dm_queue_mode type = dm_table_get_type(t);
1040 unsigned int per_io_data_size = 0, front_pad, io_front_pad;
1041 unsigned int min_pool_size = 0, pool_size;
1042 struct dm_md_mempools *pools;
1043 unsigned int bioset_flags = 0;
1044
1045 if (unlikely(type == DM_TYPE_NONE)) {
1046 DMERR("no table type is set, can't allocate mempools");
1047 return -EINVAL;
1048 }
1049
1050 pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
1051 if (!pools)
1052 return -ENOMEM;
1053
1054 if (type == DM_TYPE_REQUEST_BASED) {
1055 pool_size = dm_get_reserved_rq_based_ios();
1056 front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
1057 goto init_bs;
1058 }
1059
1060 if (md->queue->limits.features & BLK_FEAT_POLL)
1061 bioset_flags |= BIOSET_PERCPU_CACHE;
1062
1063 for (unsigned int i = 0; i < t->num_targets; i++) {
1064 struct dm_target *ti = dm_table_get_target(t, i);
1065
1066 per_io_data_size = max(per_io_data_size, ti->per_io_data_size);
1067 min_pool_size = max(min_pool_size, ti->num_flush_bios);
1068 }
1069 pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size);
1070 front_pad = roundup(per_io_data_size,
1071 __alignof__(struct dm_target_io)) + DM_TARGET_IO_BIO_OFFSET;
1072
1073 io_front_pad = roundup(per_io_data_size,
1074 __alignof__(struct dm_io)) + DM_IO_BIO_OFFSET;
1075 if (bioset_init(&pools->io_bs, pool_size, io_front_pad, bioset_flags))
1076 goto out_free_pools;
1077init_bs:
1078 if (bioset_init(&pools->bs, pool_size, front_pad, 0))
1079 goto out_free_pools;
1080
1081 t->mempools = pools;
1082 return 0;
1083
1084out_free_pools:
1085 dm_free_md_mempools(pools);
1086 return -ENOMEM;
1087}
1088
1089static int setup_indexes(struct dm_table *t)
1090{
1091 int i;
1092 unsigned int total = 0;
1093 sector_t *indexes;
1094
1095 /* allocate the space for *all* the indexes */
1096 for (i = t->depth - 2; i >= 0; i--) {
1097 t->counts[i] = dm_div_up(t->counts[i + 1], CHILDREN_PER_NODE);
1098 total += t->counts[i];
1099 }
1100
1101 indexes = kvcalloc(total, NODE_SIZE, GFP_KERNEL);
1102 if (!indexes)
1103 return -ENOMEM;
1104
1105 /* set up internal nodes, bottom-up */
1106 for (i = t->depth - 2; i >= 0; i--) {
1107 t->index[i] = indexes;
1108 indexes += (KEYS_PER_NODE * t->counts[i]);
1109 setup_btree_index(i, t);
1110 }
1111
1112 return 0;
1113}
1114
1115/*
1116 * Builds the btree to index the map.
1117 */
1118static int dm_table_build_index(struct dm_table *t)
1119{
1120 int r = 0;
1121 unsigned int leaf_nodes;
1122
1123 /* how many indexes will the btree have ? */
1124 leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE);
1125 t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE);
1126
1127 /* leaf layer has already been set up */
1128 t->counts[t->depth - 1] = leaf_nodes;
1129 t->index[t->depth - 1] = t->highs;
1130
1131 if (t->depth >= 2)
1132 r = setup_indexes(t);
1133
1134 return r;
1135}
1136
1137#ifdef CONFIG_BLK_INLINE_ENCRYPTION
1138
1139struct dm_crypto_profile {
1140 struct blk_crypto_profile profile;
1141 struct mapped_device *md;
1142};
1143
1144static int dm_keyslot_evict_callback(struct dm_target *ti, struct dm_dev *dev,
1145 sector_t start, sector_t len, void *data)
1146{
1147 const struct blk_crypto_key *key = data;
1148
1149 blk_crypto_evict_key(dev->bdev, key);
1150 return 0;
1151}
1152
1153/*
1154 * When an inline encryption key is evicted from a device-mapper device, evict
1155 * it from all the underlying devices.
1156 */
1157static int dm_keyslot_evict(struct blk_crypto_profile *profile,
1158 const struct blk_crypto_key *key, unsigned int slot)
1159{
1160 struct mapped_device *md =
1161 container_of(profile, struct dm_crypto_profile, profile)->md;
1162 struct dm_table *t;
1163 int srcu_idx;
1164
1165 t = dm_get_live_table(md, &srcu_idx);
1166 if (!t)
1167 goto put_live_table;
1168
1169 for (unsigned int i = 0; i < t->num_targets; i++) {
1170 struct dm_target *ti = dm_table_get_target(t, i);
1171
1172 if (!ti->type->iterate_devices)
1173 continue;
1174 ti->type->iterate_devices(ti, dm_keyslot_evict_callback,
1175 (void *)key);
1176 }
1177
1178put_live_table:
1179 dm_put_live_table(md, srcu_idx);
1180 return 0;
1181}
1182
1183enum dm_wrappedkey_op {
1184 DERIVE_SW_SECRET,
1185 IMPORT_KEY,
1186 GENERATE_KEY,
1187 PREPARE_KEY,
1188};
1189
1190struct dm_wrappedkey_op_args {
1191 enum dm_wrappedkey_op op;
1192 int err;
1193 union {
1194 struct {
1195 const u8 *eph_key;
1196 size_t eph_key_size;
1197 u8 *sw_secret;
1198 } derive_sw_secret;
1199 struct {
1200 const u8 *raw_key;
1201 size_t raw_key_size;
1202 u8 *lt_key;
1203 } import_key;
1204 struct {
1205 u8 *lt_key;
1206 } generate_key;
1207 struct {
1208 const u8 *lt_key;
1209 size_t lt_key_size;
1210 u8 *eph_key;
1211 } prepare_key;
1212 };
1213};
1214
1215static int dm_wrappedkey_op_callback(struct dm_target *ti, struct dm_dev *dev,
1216 sector_t start, sector_t len, void *data)
1217{
1218 struct dm_wrappedkey_op_args *args = data;
1219 struct block_device *bdev = dev->bdev;
1220 struct blk_crypto_profile *profile =
1221 bdev_get_queue(bdev)->crypto_profile;
1222 int err = -EOPNOTSUPP;
1223
1224 switch (args->op) {
1225 case DERIVE_SW_SECRET:
1226 err = blk_crypto_derive_sw_secret(
1227 bdev,
1228 args->derive_sw_secret.eph_key,
1229 args->derive_sw_secret.eph_key_size,
1230 args->derive_sw_secret.sw_secret);
1231 break;
1232 case IMPORT_KEY:
1233 err = blk_crypto_import_key(profile,
1234 args->import_key.raw_key,
1235 args->import_key.raw_key_size,
1236 args->import_key.lt_key);
1237 break;
1238 case GENERATE_KEY:
1239 err = blk_crypto_generate_key(profile,
1240 args->generate_key.lt_key);
1241 break;
1242 case PREPARE_KEY:
1243 err = blk_crypto_prepare_key(profile,
1244 args->prepare_key.lt_key,
1245 args->prepare_key.lt_key_size,
1246 args->prepare_key.eph_key);
1247 break;
1248 }
1249 args->err = err;
1250 return 1; /* No need to continue the iteration. */
1251}
1252
1253static int dm_exec_wrappedkey_op(struct blk_crypto_profile *profile,
1254 struct dm_wrappedkey_op_args *args)
1255{
1256 struct mapped_device *md =
1257 container_of(profile, struct dm_crypto_profile, profile)->md;
1258 struct dm_target *ti;
1259 struct dm_table *t;
1260 int srcu_idx;
1261 int i;
1262
1263 args->err = -EOPNOTSUPP;
1264
1265 t = dm_get_live_table(md, &srcu_idx);
1266 if (!t)
1267 goto out;
1268
1269 /*
1270 * blk-crypto currently has no support for multiple incompatible
1271 * implementations of wrapped inline crypto keys on a single system.
1272 * It was already checked earlier that support for wrapped keys was
1273 * declared on all underlying devices. Thus, all the underlying devices
1274 * should support all wrapped key operations and they should behave
1275 * identically, i.e. work with the same keys. So, just executing the
1276 * operation on the first device suffices for now.
1277 */
1278 for (i = 0; i < t->num_targets; i++) {
1279 ti = dm_table_get_target(t, i);
1280 if (!ti->type->iterate_devices)
1281 continue;
1282 if (ti->type->iterate_devices(ti, dm_wrappedkey_op_callback, args) != 0)
1283 break;
1284 }
1285out:
1286 dm_put_live_table(md, srcu_idx);
1287 return args->err;
1288}
1289
1290static int dm_derive_sw_secret(struct blk_crypto_profile *profile,
1291 const u8 *eph_key, size_t eph_key_size,
1292 u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE])
1293{
1294 struct dm_wrappedkey_op_args args = {
1295 .op = DERIVE_SW_SECRET,
1296 .derive_sw_secret = {
1297 .eph_key = eph_key,
1298 .eph_key_size = eph_key_size,
1299 .sw_secret = sw_secret,
1300 },
1301 };
1302 return dm_exec_wrappedkey_op(profile, &args);
1303}
1304
1305static int dm_import_key(struct blk_crypto_profile *profile,
1306 const u8 *raw_key, size_t raw_key_size,
1307 u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE])
1308{
1309 struct dm_wrappedkey_op_args args = {
1310 .op = IMPORT_KEY,
1311 .import_key = {
1312 .raw_key = raw_key,
1313 .raw_key_size = raw_key_size,
1314 .lt_key = lt_key,
1315 },
1316 };
1317 return dm_exec_wrappedkey_op(profile, &args);
1318}
1319
1320static int dm_generate_key(struct blk_crypto_profile *profile,
1321 u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE])
1322{
1323 struct dm_wrappedkey_op_args args = {
1324 .op = GENERATE_KEY,
1325 .generate_key = {
1326 .lt_key = lt_key,
1327 },
1328 };
1329 return dm_exec_wrappedkey_op(profile, &args);
1330}
1331
1332static int dm_prepare_key(struct blk_crypto_profile *profile,
1333 const u8 *lt_key, size_t lt_key_size,
1334 u8 eph_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE])
1335{
1336 struct dm_wrappedkey_op_args args = {
1337 .op = PREPARE_KEY,
1338 .prepare_key = {
1339 .lt_key = lt_key,
1340 .lt_key_size = lt_key_size,
1341 .eph_key = eph_key,
1342 },
1343 };
1344 return dm_exec_wrappedkey_op(profile, &args);
1345}
1346
1347static int
1348device_intersect_crypto_capabilities(struct dm_target *ti, struct dm_dev *dev,
1349 sector_t start, sector_t len, void *data)
1350{
1351 struct blk_crypto_profile *parent = data;
1352 struct blk_crypto_profile *child =
1353 bdev_get_queue(dev->bdev)->crypto_profile;
1354
1355 blk_crypto_intersect_capabilities(parent, child);
1356 return 0;
1357}
1358
1359void dm_destroy_crypto_profile(struct blk_crypto_profile *profile)
1360{
1361 struct dm_crypto_profile *dmcp = container_of(profile,
1362 struct dm_crypto_profile,
1363 profile);
1364
1365 if (!profile)
1366 return;
1367
1368 blk_crypto_profile_destroy(profile);
1369 kfree(dmcp);
1370}
1371
1372static void dm_table_destroy_crypto_profile(struct dm_table *t)
1373{
1374 dm_destroy_crypto_profile(t->crypto_profile);
1375 t->crypto_profile = NULL;
1376}
1377
1378/*
1379 * Constructs and initializes t->crypto_profile with a crypto profile that
1380 * represents the common set of crypto capabilities of the devices described by
1381 * the dm_table. However, if the constructed crypto profile doesn't support all
1382 * crypto capabilities that are supported by the current mapped_device, it
1383 * returns an error instead, since we don't support removing crypto capabilities
1384 * on table changes. Finally, if the constructed crypto profile is "empty" (has
1385 * no crypto capabilities at all), it just sets t->crypto_profile to NULL.
1386 */
1387static int dm_table_construct_crypto_profile(struct dm_table *t)
1388{
1389 struct dm_crypto_profile *dmcp;
1390 struct blk_crypto_profile *profile;
1391 unsigned int i;
1392 bool empty_profile = true;
1393
1394 dmcp = kmalloc_obj(*dmcp);
1395 if (!dmcp)
1396 return -ENOMEM;
1397 dmcp->md = t->md;
1398
1399 profile = &dmcp->profile;
1400 blk_crypto_profile_init(profile, 0);
1401 profile->ll_ops.keyslot_evict = dm_keyslot_evict;
1402 profile->max_dun_bytes_supported = UINT_MAX;
1403 memset(profile->modes_supported, 0xFF,
1404 sizeof(profile->modes_supported));
1405 profile->key_types_supported = ~0;
1406
1407 for (i = 0; i < t->num_targets; i++) {
1408 struct dm_target *ti = dm_table_get_target(t, i);
1409
1410 if (!dm_target_passes_crypto(ti->type)) {
1411 blk_crypto_intersect_capabilities(profile, NULL);
1412 break;
1413 }
1414 if (!ti->type->iterate_devices)
1415 continue;
1416 ti->type->iterate_devices(ti,
1417 device_intersect_crypto_capabilities,
1418 profile);
1419 }
1420
1421 if (profile->key_types_supported & BLK_CRYPTO_KEY_TYPE_HW_WRAPPED) {
1422 profile->ll_ops.derive_sw_secret = dm_derive_sw_secret;
1423 profile->ll_ops.import_key = dm_import_key;
1424 profile->ll_ops.generate_key = dm_generate_key;
1425 profile->ll_ops.prepare_key = dm_prepare_key;
1426 }
1427
1428 if (t->md->queue &&
1429 !blk_crypto_has_capabilities(profile,
1430 t->md->queue->crypto_profile)) {
1431 DMERR("Inline encryption capabilities of new DM table were more restrictive than the old table's. This is not supported!");
1432 dm_destroy_crypto_profile(profile);
1433 return -EINVAL;
1434 }
1435
1436 /*
1437 * If the new profile doesn't actually support any crypto capabilities,
1438 * we may as well represent it with a NULL profile.
1439 */
1440 for (i = 0; i < ARRAY_SIZE(profile->modes_supported); i++) {
1441 if (profile->modes_supported[i]) {
1442 empty_profile = false;
1443 break;
1444 }
1445 }
1446
1447 if (empty_profile) {
1448 dm_destroy_crypto_profile(profile);
1449 profile = NULL;
1450 }
1451
1452 /*
1453 * t->crypto_profile is only set temporarily while the table is being
1454 * set up, and it gets set to NULL after the profile has been
1455 * transferred to the request_queue.
1456 */
1457 t->crypto_profile = profile;
1458
1459 return 0;
1460}
1461
1462static void dm_update_crypto_profile(struct request_queue *q,
1463 struct dm_table *t)
1464{
1465 if (!t->crypto_profile)
1466 return;
1467
1468 /* Make the crypto profile less restrictive. */
1469 if (!q->crypto_profile) {
1470 blk_crypto_register(t->crypto_profile, q);
1471 } else {
1472 blk_crypto_update_capabilities(q->crypto_profile,
1473 t->crypto_profile);
1474 dm_destroy_crypto_profile(t->crypto_profile);
1475 }
1476 t->crypto_profile = NULL;
1477}
1478
1479#else /* CONFIG_BLK_INLINE_ENCRYPTION */
1480
1481static int dm_table_construct_crypto_profile(struct dm_table *t)
1482{
1483 return 0;
1484}
1485
1486void dm_destroy_crypto_profile(struct blk_crypto_profile *profile)
1487{
1488}
1489
1490static void dm_table_destroy_crypto_profile(struct dm_table *t)
1491{
1492}
1493
1494static void dm_update_crypto_profile(struct request_queue *q,
1495 struct dm_table *t)
1496{
1497}
1498
1499#endif /* !CONFIG_BLK_INLINE_ENCRYPTION */
1500
1501/*
1502 * Prepares the table for use by building the indices,
1503 * setting the type, and allocating mempools.
1504 */
1505int dm_table_complete(struct dm_table *t)
1506{
1507 int r;
1508
1509 r = dm_table_determine_type(t);
1510 if (r) {
1511 DMERR("unable to determine table type");
1512 return r;
1513 }
1514
1515 r = dm_table_build_index(t);
1516 if (r) {
1517 DMERR("unable to build btrees");
1518 return r;
1519 }
1520
1521 r = dm_table_construct_crypto_profile(t);
1522 if (r) {
1523 DMERR("could not construct crypto profile.");
1524 return r;
1525 }
1526
1527 r = dm_table_alloc_md_mempools(t, t->md);
1528 if (r)
1529 DMERR("unable to allocate mempools");
1530
1531 return r;
1532}
1533
1534static DEFINE_MUTEX(_event_lock);
1535void dm_table_event_callback(struct dm_table *t,
1536 void (*fn)(void *), void *context)
1537{
1538 mutex_lock(&_event_lock);
1539 t->event_fn = fn;
1540 t->event_context = context;
1541 mutex_unlock(&_event_lock);
1542}
1543
1544void dm_table_event(struct dm_table *t)
1545{
1546 mutex_lock(&_event_lock);
1547 if (t->event_fn)
1548 t->event_fn(t->event_context);
1549 mutex_unlock(&_event_lock);
1550}
1551EXPORT_SYMBOL(dm_table_event);
1552
1553inline sector_t dm_table_get_size(struct dm_table *t)
1554{
1555 return t->num_targets ? (t->highs[t->num_targets - 1] + 1) : 0;
1556}
1557EXPORT_SYMBOL(dm_table_get_size);
1558
1559/*
1560 * Search the btree for the correct target.
1561 *
1562 * Caller should check returned pointer for NULL
1563 * to trap I/O beyond end of device.
1564 */
1565struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector)
1566{
1567 unsigned int l, n = 0, k = 0;
1568 sector_t *node;
1569
1570 if (unlikely(sector >= dm_table_get_size(t)))
1571 return NULL;
1572
1573 for (l = 0; l < t->depth; l++) {
1574 n = get_child(n, k);
1575 node = get_node(t, l, n);
1576
1577 for (k = 0; k < KEYS_PER_NODE; k++)
1578 if (node[k] >= sector)
1579 break;
1580 }
1581
1582 return &t->targets[(KEYS_PER_NODE * n) + k];
1583}
1584
1585/*
1586 * type->iterate_devices() should be called when the sanity check needs to
1587 * iterate and check all underlying data devices. iterate_devices() will
1588 * iterate all underlying data devices until it encounters a non-zero return
1589 * code, returned by whether the input iterate_devices_callout_fn, or
1590 * iterate_devices() itself internally.
1591 *
1592 * For some target type (e.g. dm-stripe), one call of iterate_devices() may
1593 * iterate multiple underlying devices internally, in which case a non-zero
1594 * return code returned by iterate_devices_callout_fn will stop the iteration
1595 * in advance.
1596 *
1597 * Cases requiring _any_ underlying device supporting some kind of attribute,
1598 * should use the iteration structure like dm_table_any_dev_attr(), or call
1599 * it directly. @func should handle semantics of positive examples, e.g.
1600 * capable of something.
1601 *
1602 * Cases requiring _all_ underlying devices supporting some kind of attribute,
1603 * should use the iteration structure like dm_table_supports_nowait() or
1604 * dm_table_supports_discards(). Or introduce dm_table_all_devs_attr() that
1605 * uses an @anti_func that handle semantics of counter examples, e.g. not
1606 * capable of something. So: return !dm_table_any_dev_attr(t, anti_func, data);
1607 */
1608static bool dm_table_any_dev_attr(struct dm_table *t,
1609 iterate_devices_callout_fn func, void *data)
1610{
1611 for (unsigned int i = 0; i < t->num_targets; i++) {
1612 struct dm_target *ti = dm_table_get_target(t, i);
1613
1614 if (ti->type->iterate_devices &&
1615 ti->type->iterate_devices(ti, func, data))
1616 return true;
1617 }
1618
1619 return false;
1620}
1621
1622static int count_device(struct dm_target *ti, struct dm_dev *dev,
1623 sector_t start, sector_t len, void *data)
1624{
1625 unsigned int *num_devices = data;
1626
1627 (*num_devices)++;
1628
1629 return 0;
1630}
1631
1632/*
1633 * Check whether a table has no data devices attached using each
1634 * target's iterate_devices method.
1635 * Returns false if the result is unknown because a target doesn't
1636 * support iterate_devices.
1637 */
1638bool dm_table_has_no_data_devices(struct dm_table *t)
1639{
1640 for (unsigned int i = 0; i < t->num_targets; i++) {
1641 struct dm_target *ti = dm_table_get_target(t, i);
1642 unsigned int num_devices = 0;
1643
1644 if (!ti->type->iterate_devices)
1645 return false;
1646
1647 ti->type->iterate_devices(ti, count_device, &num_devices);
1648 if (num_devices)
1649 return false;
1650 }
1651
1652 return true;
1653}
1654
1655bool dm_table_is_wildcard(struct dm_table *t)
1656{
1657 for (unsigned int i = 0; i < t->num_targets; i++) {
1658 struct dm_target *ti = dm_table_get_target(t, i);
1659
1660 if (!dm_target_is_wildcard(ti->type))
1661 return false;
1662 }
1663
1664 return true;
1665}
1666
1667static int device_not_zoned(struct dm_target *ti, struct dm_dev *dev,
1668 sector_t start, sector_t len, void *data)
1669{
1670 bool *zoned = data;
1671
1672 return bdev_is_zoned(dev->bdev) != *zoned;
1673}
1674
1675static int device_is_zoned_model(struct dm_target *ti, struct dm_dev *dev,
1676 sector_t start, sector_t len, void *data)
1677{
1678 return bdev_is_zoned(dev->bdev);
1679}
1680
1681/*
1682 * Check the device zoned model based on the target feature flag. If the target
1683 * has the DM_TARGET_ZONED_HM feature flag set, host-managed zoned devices are
1684 * also accepted but all devices must have the same zoned model. If the target
1685 * has the DM_TARGET_MIXED_ZONED_MODEL feature set, the devices can have any
1686 * zoned model with all zoned devices having the same zone size.
1687 */
1688static bool dm_table_supports_zoned(struct dm_table *t, bool zoned)
1689{
1690 for (unsigned int i = 0; i < t->num_targets; i++) {
1691 struct dm_target *ti = dm_table_get_target(t, i);
1692
1693 /*
1694 * For the wildcard target (dm-error), if we do not have a
1695 * backing device, we must always return false. If we have a
1696 * backing device, the result must depend on checking zoned
1697 * model, like for any other target. So for this, check directly
1698 * if the target backing device is zoned as we get "false" when
1699 * dm-error was set without a backing device.
1700 */
1701 if (dm_target_is_wildcard(ti->type) &&
1702 !ti->type->iterate_devices(ti, device_is_zoned_model, NULL))
1703 return false;
1704
1705 if (dm_target_supports_zoned_hm(ti->type)) {
1706 if (!ti->type->iterate_devices ||
1707 ti->type->iterate_devices(ti, device_not_zoned,
1708 &zoned))
1709 return false;
1710 } else if (!dm_target_supports_mixed_zoned_model(ti->type)) {
1711 if (zoned)
1712 return false;
1713 }
1714 }
1715
1716 return true;
1717}
1718
1719static int device_not_matches_zone_sectors(struct dm_target *ti, struct dm_dev *dev,
1720 sector_t start, sector_t len, void *data)
1721{
1722 unsigned int *zone_sectors = data;
1723
1724 if (!bdev_is_zoned(dev->bdev))
1725 return 0;
1726 return bdev_zone_sectors(dev->bdev) != *zone_sectors;
1727}
1728
1729/*
1730 * Check consistency of zoned model and zone sectors across all targets. For
1731 * zone sectors, if the destination device is a zoned block device, it shall
1732 * have the specified zone_sectors.
1733 */
1734static int validate_hardware_zoned(struct dm_table *t, bool zoned,
1735 unsigned int zone_sectors)
1736{
1737 if (!zoned)
1738 return 0;
1739
1740 if (!dm_table_supports_zoned(t, zoned)) {
1741 DMERR("%s: zoned model is not consistent across all devices",
1742 dm_device_name(t->md));
1743 return -EINVAL;
1744 }
1745
1746 /* Check zone size validity and compatibility */
1747 if (!zone_sectors || !is_power_of_2(zone_sectors))
1748 return -EINVAL;
1749
1750 if (dm_table_any_dev_attr(t, device_not_matches_zone_sectors, &zone_sectors)) {
1751 DMERR("%s: zone sectors is not consistent across all zoned devices",
1752 dm_device_name(t->md));
1753 return -EINVAL;
1754 }
1755
1756 return 0;
1757}
1758
1759/*
1760 * Establish the new table's queue_limits and validate them.
1761 */
1762int dm_calculate_queue_limits(struct dm_table *t,
1763 struct queue_limits *limits)
1764{
1765 struct queue_limits ti_limits;
1766 unsigned int zone_sectors = 0;
1767 bool zoned = false;
1768
1769 dm_set_stacking_limits(limits);
1770
1771 t->integrity_supported = true;
1772 for (unsigned int i = 0; i < t->num_targets; i++) {
1773 struct dm_target *ti = dm_table_get_target(t, i);
1774
1775 if (!dm_target_passes_integrity(ti->type))
1776 t->integrity_supported = false;
1777 }
1778
1779 for (unsigned int i = 0; i < t->num_targets; i++) {
1780 struct dm_target *ti = dm_table_get_target(t, i);
1781
1782 dm_set_stacking_limits(&ti_limits);
1783
1784 if (!ti->type->iterate_devices) {
1785 /* Set I/O hints portion of queue limits */
1786 if (ti->type->io_hints)
1787 ti->type->io_hints(ti, &ti_limits);
1788 goto combine_limits;
1789 }
1790
1791 /*
1792 * Combine queue limits of all the devices this target uses.
1793 */
1794 ti->type->iterate_devices(ti, dm_set_device_limits,
1795 &ti_limits);
1796
1797 if (!zoned && (ti_limits.features & BLK_FEAT_ZONED)) {
1798 /*
1799 * After stacking all limits, validate all devices
1800 * in table support this zoned model and zone sectors.
1801 */
1802 zoned = (ti_limits.features & BLK_FEAT_ZONED);
1803 zone_sectors = ti_limits.chunk_sectors;
1804 }
1805
1806 /* Set I/O hints portion of queue limits */
1807 if (ti->type->io_hints)
1808 ti->type->io_hints(ti, &ti_limits);
1809
1810 /*
1811 * Check each device area is consistent with the target's
1812 * overall queue limits.
1813 */
1814 if (ti->type->iterate_devices(ti, device_area_is_invalid,
1815 &ti_limits))
1816 return -EINVAL;
1817
1818combine_limits:
1819 /*
1820 * Merge this target's queue limits into the overall limits
1821 * for the table.
1822 */
1823 if (blk_stack_limits(limits, &ti_limits, 0) < 0)
1824 DMWARN("%s: adding target device (start sect %llu len %llu) "
1825 "caused an alignment inconsistency",
1826 dm_device_name(t->md),
1827 (unsigned long long) ti->begin,
1828 (unsigned long long) ti->len);
1829
1830 if (t->integrity_supported ||
1831 dm_target_has_integrity(ti->type)) {
1832 if (!queue_limits_stack_integrity(limits, &ti_limits)) {
1833 DMWARN("%s: adding target device (start sect %llu len %llu) "
1834 "disabled integrity support due to incompatibility",
1835 dm_device_name(t->md),
1836 (unsigned long long) ti->begin,
1837 (unsigned long long) ti->len);
1838 t->integrity_supported = false;
1839 }
1840 }
1841 }
1842
1843 /*
1844 * Verify that the zoned model and zone sectors, as determined before
1845 * any .io_hints override, are the same across all devices in the table.
1846 * - this is especially relevant if .io_hints is emulating a disk-managed
1847 * zoned model on host-managed zoned block devices.
1848 * BUT...
1849 */
1850 if (limits->features & BLK_FEAT_ZONED) {
1851 /*
1852 * ...IF the above limits stacking determined a zoned model
1853 * validate that all of the table's devices conform to it.
1854 */
1855 zoned = limits->features & BLK_FEAT_ZONED;
1856 zone_sectors = limits->chunk_sectors;
1857 }
1858 if (validate_hardware_zoned(t, zoned, zone_sectors))
1859 return -EINVAL;
1860
1861 return validate_hardware_logical_block_alignment(t, limits);
1862}
1863
1864/*
1865 * Check if a target requires flush support even if none of the underlying
1866 * devices need it (e.g. to persist target-specific metadata).
1867 */
1868static bool dm_table_supports_flush(struct dm_table *t)
1869{
1870 for (unsigned int i = 0; i < t->num_targets; i++) {
1871 struct dm_target *ti = dm_table_get_target(t, i);
1872
1873 if (ti->num_flush_bios && ti->flush_supported)
1874 return true;
1875 }
1876
1877 return false;
1878}
1879
1880static int device_dax_write_cache_enabled(struct dm_target *ti,
1881 struct dm_dev *dev, sector_t start,
1882 sector_t len, void *data)
1883{
1884 struct dax_device *dax_dev = dev->dax_dev;
1885
1886 if (!dax_dev)
1887 return false;
1888
1889 if (dax_write_cache_enabled(dax_dev))
1890 return true;
1891 return false;
1892}
1893
1894static int device_not_write_zeroes_capable(struct dm_target *ti, struct dm_dev *dev,
1895 sector_t start, sector_t len, void *data)
1896{
1897 struct request_queue *q = bdev_get_queue(dev->bdev);
1898 int b;
1899
1900 mutex_lock(&q->limits_lock);
1901 b = !q->limits.max_write_zeroes_sectors;
1902 mutex_unlock(&q->limits_lock);
1903 return b;
1904}
1905
1906static bool dm_table_supports_write_zeroes(struct dm_table *t)
1907{
1908 for (unsigned int i = 0; i < t->num_targets; i++) {
1909 struct dm_target *ti = dm_table_get_target(t, i);
1910
1911 if (!ti->num_write_zeroes_bios)
1912 return false;
1913
1914 if (!ti->type->iterate_devices ||
1915 ti->type->iterate_devices(ti, device_not_write_zeroes_capable, NULL))
1916 return false;
1917 }
1918
1919 return true;
1920}
1921
1922static bool dm_table_supports_nowait(struct dm_table *t)
1923{
1924 for (unsigned int i = 0; i < t->num_targets; i++) {
1925 struct dm_target *ti = dm_table_get_target(t, i);
1926
1927 if (!dm_target_supports_nowait(ti->type))
1928 return false;
1929 }
1930
1931 return true;
1932}
1933
1934static int device_not_discard_capable(struct dm_target *ti, struct dm_dev *dev,
1935 sector_t start, sector_t len, void *data)
1936{
1937 return !bdev_max_discard_sectors(dev->bdev);
1938}
1939
1940static bool dm_table_supports_discards(struct dm_table *t)
1941{
1942 for (unsigned int i = 0; i < t->num_targets; i++) {
1943 struct dm_target *ti = dm_table_get_target(t, i);
1944
1945 if (!ti->num_discard_bios)
1946 return false;
1947
1948 /*
1949 * Either the target provides discard support (as implied by setting
1950 * 'discards_supported') or it relies on _all_ data devices having
1951 * discard support.
1952 */
1953 if (!ti->discards_supported &&
1954 (!ti->type->iterate_devices ||
1955 ti->type->iterate_devices(ti, device_not_discard_capable, NULL)))
1956 return false;
1957 }
1958
1959 return true;
1960}
1961
1962static int device_not_secure_erase_capable(struct dm_target *ti,
1963 struct dm_dev *dev, sector_t start,
1964 sector_t len, void *data)
1965{
1966 return !bdev_max_secure_erase_sectors(dev->bdev);
1967}
1968
1969static bool dm_table_supports_secure_erase(struct dm_table *t)
1970{
1971 for (unsigned int i = 0; i < t->num_targets; i++) {
1972 struct dm_target *ti = dm_table_get_target(t, i);
1973
1974 if (!ti->num_secure_erase_bios)
1975 return false;
1976
1977 if (!ti->type->iterate_devices ||
1978 ti->type->iterate_devices(ti, device_not_secure_erase_capable, NULL))
1979 return false;
1980 }
1981
1982 return true;
1983}
1984
1985static int device_not_atomic_write_capable(struct dm_target *ti,
1986 struct dm_dev *dev, sector_t start,
1987 sector_t len, void *data)
1988{
1989 return !bdev_can_atomic_write(dev->bdev);
1990}
1991
1992static bool dm_table_supports_atomic_writes(struct dm_table *t)
1993{
1994 for (unsigned int i = 0; i < t->num_targets; i++) {
1995 struct dm_target *ti = dm_table_get_target(t, i);
1996
1997 if (!dm_target_supports_atomic_writes(ti->type))
1998 return false;
1999
2000 if (!ti->type->iterate_devices)
2001 return false;
2002
2003 if (ti->type->iterate_devices(ti,
2004 device_not_atomic_write_capable, NULL)) {
2005 return false;
2006 }
2007 }
2008 return true;
2009}
2010
2011bool dm_table_supports_size_change(struct dm_table *t, sector_t old_size,
2012 sector_t new_size)
2013{
2014 if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) && dm_has_zone_plugs(t->md) &&
2015 old_size != new_size) {
2016 DMWARN("%s: device has zone write plug resources. "
2017 "Cannot change size",
2018 dm_device_name(t->md));
2019 return false;
2020 }
2021 return true;
2022}
2023
2024/*
2025 * This function will be skipped by noflush reloads of immutable request
2026 * based devices (dm-mpath).
2027 */
2028int dm_table_set_restrictions(struct dm_table *t, struct request_queue *q,
2029 struct queue_limits *limits)
2030{
2031 int r;
2032 struct queue_limits old_limits;
2033
2034 if (!dm_table_supports_nowait(t))
2035 limits->features &= ~BLK_FEAT_NOWAIT;
2036
2037 /*
2038 * The current polling implementation does not support request based
2039 * stacking.
2040 */
2041 if (!__table_type_bio_based(t->type))
2042 limits->features &= ~BLK_FEAT_POLL;
2043
2044 if (!dm_table_supports_discards(t)) {
2045 limits->max_hw_discard_sectors = 0;
2046 limits->discard_granularity = 0;
2047 limits->discard_alignment = 0;
2048 }
2049
2050 if (!dm_table_supports_write_zeroes(t)) {
2051 limits->max_write_zeroes_sectors = 0;
2052 limits->max_hw_wzeroes_unmap_sectors = 0;
2053 }
2054
2055 if (!dm_table_supports_secure_erase(t))
2056 limits->max_secure_erase_sectors = 0;
2057
2058 if (dm_table_supports_flush(t))
2059 limits->features |= BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA;
2060
2061 if (dm_table_supports_dax(t, device_not_dax_capable))
2062 limits->features |= BLK_FEAT_DAX;
2063 else
2064 limits->features &= ~BLK_FEAT_DAX;
2065
2066 /* For a zoned table, setup the zone related queue attributes. */
2067 if (IS_ENABLED(CONFIG_BLK_DEV_ZONED)) {
2068 if (limits->features & BLK_FEAT_ZONED) {
2069 r = dm_set_zones_restrictions(t, q, limits);
2070 if (r)
2071 return r;
2072 } else if (dm_has_zone_plugs(t->md)) {
2073 DMWARN("%s: device has zone write plug resources. "
2074 "Cannot switch to non-zoned table.",
2075 dm_device_name(t->md));
2076 return -EINVAL;
2077 }
2078 }
2079
2080 if (dm_table_supports_atomic_writes(t))
2081 limits->features |= BLK_FEAT_ATOMIC_WRITES;
2082
2083 old_limits = queue_limits_start_update(q);
2084 r = queue_limits_commit_update(q, limits);
2085 if (r)
2086 return r;
2087
2088 /*
2089 * Now that the limits are set, check the zones mapped by the table
2090 * and setup the resources for zone append emulation if necessary.
2091 */
2092 if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
2093 (limits->features & BLK_FEAT_ZONED)) {
2094 r = dm_revalidate_zones(t, q);
2095 if (r) {
2096 queue_limits_set(q, &old_limits);
2097 return r;
2098 }
2099 }
2100
2101 if (IS_ENABLED(CONFIG_BLK_DEV_ZONED))
2102 dm_finalize_zone_settings(t, limits);
2103
2104 if (dm_table_supports_dax(t, device_not_dax_synchronous_capable))
2105 set_dax_synchronous(t->md->dax_dev);
2106
2107 if (dm_table_any_dev_attr(t, device_dax_write_cache_enabled, NULL))
2108 dax_write_cache(t->md->dax_dev, true);
2109
2110 dm_update_crypto_profile(q, t);
2111 return 0;
2112}
2113
2114struct list_head *dm_table_get_devices(struct dm_table *t)
2115{
2116 return &t->devices;
2117}
2118
2119blk_mode_t dm_table_get_mode(struct dm_table *t)
2120{
2121 return t->mode;
2122}
2123EXPORT_SYMBOL(dm_table_get_mode);
2124
2125enum suspend_mode {
2126 PRESUSPEND,
2127 PRESUSPEND_UNDO,
2128 POSTSUSPEND,
2129};
2130
2131static void suspend_targets(struct dm_table *t, enum suspend_mode mode)
2132{
2133 lockdep_assert_held(&t->md->suspend_lock);
2134
2135 for (unsigned int i = 0; i < t->num_targets; i++) {
2136 struct dm_target *ti = dm_table_get_target(t, i);
2137
2138 switch (mode) {
2139 case PRESUSPEND:
2140 if (ti->type->presuspend)
2141 ti->type->presuspend(ti);
2142 break;
2143 case PRESUSPEND_UNDO:
2144 if (ti->type->presuspend_undo)
2145 ti->type->presuspend_undo(ti);
2146 break;
2147 case POSTSUSPEND:
2148 if (ti->type->postsuspend)
2149 ti->type->postsuspend(ti);
2150 break;
2151 }
2152 }
2153}
2154
2155void dm_table_presuspend_targets(struct dm_table *t)
2156{
2157 if (!t)
2158 return;
2159
2160 suspend_targets(t, PRESUSPEND);
2161}
2162
2163void dm_table_presuspend_undo_targets(struct dm_table *t)
2164{
2165 if (!t)
2166 return;
2167
2168 suspend_targets(t, PRESUSPEND_UNDO);
2169}
2170
2171void dm_table_postsuspend_targets(struct dm_table *t)
2172{
2173 if (!t)
2174 return;
2175
2176 suspend_targets(t, POSTSUSPEND);
2177}
2178
2179int dm_table_resume_targets(struct dm_table *t)
2180{
2181 unsigned int i;
2182 int r = 0;
2183
2184 lockdep_assert_held(&t->md->suspend_lock);
2185
2186 for (i = 0; i < t->num_targets; i++) {
2187 struct dm_target *ti = dm_table_get_target(t, i);
2188
2189 if (!ti->type->preresume)
2190 continue;
2191
2192 r = ti->type->preresume(ti);
2193 if (r) {
2194 DMERR("%s: %s: preresume failed, error = %d",
2195 dm_device_name(t->md), ti->type->name, r);
2196 return r;
2197 }
2198 }
2199
2200 for (i = 0; i < t->num_targets; i++) {
2201 struct dm_target *ti = dm_table_get_target(t, i);
2202
2203 if (ti->type->resume)
2204 ti->type->resume(ti);
2205 }
2206
2207 return 0;
2208}
2209
2210struct mapped_device *dm_table_get_md(struct dm_table *t)
2211{
2212 return t->md;
2213}
2214EXPORT_SYMBOL(dm_table_get_md);
2215
2216const char *dm_table_device_name(struct dm_table *t)
2217{
2218 return dm_device_name(t->md);
2219}
2220EXPORT_SYMBOL_GPL(dm_table_device_name);
2221
2222void dm_table_run_md_queue_async(struct dm_table *t)
2223{
2224 if (!dm_table_request_based(t))
2225 return;
2226
2227 if (t->md->queue)
2228 blk_mq_run_hw_queues(t->md->queue, true);
2229}
2230EXPORT_SYMBOL(dm_table_run_md_queue_async);
2231