projects/linux·blob assoc-array-fixes · refs

1447 lines · 37.6 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 1991, 1992 Linus Torvalds
4 * Copyright (C) 2001 Andrea Arcangeli <[email protected]> SuSE
5 * Copyright (C) 2016 - 2020 Christoph Hellwig
6 */
7
8#include <linux/init.h>
9#include <linux/mm.h>
10#include <linux/slab.h>
11#include <linux/kmod.h>
12#include <linux/major.h>
13#include <linux/device_cgroup.h>
14#include <linux/blkdev.h>
15#include <linux/backing-dev.h>
16#include <linux/module.h>
17#include <linux/blkpg.h>
18#include <linux/magic.h>
19#include <linux/buffer_head.h>
20#include <linux/swap.h>
21#include <linux/writeback.h>
22#include <linux/mount.h>
23#include <linux/pseudo_fs.h>
24#include <linux/uio.h>
25#include <linux/namei.h>
26#include <linux/security.h>
27#include <linux/part_stat.h>
28#include <linux/uaccess.h>
29#include <linux/stat.h>
30#include "../fs/internal.h"
31#include "blk.h"
32
33/* Should we allow writing to mounted block devices? */
34static bool bdev_allow_write_mounted = IS_ENABLED(CONFIG_BLK_DEV_WRITE_MOUNTED);
35
36struct bdev_inode {
37 struct block_device bdev;
38 struct inode vfs_inode;
39};
40
41static inline struct bdev_inode *BDEV_I(struct inode *inode)
42{
43 return container_of(inode, struct bdev_inode, vfs_inode);
44}
45
46static inline struct inode *BD_INODE(struct block_device *bdev)
47{
48 return &container_of(bdev, struct bdev_inode, bdev)->vfs_inode;
49}
50
51struct block_device *I_BDEV(struct inode *inode)
52{
53 return &BDEV_I(inode)->bdev;
54}
55EXPORT_SYMBOL(I_BDEV);
56
57struct block_device *file_bdev(struct file *bdev_file)
58{
59 return I_BDEV(bdev_file->f_mapping->host);
60}
61EXPORT_SYMBOL(file_bdev);
62
63static void bdev_write_inode(struct block_device *bdev)
64{
65 struct inode *inode = BD_INODE(bdev);
66 int ret;
67
68 spin_lock(&inode->i_lock);
69 while (inode_state_read(inode) & I_DIRTY) {
70 spin_unlock(&inode->i_lock);
71 ret = write_inode_now(inode, true);
72 if (ret)
73 pr_warn_ratelimited(
74 "VFS: Dirty inode writeback failed for block device %pg (err=%d).\n",
75 bdev, ret);
76 spin_lock(&inode->i_lock);
77 }
78 spin_unlock(&inode->i_lock);
79}
80
81/* Kill _all_ buffers and pagecache , dirty or not.. */
82static void kill_bdev(struct block_device *bdev)
83{
84 struct address_space *mapping = bdev->bd_mapping;
85
86 if (mapping_empty(mapping))
87 return;
88
89 invalidate_bh_lrus();
90 truncate_inode_pages(mapping, 0);
91}
92
93/* Invalidate clean unused buffers and pagecache. */
94void invalidate_bdev(struct block_device *bdev)
95{
96 struct address_space *mapping = bdev->bd_mapping;
97
98 if (mapping->nrpages) {
99 invalidate_bh_lrus();
100 lru_add_drain_all(); /* make sure all lru add caches are flushed */
101 invalidate_mapping_pages(mapping, 0, -1);
102 }
103}
104EXPORT_SYMBOL(invalidate_bdev);
105
106/*
107 * Drop all buffers & page cache for given bdev range. This function bails
108 * with error if bdev has other exclusive owner (such as filesystem).
109 */
110int truncate_bdev_range(struct block_device *bdev, blk_mode_t mode,
111 loff_t lstart, loff_t lend)
112{
113 /*
114 * If we don't hold exclusive handle for the device, upgrade to it
115 * while we discard the buffer cache to avoid discarding buffers
116 * under live filesystem.
117 */
118 if (!(mode & BLK_OPEN_EXCL)) {
119 int err = bd_prepare_to_claim(bdev, truncate_bdev_range, NULL);
120 if (err)
121 goto invalidate;
122 }
123
124 truncate_inode_pages_range(bdev->bd_mapping, lstart, lend);
125 if (!(mode & BLK_OPEN_EXCL))
126 bd_abort_claiming(bdev, truncate_bdev_range);
127 return 0;
128
129invalidate:
130 /*
131 * Someone else has handle exclusively open. Try invalidating instead.
132 * The 'end' argument is inclusive so the rounding is safe.
133 */
134 return invalidate_inode_pages2_range(bdev->bd_mapping,
135 lstart >> PAGE_SHIFT,
136 lend >> PAGE_SHIFT);
137}
138
139static void set_init_blocksize(struct block_device *bdev)
140{
141 unsigned int bsize = bdev_logical_block_size(bdev);
142 loff_t size = i_size_read(BD_INODE(bdev));
143
144 while (bsize < PAGE_SIZE) {
145 if (size & bsize)
146 break;
147 bsize <<= 1;
148 }
149 BD_INODE(bdev)->i_blkbits = blksize_bits(bsize);
150 mapping_set_folio_min_order(BD_INODE(bdev)->i_mapping,
151 get_order(bsize));
152}
153
154/**
155 * bdev_validate_blocksize - check that this block size is acceptable
156 * @bdev: blockdevice to check
157 * @block_size: block size to check
158 *
159 * For block device users that do not use buffer heads or the block device
160 * page cache, make sure that this block size can be used with the device.
161 *
162 * Return: On success zero is returned, negative error code on failure.
163 */
164int bdev_validate_blocksize(struct block_device *bdev, int block_size)
165{
166 if (blk_validate_block_size(block_size))
167 return -EINVAL;
168
169 /* Size cannot be smaller than the size supported by the device */
170 if (block_size < bdev_logical_block_size(bdev))
171 return -EINVAL;
172
173 return 0;
174}
175EXPORT_SYMBOL_GPL(bdev_validate_blocksize);
176
177int set_blocksize(struct file *file, int size)
178{
179 struct inode *inode = file->f_mapping->host;
180 struct block_device *bdev = I_BDEV(inode);
181 int ret;
182
183 ret = bdev_validate_blocksize(bdev, size);
184 if (ret)
185 return ret;
186
187 if (!file->private_data)
188 return -EINVAL;
189
190 /* Don't change the size if it is same as current */
191 if (inode->i_blkbits != blksize_bits(size)) {
192 /*
193 * Flush and truncate the pagecache before we reconfigure the
194 * mapping geometry because folio sizes are variable now. If a
195 * reader has already allocated a folio whose size is smaller
196 * than the new min_order but invokes readahead after the new
197 * min_order becomes visible, readahead will think there are
198 * "zero" blocks per folio and crash. Take the inode and
199 * invalidation locks to avoid racing with
200 * read/write/fallocate.
201 */
202 inode_lock(inode);
203 filemap_invalidate_lock(inode->i_mapping);
204
205 sync_blockdev(bdev);
206 kill_bdev(bdev);
207
208 inode->i_blkbits = blksize_bits(size);
209 mapping_set_folio_min_order(inode->i_mapping, get_order(size));
210 filemap_invalidate_unlock(inode->i_mapping);
211 inode_unlock(inode);
212 }
213 return 0;
214}
215
216EXPORT_SYMBOL(set_blocksize);
217
218static int sb_validate_large_blocksize(struct super_block *sb, int size)
219{
220 const char *err_str = NULL;
221
222 if (!(sb->s_type->fs_flags & FS_LBS))
223 err_str = "not supported by filesystem";
224 else if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
225 err_str = "is only supported with CONFIG_TRANSPARENT_HUGEPAGE";
226
227 if (!err_str)
228 return 0;
229
230 pr_warn_ratelimited("%s: block size(%d) > page size(%lu) %s\n",
231 sb->s_type->name, size, PAGE_SIZE, err_str);
232 return -EINVAL;
233}
234
235int sb_set_blocksize(struct super_block *sb, int size)
236{
237 if (size > PAGE_SIZE && sb_validate_large_blocksize(sb, size))
238 return 0;
239 if (set_blocksize(sb->s_bdev_file, size))
240 return 0;
241 /* If we get here, we know size is validated */
242 sb->s_blocksize = size;
243 sb->s_blocksize_bits = blksize_bits(size);
244 return sb->s_blocksize;
245}
246
247EXPORT_SYMBOL(sb_set_blocksize);
248
249int __must_check sb_min_blocksize(struct super_block *sb, int size)
250{
251 int minsize = bdev_logical_block_size(sb->s_bdev);
252 if (size < minsize)
253 size = minsize;
254 return sb_set_blocksize(sb, size);
255}
256
257EXPORT_SYMBOL(sb_min_blocksize);
258
259int sync_blockdev_nowait(struct block_device *bdev)
260{
261 if (!bdev)
262 return 0;
263 return filemap_flush(bdev->bd_mapping);
264}
265EXPORT_SYMBOL_GPL(sync_blockdev_nowait);
266
267/*
268 * Write out and wait upon all the dirty data associated with a block
269 * device via its mapping. Does not take the superblock lock.
270 */
271int sync_blockdev(struct block_device *bdev)
272{
273 if (!bdev)
274 return 0;
275 return filemap_write_and_wait(bdev->bd_mapping);
276}
277EXPORT_SYMBOL(sync_blockdev);
278
279int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend)
280{
281 return filemap_write_and_wait_range(bdev->bd_mapping,
282 lstart, lend);
283}
284EXPORT_SYMBOL(sync_blockdev_range);
285
286/**
287 * bdev_freeze - lock a filesystem and force it into a consistent state
288 * @bdev: blockdevice to lock
289 *
290 * If a superblock is found on this device, we take the s_umount semaphore
291 * on it to make sure nobody unmounts until the snapshot creation is done.
292 * The reference counter (bd_fsfreeze_count) guarantees that only the last
293 * unfreeze process can unfreeze the frozen filesystem actually when multiple
294 * freeze requests arrive simultaneously. It counts up in bdev_freeze() and
295 * count down in bdev_thaw(). When it becomes 0, thaw_bdev() will unfreeze
296 * actually.
297 *
298 * Return: On success zero is returned, negative error code on failure.
299 */
300int bdev_freeze(struct block_device *bdev)
301{
302 int error = 0;
303
304 mutex_lock(&bdev->bd_fsfreeze_mutex);
305
306 /* A device being removed from its filesystem refuses freezes. */
307 if (!atomic_inc_unless_negative(&bdev->bd_fsfreeze_count)) {
308 mutex_unlock(&bdev->bd_fsfreeze_mutex);
309 return -EBUSY;
310 }
311 if (atomic_read(&bdev->bd_fsfreeze_count) > 1) {
312 mutex_unlock(&bdev->bd_fsfreeze_mutex);
313 return 0;
314 }
315
316 mutex_lock(&bdev->bd_holder_lock);
317 if (bdev->bd_holder_ops && bdev->bd_holder_ops->freeze) {
318 error = bdev->bd_holder_ops->freeze(bdev);
319 lockdep_assert_not_held(&bdev->bd_holder_lock);
320 } else {
321 mutex_unlock(&bdev->bd_holder_lock);
322 error = sync_blockdev(bdev);
323 }
324
325 if (error)
326 atomic_dec(&bdev->bd_fsfreeze_count);
327
328 mutex_unlock(&bdev->bd_fsfreeze_mutex);
329 return error;
330}
331EXPORT_SYMBOL(bdev_freeze);
332
333/**
334 * bdev_thaw - unlock filesystem
335 * @bdev: blockdevice to unlock
336 *
337 * Unlocks the filesystem and marks it writeable again after bdev_freeze().
338 *
339 * Return: On success zero is returned, negative error code on failure.
340 */
341int bdev_thaw(struct block_device *bdev)
342{
343 int error = -EINVAL, nr_freeze;
344
345 mutex_lock(&bdev->bd_fsfreeze_mutex);
346
347 /* <= 0: not frozen (0) or a freeze deny is held (< 0); leave it. */
348 nr_freeze = atomic_read(&bdev->bd_fsfreeze_count);
349 if (nr_freeze <= 0)
350 goto out;
351
352 error = 0;
353 if (nr_freeze > 1) {
354 atomic_dec(&bdev->bd_fsfreeze_count);
355 goto out;
356 }
357
358 /* Keep the count positive across the thaw so a deny is refused. */
359 mutex_lock(&bdev->bd_holder_lock);
360 if (bdev->bd_holder_ops && bdev->bd_holder_ops->thaw) {
361 error = bdev->bd_holder_ops->thaw(bdev);
362 lockdep_assert_not_held(&bdev->bd_holder_lock);
363 } else {
364 mutex_unlock(&bdev->bd_holder_lock);
365 }
366
367 if (!error)
368 atomic_dec(&bdev->bd_fsfreeze_count);
369out:
370 mutex_unlock(&bdev->bd_fsfreeze_mutex);
371 return error;
372}
373EXPORT_SYMBOL(bdev_thaw);
374
375/**
376 * bdev_deny_freeze - make a block device unfreezable
377 * @bdev: block device
378 *
379 * Reserve @bdev against bdev_freeze() the way deny_write_access() reserves a
380 * file against writers. bd_fsfreeze_count is sign-encoded: > 0 counts active
381 * freezes, < 0 counts deniers, so a deny succeeds only while no freeze is in
382 * progress. While held, bdev_freeze() returns -EBUSY. Pair with
383 * bdev_allow_freeze().
384 *
385 * A filesystem removing, adding or replacing a member device denies freezes on
386 * it for the duration, so a claim a freeze walk might act on is never torn down
387 * behind the freezer's back. The deny is device-scoped, not (device,
388 * superblock)-scoped: a device shared by several superblocks is refused for all
389 * of them. No in-tree filesystem removes a shared claim from a live superblock.
390 *
391 * Return: 0, or -EBUSY if the device is currently frozen.
392 */
393int bdev_deny_freeze(struct block_device *bdev)
394{
395 return atomic_dec_unless_positive(&bdev->bd_fsfreeze_count) ? 0 : -EBUSY;
396}
397EXPORT_SYMBOL_GPL(bdev_deny_freeze);
398
399/**
400 * bdev_allow_freeze - allow freezing a block device again
401 * @bdev: block device
402 *
403 * Undo one bdev_deny_freeze().
404 */
405void bdev_allow_freeze(struct block_device *bdev)
406{
407 /* A deny must be held, i.e. the count must be negative. */
408 WARN_ON_ONCE(atomic_read(&bdev->bd_fsfreeze_count) >= 0);
409 atomic_inc(&bdev->bd_fsfreeze_count);
410}
411EXPORT_SYMBOL_GPL(bdev_allow_freeze);
412
413/*
414 * pseudo-fs
415 */
416
417static __cacheline_aligned_in_smp DEFINE_MUTEX(bdev_lock);
418static struct kmem_cache *bdev_cachep __ro_after_init;
419
420static struct inode *bdev_alloc_inode(struct super_block *sb)
421{
422 struct bdev_inode *ei = alloc_inode_sb(sb, bdev_cachep, GFP_KERNEL);
423
424 if (!ei)
425 return NULL;
426 memset(&ei->bdev, 0, sizeof(ei->bdev));
427
428 if (security_bdev_alloc(&ei->bdev)) {
429 kmem_cache_free(bdev_cachep, ei);
430 return NULL;
431 }
432 return &ei->vfs_inode;
433}
434
435static void bdev_free_inode(struct inode *inode)
436{
437 struct block_device *bdev = I_BDEV(inode);
438
439 free_percpu(bdev->bd_stats);
440 kfree(bdev->bd_meta_info);
441 security_bdev_free(bdev);
442
443 if (!bdev_is_partition(bdev)) {
444 if (bdev->bd_disk && bdev->bd_disk->bdi)
445 bdi_put(bdev->bd_disk->bdi);
446 kfree(bdev->bd_disk);
447 }
448
449 if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR)
450 blk_free_ext_minor(MINOR(bdev->bd_dev));
451
452 kmem_cache_free(bdev_cachep, BDEV_I(inode));
453}
454
455static void init_once(void *data)
456{
457 struct bdev_inode *ei = data;
458
459 inode_init_once(&ei->vfs_inode);
460}
461
462static const struct super_operations bdev_sops = {
463 .statfs = simple_statfs,
464 .alloc_inode = bdev_alloc_inode,
465 .free_inode = bdev_free_inode,
466 .drop_inode = inode_just_drop,
467};
468
469static int bd_init_fs_context(struct fs_context *fc)
470{
471 struct pseudo_fs_context *ctx = init_pseudo(fc, BDEVFS_MAGIC);
472 if (!ctx)
473 return -ENOMEM;
474 fc->s_iflags |= SB_I_CGROUPWB;
475 ctx->ops = &bdev_sops;
476 return 0;
477}
478
479static struct file_system_type bd_type = {
480 .name = "bdev",
481 .init_fs_context = bd_init_fs_context,
482 .kill_sb = kill_anon_super,
483};
484
485struct super_block *blockdev_superblock __ro_after_init;
486static struct vfsmount *blockdev_mnt __ro_after_init;
487EXPORT_SYMBOL_GPL(blockdev_superblock);
488
489void __init bdev_cache_init(void)
490{
491 bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode),
492 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
493 SLAB_ACCOUNT|SLAB_PANIC),
494 init_once);
495 blockdev_mnt = kern_mount(&bd_type);
496 if (IS_ERR(blockdev_mnt))
497 panic("Cannot create bdev pseudo-fs");
498 blockdev_superblock = blockdev_mnt->mnt_sb; /* For writeback */
499}
500
501struct block_device *bdev_alloc(struct gendisk *disk, u8 partno)
502{
503 struct block_device *bdev;
504 struct inode *inode;
505
506 inode = new_inode(blockdev_superblock);
507 if (!inode)
508 return NULL;
509 inode->i_mode = S_IFBLK;
510 inode->i_rdev = 0;
511 inode->i_data.a_ops = &def_blk_aops;
512 mapping_set_gfp_mask(&inode->i_data, GFP_USER);
513
514 bdev = I_BDEV(inode);
515 mutex_init(&bdev->bd_fsfreeze_mutex);
516 spin_lock_init(&bdev->bd_size_lock);
517 mutex_init(&bdev->bd_holder_lock);
518 atomic_set(&bdev->__bd_flags, partno);
519 bdev->bd_mapping = &inode->i_data;
520 bdev->bd_queue = disk->queue;
521 if (partno && bdev_test_flag(disk->part0, BD_HAS_SUBMIT_BIO))
522 bdev_set_flag(bdev, BD_HAS_SUBMIT_BIO);
523 bdev->bd_stats = alloc_percpu(struct disk_stats);
524 if (!bdev->bd_stats) {
525 iput(inode);
526 return NULL;
527 }
528 bdev->bd_disk = disk;
529 return bdev;
530}
531
532void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors)
533{
534 spin_lock(&bdev->bd_size_lock);
535 i_size_write(BD_INODE(bdev), (loff_t)sectors << SECTOR_SHIFT);
536 bdev->bd_nr_sectors = sectors;
537 spin_unlock(&bdev->bd_size_lock);
538}
539
540void bdev_add(struct block_device *bdev, dev_t dev)
541{
542 struct inode *inode = BD_INODE(bdev);
543 if (bdev_stable_writes(bdev))
544 mapping_set_stable_writes(bdev->bd_mapping);
545 bdev->bd_dev = dev;
546 inode->i_rdev = dev;
547 inode->i_ino = dev;
548 insert_inode_hash(inode);
549}
550
551void bdev_unhash(struct block_device *bdev)
552{
553 remove_inode_hash(BD_INODE(bdev));
554}
555
556void bdev_drop(struct block_device *bdev)
557{
558 iput(BD_INODE(bdev));
559}
560
561long nr_blockdev_pages(void)
562{
563 struct inode *inode;
564 long ret = 0;
565
566 spin_lock(&blockdev_superblock->s_inode_list_lock);
567 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list)
568 ret += inode->i_mapping->nrpages;
569 spin_unlock(&blockdev_superblock->s_inode_list_lock);
570
571 return ret;
572}
573
574/**
575 * bd_may_claim - test whether a block device can be claimed
576 * @bdev: block device of interest
577 * @holder: holder trying to claim @bdev
578 * @hops: holder ops
579 *
580 * Test whether @bdev can be claimed by @holder.
581 *
582 * RETURNS:
583 * %true if @bdev can be claimed, %false otherwise.
584 */
585static bool bd_may_claim(struct block_device *bdev, void *holder,
586 const struct blk_holder_ops *hops)
587{
588 struct block_device *whole = bdev_whole(bdev);
589
590 lockdep_assert_held(&bdev_lock);
591
592 if (bdev->bd_holder) {
593 /*
594 * The same holder can always re-claim.
595 */
596 if (bdev->bd_holder == holder) {
597 if (WARN_ON_ONCE(bdev->bd_holder_ops != hops))
598 return false;
599 return true;
600 }
601 return false;
602 }
603
604 /*
605 * If the whole devices holder is set to bd_may_claim, a partition on
606 * the device is claimed, but not the whole device.
607 */
608 if (whole != bdev &&
609 whole->bd_holder && whole->bd_holder != bd_may_claim)
610 return false;
611 return true;
612}
613
614/**
615 * bd_prepare_to_claim - claim a block device
616 * @bdev: block device of interest
617 * @holder: holder trying to claim @bdev
618 * @hops: holder ops.
619 *
620 * Claim @bdev. This function fails if @bdev is already claimed by another
621 * holder and waits if another claiming is in progress. return, the caller
622 * has ownership of bd_claiming and bd_holder[s].
623 *
624 * RETURNS:
625 * 0 if @bdev can be claimed, -EBUSY otherwise.
626 */
627int bd_prepare_to_claim(struct block_device *bdev, void *holder,
628 const struct blk_holder_ops *hops)
629{
630 struct block_device *whole = bdev_whole(bdev);
631
632 if (WARN_ON_ONCE(!holder))
633 return -EINVAL;
634retry:
635 mutex_lock(&bdev_lock);
636 /* if someone else claimed, fail */
637 if (!bd_may_claim(bdev, holder, hops)) {
638 mutex_unlock(&bdev_lock);
639 return -EBUSY;
640 }
641
642 /* if claiming is already in progress, wait for it to finish */
643 if (whole->bd_claiming) {
644 wait_queue_head_t *wq = __var_waitqueue(&whole->bd_claiming);
645 DEFINE_WAIT(wait);
646
647 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
648 mutex_unlock(&bdev_lock);
649 schedule();
650 finish_wait(wq, &wait);
651 goto retry;
652 }
653
654 /* yay, all mine */
655 whole->bd_claiming = holder;
656 mutex_unlock(&bdev_lock);
657 return 0;
658}
659EXPORT_SYMBOL_GPL(bd_prepare_to_claim); /* only for the loop driver */
660
661static void bd_clear_claiming(struct block_device *whole, void *holder)
662{
663 lockdep_assert_held(&bdev_lock);
664 /* tell others that we're done */
665 BUG_ON(whole->bd_claiming != holder);
666 whole->bd_claiming = NULL;
667 wake_up_var(&whole->bd_claiming);
668}
669
670/**
671 * bd_finish_claiming - finish claiming of a block device
672 * @bdev: block device of interest
673 * @holder: holder that has claimed @bdev
674 * @hops: block device holder operations
675 *
676 * Finish exclusive open of a block device. Mark the device as exlusively
677 * open by the holder and wake up all waiters for exclusive open to finish.
678 */
679static void bd_finish_claiming(struct block_device *bdev, void *holder,
680 const struct blk_holder_ops *hops)
681{
682 struct block_device *whole = bdev_whole(bdev);
683
684 mutex_lock(&bdev_lock);
685 BUG_ON(!bd_may_claim(bdev, holder, hops));
686 /*
687 * Note that for a whole device bd_holders will be incremented twice,
688 * and bd_holder will be set to bd_may_claim before being set to holder
689 */
690 whole->bd_holders++;
691 whole->bd_holder = bd_may_claim;
692 bdev->bd_holders++;
693 mutex_lock(&bdev->bd_holder_lock);
694 bdev->bd_holder = holder;
695 bdev->bd_holder_ops = hops;
696 mutex_unlock(&bdev->bd_holder_lock);
697 bd_clear_claiming(whole, holder);
698 mutex_unlock(&bdev_lock);
699}
700
701/**
702 * bd_abort_claiming - abort claiming of a block device
703 * @bdev: block device of interest
704 * @holder: holder that has claimed @bdev
705 *
706 * Abort claiming of a block device when the exclusive open failed. This can be
707 * also used when exclusive open is not actually desired and we just needed
708 * to block other exclusive openers for a while.
709 */
710void bd_abort_claiming(struct block_device *bdev, void *holder)
711{
712 mutex_lock(&bdev_lock);
713 bd_clear_claiming(bdev_whole(bdev), holder);
714 mutex_unlock(&bdev_lock);
715}
716EXPORT_SYMBOL(bd_abort_claiming);
717
718static void bd_end_claim(struct block_device *bdev, void *holder)
719{
720 struct block_device *whole = bdev_whole(bdev);
721 bool unblock = false;
722
723 /*
724 * Release a claim on the device. The holder fields are protected with
725 * bdev_lock. open_mutex is used to synchronize disk_holder unlinking.
726 */
727 mutex_lock(&bdev_lock);
728 WARN_ON_ONCE(bdev->bd_holder != holder);
729 WARN_ON_ONCE(--bdev->bd_holders < 0);
730 WARN_ON_ONCE(--whole->bd_holders < 0);
731 if (!bdev->bd_holders) {
732 mutex_lock(&bdev->bd_holder_lock);
733 bdev->bd_holder = NULL;
734 bdev->bd_holder_ops = NULL;
735 mutex_unlock(&bdev->bd_holder_lock);
736 if (bdev_test_flag(bdev, BD_WRITE_HOLDER))
737 unblock = true;
738 }
739 if (!whole->bd_holders)
740 whole->bd_holder = NULL;
741 mutex_unlock(&bdev_lock);
742
743 /*
744 * If this was the last claim, remove holder link and unblock evpoll if
745 * it was a write holder.
746 */
747 if (unblock) {
748 disk_unblock_events(bdev->bd_disk);
749 bdev_clear_flag(bdev, BD_WRITE_HOLDER);
750 }
751}
752
753static void blkdev_flush_mapping(struct block_device *bdev)
754{
755 WARN_ON_ONCE(bdev->bd_holders);
756 sync_blockdev(bdev);
757 kill_bdev(bdev);
758 bdev_write_inode(bdev);
759}
760
761static void blkdev_put_whole(struct block_device *bdev)
762{
763 if (atomic_dec_and_test(&bdev->bd_openers))
764 blkdev_flush_mapping(bdev);
765 if (bdev->bd_disk->fops->release)
766 bdev->bd_disk->fops->release(bdev->bd_disk);
767}
768
769static int blkdev_get_whole(struct block_device *bdev, blk_mode_t mode)
770{
771 struct gendisk *disk = bdev->bd_disk;
772 int ret;
773
774 if (disk->fops->open) {
775 ret = disk->fops->open(disk, mode);
776 if (ret) {
777 /* avoid ghost partitions on a removed medium */
778 if (ret == -ENOMEDIUM &&
779 test_bit(GD_NEED_PART_SCAN, &disk->state))
780 bdev_disk_changed(disk, true);
781 return ret;
782 }
783 }
784
785 if (!atomic_read(&bdev->bd_openers))
786 set_init_blocksize(bdev);
787 atomic_inc(&bdev->bd_openers);
788 if (test_bit(GD_NEED_PART_SCAN, &disk->state)) {
789 /*
790 * Only return scanning errors if we are called from contexts
791 * that explicitly want them, e.g. the BLKRRPART ioctl.
792 */
793 ret = bdev_disk_changed(disk, false);
794 if (ret && (mode & BLK_OPEN_STRICT_SCAN)) {
795 blkdev_put_whole(bdev);
796 return ret;
797 }
798 }
799 return 0;
800}
801
802static int blkdev_get_part(struct block_device *part, blk_mode_t mode)
803{
804 struct gendisk *disk = part->bd_disk;
805 int ret;
806
807 ret = blkdev_get_whole(bdev_whole(part), mode);
808 if (ret)
809 return ret;
810
811 ret = -ENXIO;
812 if (!bdev_nr_sectors(part))
813 goto out_blkdev_put;
814
815 if (!atomic_read(&part->bd_openers)) {
816 disk->open_partitions++;
817 set_init_blocksize(part);
818 }
819 atomic_inc(&part->bd_openers);
820 return 0;
821
822out_blkdev_put:
823 blkdev_put_whole(bdev_whole(part));
824 return ret;
825}
826
827int bdev_permission(dev_t dev, blk_mode_t mode, void *holder)
828{
829 int ret;
830
831 ret = devcgroup_check_permission(DEVCG_DEV_BLOCK,
832 MAJOR(dev), MINOR(dev),
833 ((mode & BLK_OPEN_READ) ? DEVCG_ACC_READ : 0) |
834 ((mode & BLK_OPEN_WRITE) ? DEVCG_ACC_WRITE : 0));
835 if (ret)
836 return ret;
837
838 /* Blocking writes requires exclusive opener */
839 if (mode & BLK_OPEN_RESTRICT_WRITES && !holder)
840 return -EINVAL;
841
842 /*
843 * We're using error pointers to indicate to ->release() when we
844 * failed to open that block device. Also this doesn't make sense.
845 */
846 if (WARN_ON_ONCE(IS_ERR(holder)))
847 return -EINVAL;
848
849 return 0;
850}
851
852static void blkdev_put_part(struct block_device *part)
853{
854 struct block_device *whole = bdev_whole(part);
855
856 if (atomic_dec_and_test(&part->bd_openers)) {
857 blkdev_flush_mapping(part);
858 whole->bd_disk->open_partitions--;
859 }
860 blkdev_put_whole(whole);
861}
862
863struct block_device *blkdev_get_no_open(dev_t dev, bool autoload)
864{
865 struct block_device *bdev;
866 struct inode *inode;
867
868 inode = ilookup(blockdev_superblock, dev);
869 if (!inode && autoload && IS_ENABLED(CONFIG_BLOCK_LEGACY_AUTOLOAD)) {
870 blk_request_module(dev);
871 inode = ilookup(blockdev_superblock, dev);
872 if (inode)
873 pr_warn_ratelimited(
874"block device autoloading is deprecated and will be removed.\n");
875 }
876 if (!inode)
877 return NULL;
878
879 /* switch from the inode reference to a device mode one: */
880 bdev = &BDEV_I(inode)->bdev;
881 if (!kobject_get_unless_zero(&bdev->bd_device.kobj))
882 bdev = NULL;
883 iput(inode);
884 return bdev;
885}
886
887void blkdev_put_no_open(struct block_device *bdev)
888{
889 put_device(&bdev->bd_device);
890}
891
892static bool bdev_writes_blocked(struct block_device *bdev)
893{
894 return bdev->bd_writers < 0;
895}
896
897static void bdev_block_writes(struct block_device *bdev)
898{
899 bdev->bd_writers--;
900}
901
902static void bdev_unblock_writes(struct block_device *bdev)
903{
904 bdev->bd_writers++;
905}
906
907static bool bdev_may_open(struct block_device *bdev, blk_mode_t mode)
908{
909 if (bdev_allow_write_mounted)
910 return true;
911 /* Writes blocked? */
912 if (mode & BLK_OPEN_WRITE && bdev_writes_blocked(bdev))
913 return false;
914 if (mode & BLK_OPEN_RESTRICT_WRITES && bdev->bd_writers > 0)
915 return false;
916 return true;
917}
918
919static void bdev_claim_write_access(struct block_device *bdev, blk_mode_t mode)
920{
921 if (bdev_allow_write_mounted)
922 return;
923
924 /* Claim exclusive or shared write access. */
925 if (mode & BLK_OPEN_RESTRICT_WRITES)
926 bdev_block_writes(bdev);
927 else if (mode & BLK_OPEN_WRITE)
928 bdev->bd_writers++;
929}
930
931static inline bool bdev_unclaimed(const struct file *bdev_file)
932{
933 return bdev_file->private_data == BDEV_I(bdev_file->f_mapping->host);
934}
935
936static void bdev_yield_write_access(struct file *bdev_file)
937{
938 struct block_device *bdev;
939
940 if (bdev_allow_write_mounted)
941 return;
942
943 if (bdev_unclaimed(bdev_file))
944 return;
945
946 bdev = file_bdev(bdev_file);
947
948 if (bdev_file->f_mode & FMODE_WRITE_RESTRICTED)
949 bdev_unblock_writes(bdev);
950 else if (bdev_file->f_mode & FMODE_WRITE)
951 bdev->bd_writers--;
952}
953
954/**
955 * bdev_open - open a block device
956 * @bdev: block device to open
957 * @mode: open mode (BLK_OPEN_*)
958 * @holder: exclusive holder identifier
959 * @hops: holder operations
960 * @bdev_file: file for the block device
961 *
962 * Open the block device. If @holder is not %NULL, the block device is opened
963 * with exclusive access. Exclusive opens may nest for the same @holder.
964 *
965 * CONTEXT:
966 * Might sleep.
967 *
968 * RETURNS:
969 * zero on success, -errno on failure.
970 */
971int bdev_open(struct block_device *bdev, blk_mode_t mode, void *holder,
972 const struct blk_holder_ops *hops, struct file *bdev_file)
973{
974 bool unblock_events = true;
975 struct gendisk *disk = bdev->bd_disk;
976 int ret;
977
978 if (holder) {
979 mode |= BLK_OPEN_EXCL;
980 ret = bd_prepare_to_claim(bdev, holder, hops);
981 if (ret)
982 return ret;
983 } else {
984 if (WARN_ON_ONCE(mode & BLK_OPEN_EXCL))
985 return -EIO;
986 }
987
988 disk_block_events(disk);
989
990 mutex_lock(&disk->open_mutex);
991 ret = -ENXIO;
992 if (!disk_live(disk))
993 goto abort_claiming;
994 if (!try_module_get(disk->fops->owner))
995 goto abort_claiming;
996 ret = -EBUSY;
997 if (!bdev_may_open(bdev, mode))
998 goto put_module;
999 if (bdev_is_partition(bdev))
1000 ret = blkdev_get_part(bdev, mode);
1001 else
1002 ret = blkdev_get_whole(bdev, mode);
1003 if (ret)
1004 goto put_module;
1005 bdev_claim_write_access(bdev, mode);
1006 if (holder) {
1007 bd_finish_claiming(bdev, holder, hops);
1008
1009 /*
1010 * Block event polling for write claims if requested. Any write
1011 * holder makes the write_holder state stick until all are
1012 * released. This is good enough and tracking individual
1013 * writeable reference is too fragile given the way @mode is
1014 * used in blkdev_get/put().
1015 */
1016 if ((mode & BLK_OPEN_WRITE) &&
1017 !bdev_test_flag(bdev, BD_WRITE_HOLDER) &&
1018 (disk->event_flags & DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE)) {
1019 bdev_set_flag(bdev, BD_WRITE_HOLDER);
1020 unblock_events = false;
1021 }
1022 }
1023 mutex_unlock(&disk->open_mutex);
1024
1025 if (unblock_events)
1026 disk_unblock_events(disk);
1027
1028 bdev_file->f_flags |= O_LARGEFILE;
1029 bdev_file->f_mode |= FMODE_CAN_ODIRECT;
1030 if (bdev_nowait(bdev))
1031 bdev_file->f_mode |= FMODE_NOWAIT;
1032 if (mode & BLK_OPEN_RESTRICT_WRITES)
1033 bdev_file->f_mode |= FMODE_WRITE_RESTRICTED;
1034 bdev_file->f_mapping = bdev->bd_mapping;
1035 bdev_file->f_wb_err = filemap_sample_wb_err(bdev_file->f_mapping);
1036 bdev_file->private_data = holder;
1037
1038 return 0;
1039put_module:
1040 module_put(disk->fops->owner);
1041abort_claiming:
1042 if (holder)
1043 bd_abort_claiming(bdev, holder);
1044 mutex_unlock(&disk->open_mutex);
1045 disk_unblock_events(disk);
1046 return ret;
1047}
1048
1049/*
1050 * If BLK_OPEN_WRITE_IOCTL is set then this is a historical quirk
1051 * associated with the floppy driver where it has allowed ioctls if the
1052 * file was opened for writing, but does not allow reads or writes.
1053 * Make sure that this quirk is reflected in @f_flags.
1054 *
1055 * It can also happen if a block device is opened as O_RDWR | O_WRONLY.
1056 */
1057static unsigned blk_to_file_flags(blk_mode_t mode)
1058{
1059 unsigned int flags = 0;
1060
1061 if ((mode & (BLK_OPEN_READ | BLK_OPEN_WRITE)) ==
1062 (BLK_OPEN_READ | BLK_OPEN_WRITE))
1063 flags |= O_RDWR;
1064 else if (mode & BLK_OPEN_WRITE_IOCTL)
1065 flags |= O_RDWR | O_WRONLY;
1066 else if (mode & BLK_OPEN_WRITE)
1067 flags |= O_WRONLY;
1068 else if (mode & BLK_OPEN_READ)
1069 flags |= O_RDONLY; /* homeopathic, because O_RDONLY is 0 */
1070 else
1071 WARN_ON_ONCE(true);
1072
1073 if (mode & BLK_OPEN_NDELAY)
1074 flags |= O_NDELAY;
1075
1076 return flags;
1077}
1078
1079struct file *bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder,
1080 const struct blk_holder_ops *hops)
1081{
1082 struct file *bdev_file;
1083 struct block_device *bdev;
1084 unsigned int flags;
1085 int ret;
1086
1087 ret = bdev_permission(dev, mode, holder);
1088 if (ret)
1089 return ERR_PTR(ret);
1090
1091 bdev = blkdev_get_no_open(dev, true);
1092 if (!bdev)
1093 return ERR_PTR(-ENXIO);
1094
1095 flags = blk_to_file_flags(mode);
1096 bdev_file = alloc_file_pseudo_noaccount(BD_INODE(bdev),
1097 blockdev_mnt, "", flags | O_LARGEFILE, &def_blk_fops);
1098 if (IS_ERR(bdev_file)) {
1099 blkdev_put_no_open(bdev);
1100 return bdev_file;
1101 }
1102 ihold(BD_INODE(bdev));
1103
1104 ret = bdev_open(bdev, mode, holder, hops, bdev_file);
1105 if (ret) {
1106 /* We failed to open the block device. Let ->release() know. */
1107 bdev_file->private_data = ERR_PTR(ret);
1108 fput(bdev_file);
1109 return ERR_PTR(ret);
1110 }
1111 return bdev_file;
1112}
1113EXPORT_SYMBOL(bdev_file_open_by_dev);
1114
1115struct file *bdev_file_open_by_path(const char *path, blk_mode_t mode,
1116 void *holder,
1117 const struct blk_holder_ops *hops)
1118{
1119 struct file *file;
1120 dev_t dev;
1121 int error;
1122
1123 error = lookup_bdev(path, &dev);
1124 if (error)
1125 return ERR_PTR(error);
1126
1127 file = bdev_file_open_by_dev(dev, mode, holder, hops);
1128 if (!IS_ERR(file) && (mode & BLK_OPEN_WRITE)) {
1129 if (bdev_read_only(file_bdev(file))) {
1130 fput(file);
1131 file = ERR_PTR(-EACCES);
1132 }
1133 }
1134
1135 return file;
1136}
1137EXPORT_SYMBOL(bdev_file_open_by_path);
1138
1139static inline void bd_yield_claim(struct file *bdev_file)
1140{
1141 struct block_device *bdev = file_bdev(bdev_file);
1142 void *holder = bdev_file->private_data;
1143
1144 lockdep_assert_held(&bdev->bd_disk->open_mutex);
1145
1146 if (WARN_ON_ONCE(IS_ERR_OR_NULL(holder)))
1147 return;
1148
1149 if (!bdev_unclaimed(bdev_file))
1150 bd_end_claim(bdev, holder);
1151}
1152
1153void bdev_release(struct file *bdev_file)
1154{
1155 struct block_device *bdev = file_bdev(bdev_file);
1156 void *holder = bdev_file->private_data;
1157 struct gendisk *disk = bdev->bd_disk;
1158
1159 /* We failed to open that block device. */
1160 if (IS_ERR(holder))
1161 goto put_no_open;
1162
1163 /*
1164 * Sync early if it looks like we're the last one. If someone else
1165 * opens the block device between now and the decrement of bd_openers
1166 * then we did a sync that we didn't need to, but that's not the end
1167 * of the world and we want to avoid long (could be several minute)
1168 * syncs while holding the mutex.
1169 */
1170 if (atomic_read(&bdev->bd_openers) == 1)
1171 sync_blockdev(bdev);
1172
1173 mutex_lock(&disk->open_mutex);
1174 bdev_yield_write_access(bdev_file);
1175
1176 if (holder)
1177 bd_yield_claim(bdev_file);
1178
1179 /*
1180 * Trigger event checking and tell drivers to flush MEDIA_CHANGE
1181 * event. This is to ensure detection of media removal commanded
1182 * from userland - e.g. eject(1).
1183 */
1184 disk_flush_events(disk, DISK_EVENT_MEDIA_CHANGE);
1185
1186 if (bdev_is_partition(bdev))
1187 blkdev_put_part(bdev);
1188 else
1189 blkdev_put_whole(bdev);
1190 mutex_unlock(&disk->open_mutex);
1191
1192 module_put(disk->fops->owner);
1193put_no_open:
1194 blkdev_put_no_open(bdev);
1195}
1196
1197/**
1198 * bdev_yield_claim - give up the holder claim on an open block device
1199 * @bdev_file: open block device
1200 *
1201 * Yield the holder and any write access for @bdev_file without closing it, so
1202 * the caller can still act on the device - e.g. bdev_allow_freeze() it - before
1203 * the final bdev_fput(). bdev_fput() yields too, so calling it afterwards is
1204 * safe.
1205 */
1206void bdev_yield_claim(struct file *bdev_file)
1207{
1208 struct block_device *bdev;
1209 struct gendisk *disk;
1210
1211 if (!bdev_file->private_data)
1212 return;
1213
1214 bdev = file_bdev(bdev_file);
1215 disk = bdev->bd_disk;
1216
1217 mutex_lock(&disk->open_mutex);
1218 bdev_yield_write_access(bdev_file);
1219 bd_yield_claim(bdev_file);
1220 /*
1221 * Tell release we already gave up our hold on the
1222 * device and if write restrictions are available that
1223 * we already gave up write access to the device.
1224 */
1225 bdev_file->private_data = BDEV_I(bdev_file->f_mapping->host);
1226 mutex_unlock(&disk->open_mutex);
1227}
1228EXPORT_SYMBOL_GPL(bdev_yield_claim);
1229
1230/**
1231 * bdev_fput - yield claim to the block device and put the file
1232 * @bdev_file: open block device
1233 *
1234 * Yield claim on the block device and put the file. Ensure that the
1235 * block device can be reclaimed before the file is closed which is a
1236 * deferred operation.
1237 */
1238void bdev_fput(struct file *bdev_file)
1239{
1240 if (WARN_ON_ONCE(bdev_file->f_op != &def_blk_fops))
1241 return;
1242
1243 bdev_yield_claim(bdev_file);
1244 fput(bdev_file);
1245}
1246EXPORT_SYMBOL(bdev_fput);
1247
1248/**
1249 * lookup_bdev() - Look up a struct block_device by name.
1250 * @pathname: Name of the block device in the filesystem.
1251 * @dev: Pointer to the block device's dev_t, if found.
1252 *
1253 * Lookup the block device's dev_t at @pathname in the current
1254 * namespace if possible and return it in @dev.
1255 *
1256 * Context: May sleep.
1257 * Return: 0 if succeeded, negative errno otherwise.
1258 */
1259int lookup_bdev(const char *pathname, dev_t *dev)
1260{
1261 struct inode *inode;
1262 struct path path;
1263 int error;
1264
1265 if (!pathname || !*pathname)
1266 return -EINVAL;
1267
1268 error = kern_path(pathname, LOOKUP_FOLLOW, &path);
1269 if (error)
1270 return error;
1271
1272 inode = d_backing_inode(path.dentry);
1273 error = -ENOTBLK;
1274 if (!S_ISBLK(inode->i_mode))
1275 goto out_path_put;
1276 error = -EACCES;
1277 if (!may_open_dev(&path))
1278 goto out_path_put;
1279
1280 /*
1281 * Reject a block device inode with i_rdev == 0. A dev_t of 0 is
1282 * never valid for a block device: no real block device driver
1283 * registers major 0. Fake block device inodes (e.g. fuse with
1284 * rootmode=S_IFBLK) can expose i_rdev == 0, and letting that
1285 * propagate would confuse superblock lookup and trigger warnings
1286 * in the device-to-superblock table (super_dev_register).
1287 */
1288 error = -ENODEV;
1289 if (!inode->i_rdev)
1290 goto out_path_put;
1291
1292 *dev = inode->i_rdev;
1293 error = 0;
1294out_path_put:
1295 path_put(&path);
1296 return error;
1297}
1298EXPORT_SYMBOL(lookup_bdev);
1299
1300/**
1301 * bdev_mark_dead - mark a block device as dead
1302 * @bdev: block device to operate on
1303 * @surprise: indicate a surprise removal
1304 *
1305 * Tell the file system that this devices or media is dead. If @surprise is set
1306 * to %true the device or media is already gone, if not we are preparing for an
1307 * orderly removal.
1308 *
1309 * This calls into the file system, which then typicall syncs out all dirty data
1310 * and writes back inodes and then invalidates any cached data in the inodes on
1311 * the file system. In addition we also invalidate the block device mapping.
1312 */
1313void bdev_mark_dead(struct block_device *bdev, bool surprise)
1314{
1315 mutex_lock(&bdev->bd_holder_lock);
1316 if (bdev->bd_holder_ops && bdev->bd_holder_ops->mark_dead)
1317 bdev->bd_holder_ops->mark_dead(bdev, surprise);
1318 else {
1319 mutex_unlock(&bdev->bd_holder_lock);
1320 /*
1321 * On surprise removal the device is already gone; syncing is
1322 * futile and can hang forever waiting on I/O that will never
1323 * complete. Match fs_bdev_mark_dead(), which also skips it.
1324 */
1325 if (!surprise)
1326 sync_blockdev(bdev);
1327 }
1328
1329 invalidate_bdev(bdev);
1330}
1331/*
1332 * New drivers should not use this directly. There are some drivers however
1333 * that needs this for historical reasons. For example, the DASD driver has
1334 * historically had a shutdown to offline mode that doesn't actually remove the
1335 * gendisk that otherwise looks a lot like a safe device removal.
1336 */
1337EXPORT_SYMBOL_GPL(bdev_mark_dead);
1338
1339void sync_bdevs(bool wait)
1340{
1341 struct inode *inode, *old_inode = NULL;
1342
1343 spin_lock(&blockdev_superblock->s_inode_list_lock);
1344 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) {
1345 struct address_space *mapping = inode->i_mapping;
1346 struct block_device *bdev;
1347
1348 spin_lock(&inode->i_lock);
1349 if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE | I_NEW) ||
1350 mapping->nrpages == 0) {
1351 spin_unlock(&inode->i_lock);
1352 continue;
1353 }
1354 __iget(inode);
1355 spin_unlock(&inode->i_lock);
1356 spin_unlock(&blockdev_superblock->s_inode_list_lock);
1357 /*
1358 * We hold a reference to 'inode' so it couldn't have been
1359 * removed from s_inodes list while we dropped the
1360 * s_inode_list_lock We cannot iput the inode now as we can
1361 * be holding the last reference and we cannot iput it under
1362 * s_inode_list_lock. So we keep the reference and iput it
1363 * later.
1364 */
1365 iput(old_inode);
1366 old_inode = inode;
1367 bdev = I_BDEV(inode);
1368
1369 mutex_lock(&bdev->bd_disk->open_mutex);
1370 if (!atomic_read(&bdev->bd_openers)) {
1371 ; /* skip */
1372 } else if (wait) {
1373 /*
1374 * We keep the error status of individual mapping so
1375 * that applications can catch the writeback error using
1376 * fsync(2). See filemap_fdatawait_keep_errors() for
1377 * details.
1378 */
1379 filemap_fdatawait_keep_errors(inode->i_mapping);
1380 } else {
1381 filemap_fdatawrite(inode->i_mapping);
1382 }
1383 mutex_unlock(&bdev->bd_disk->open_mutex);
1384
1385 spin_lock(&blockdev_superblock->s_inode_list_lock);
1386 }
1387 spin_unlock(&blockdev_superblock->s_inode_list_lock);
1388 iput(old_inode);
1389}
1390
1391/*
1392 * Handle STATX_{DIOALIGN, WRITE_ATOMIC} for block devices.
1393 */
1394void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask)
1395{
1396 struct block_device *bdev;
1397
1398 /*
1399 * Note that d_backing_inode() returns the block device node inode, not
1400 * the block device's internal inode. Therefore it is *not* valid to
1401 * use I_BDEV() here; the block device has to be looked up by i_rdev
1402 * instead.
1403 */
1404 bdev = blkdev_get_no_open(d_backing_inode(path->dentry)->i_rdev, false);
1405 if (!bdev)
1406 return;
1407
1408 if (request_mask & STATX_DIOALIGN) {
1409 stat->dio_mem_align = bdev_dma_alignment(bdev) + 1;
1410 stat->dio_offset_align = bdev_logical_block_size(bdev);
1411 stat->result_mask |= STATX_DIOALIGN;
1412 }
1413
1414 if (request_mask & STATX_WRITE_ATOMIC && bdev_can_atomic_write(bdev)) {
1415 struct request_queue *bd_queue = bdev->bd_queue;
1416
1417 generic_fill_statx_atomic_writes(stat,
1418 queue_atomic_write_unit_min_bytes(bd_queue),
1419 queue_atomic_write_unit_max_bytes(bd_queue),
1420 0);
1421 }
1422
1423 stat->blksize = bdev_io_min(bdev);
1424
1425 blkdev_put_no_open(bdev);
1426}
1427
1428bool disk_live(struct gendisk *disk)
1429{
1430 return !inode_unhashed(BD_INODE(disk->part0));
1431}
1432EXPORT_SYMBOL_GPL(disk_live);
1433
1434unsigned int block_size(struct block_device *bdev)
1435{
1436 return 1 << BD_INODE(bdev)->i_blkbits;
1437}
1438EXPORT_SYMBOL_GPL(block_size);
1439
1440static int __init setup_bdev_allow_write_mounted(char *str)
1441{
1442 if (kstrtobool(str, &bdev_allow_write_mounted))
1443 pr_warn("Invalid option string for bdev_allow_write_mounted:"
1444 " '%s'\n", str);
1445 return 1;
1446}
1447__setup("bdev_allow_write_mounted=", setup_bdev_allow_write_mounted);