projects/linux·blob v5.19 · refs

1073 lines · 27.5 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 1991, 1992 Linus Torvalds
4 * Copyright (C) 2001 Andrea Arcangeli <[email protected]> SuSE
5 * Copyright (C) 2016 - 2020 Christoph Hellwig
6 */
7
8#include <linux/init.h>
9#include <linux/mm.h>
10#include <linux/slab.h>
11#include <linux/kmod.h>
12#include <linux/major.h>
13#include <linux/device_cgroup.h>
14#include <linux/blkdev.h>
15#include <linux/blk-integrity.h>
16#include <linux/backing-dev.h>
17#include <linux/module.h>
18#include <linux/blkpg.h>
19#include <linux/magic.h>
20#include <linux/buffer_head.h>
21#include <linux/swap.h>
22#include <linux/writeback.h>
23#include <linux/mount.h>
24#include <linux/pseudo_fs.h>
25#include <linux/uio.h>
26#include <linux/namei.h>
27#include <linux/part_stat.h>
28#include <linux/uaccess.h>
29#include "../fs/internal.h"
30#include "blk.h"
31
32struct bdev_inode {
33 struct block_device bdev;
34 struct inode vfs_inode;
35};
36
37static inline struct bdev_inode *BDEV_I(struct inode *inode)
38{
39 return container_of(inode, struct bdev_inode, vfs_inode);
40}
41
42struct block_device *I_BDEV(struct inode *inode)
43{
44 return &BDEV_I(inode)->bdev;
45}
46EXPORT_SYMBOL(I_BDEV);
47
48static void bdev_write_inode(struct block_device *bdev)
49{
50 struct inode *inode = bdev->bd_inode;
51 int ret;
52
53 spin_lock(&inode->i_lock);
54 while (inode->i_state & I_DIRTY) {
55 spin_unlock(&inode->i_lock);
56 ret = write_inode_now(inode, true);
57 if (ret) {
58 char name[BDEVNAME_SIZE];
59 pr_warn_ratelimited("VFS: Dirty inode writeback failed "
60 "for block device %s (err=%d).\n",
61 bdevname(bdev, name), ret);
62 }
63 spin_lock(&inode->i_lock);
64 }
65 spin_unlock(&inode->i_lock);
66}
67
68/* Kill _all_ buffers and pagecache , dirty or not.. */
69static void kill_bdev(struct block_device *bdev)
70{
71 struct address_space *mapping = bdev->bd_inode->i_mapping;
72
73 if (mapping_empty(mapping))
74 return;
75
76 invalidate_bh_lrus();
77 truncate_inode_pages(mapping, 0);
78}
79
80/* Invalidate clean unused buffers and pagecache. */
81void invalidate_bdev(struct block_device *bdev)
82{
83 struct address_space *mapping = bdev->bd_inode->i_mapping;
84
85 if (mapping->nrpages) {
86 invalidate_bh_lrus();
87 lru_add_drain_all(); /* make sure all lru add caches are flushed */
88 invalidate_mapping_pages(mapping, 0, -1);
89 }
90}
91EXPORT_SYMBOL(invalidate_bdev);
92
93/*
94 * Drop all buffers & page cache for given bdev range. This function bails
95 * with error if bdev has other exclusive owner (such as filesystem).
96 */
97int truncate_bdev_range(struct block_device *bdev, fmode_t mode,
98 loff_t lstart, loff_t lend)
99{
100 /*
101 * If we don't hold exclusive handle for the device, upgrade to it
102 * while we discard the buffer cache to avoid discarding buffers
103 * under live filesystem.
104 */
105 if (!(mode & FMODE_EXCL)) {
106 int err = bd_prepare_to_claim(bdev, truncate_bdev_range);
107 if (err)
108 goto invalidate;
109 }
110
111 truncate_inode_pages_range(bdev->bd_inode->i_mapping, lstart, lend);
112 if (!(mode & FMODE_EXCL))
113 bd_abort_claiming(bdev, truncate_bdev_range);
114 return 0;
115
116invalidate:
117 /*
118 * Someone else has handle exclusively open. Try invalidating instead.
119 * The 'end' argument is inclusive so the rounding is safe.
120 */
121 return invalidate_inode_pages2_range(bdev->bd_inode->i_mapping,
122 lstart >> PAGE_SHIFT,
123 lend >> PAGE_SHIFT);
124}
125
126static void set_init_blocksize(struct block_device *bdev)
127{
128 unsigned int bsize = bdev_logical_block_size(bdev);
129 loff_t size = i_size_read(bdev->bd_inode);
130
131 while (bsize < PAGE_SIZE) {
132 if (size & bsize)
133 break;
134 bsize <<= 1;
135 }
136 bdev->bd_inode->i_blkbits = blksize_bits(bsize);
137}
138
139int set_blocksize(struct block_device *bdev, int size)
140{
141 /* Size must be a power of two, and between 512 and PAGE_SIZE */
142 if (size > PAGE_SIZE || size < 512 || !is_power_of_2(size))
143 return -EINVAL;
144
145 /* Size cannot be smaller than the size supported by the device */
146 if (size < bdev_logical_block_size(bdev))
147 return -EINVAL;
148
149 /* Don't change the size if it is same as current */
150 if (bdev->bd_inode->i_blkbits != blksize_bits(size)) {
151 sync_blockdev(bdev);
152 bdev->bd_inode->i_blkbits = blksize_bits(size);
153 kill_bdev(bdev);
154 }
155 return 0;
156}
157
158EXPORT_SYMBOL(set_blocksize);
159
160int sb_set_blocksize(struct super_block *sb, int size)
161{
162 if (set_blocksize(sb->s_bdev, size))
163 return 0;
164 /* If we get here, we know size is power of two
165 * and it's value is between 512 and PAGE_SIZE */
166 sb->s_blocksize = size;
167 sb->s_blocksize_bits = blksize_bits(size);
168 return sb->s_blocksize;
169}
170
171EXPORT_SYMBOL(sb_set_blocksize);
172
173int sb_min_blocksize(struct super_block *sb, int size)
174{
175 int minsize = bdev_logical_block_size(sb->s_bdev);
176 if (size < minsize)
177 size = minsize;
178 return sb_set_blocksize(sb, size);
179}
180
181EXPORT_SYMBOL(sb_min_blocksize);
182
183int sync_blockdev_nowait(struct block_device *bdev)
184{
185 if (!bdev)
186 return 0;
187 return filemap_flush(bdev->bd_inode->i_mapping);
188}
189EXPORT_SYMBOL_GPL(sync_blockdev_nowait);
190
191/*
192 * Write out and wait upon all the dirty data associated with a block
193 * device via its mapping. Does not take the superblock lock.
194 */
195int sync_blockdev(struct block_device *bdev)
196{
197 if (!bdev)
198 return 0;
199 return filemap_write_and_wait(bdev->bd_inode->i_mapping);
200}
201EXPORT_SYMBOL(sync_blockdev);
202
203int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend)
204{
205 return filemap_write_and_wait_range(bdev->bd_inode->i_mapping,
206 lstart, lend);
207}
208EXPORT_SYMBOL(sync_blockdev_range);
209
210/*
211 * Write out and wait upon all dirty data associated with this
212 * device. Filesystem data as well as the underlying block
213 * device. Takes the superblock lock.
214 */
215int fsync_bdev(struct block_device *bdev)
216{
217 struct super_block *sb = get_super(bdev);
218 if (sb) {
219 int res = sync_filesystem(sb);
220 drop_super(sb);
221 return res;
222 }
223 return sync_blockdev(bdev);
224}
225EXPORT_SYMBOL(fsync_bdev);
226
227/**
228 * freeze_bdev -- lock a filesystem and force it into a consistent state
229 * @bdev: blockdevice to lock
230 *
231 * If a superblock is found on this device, we take the s_umount semaphore
232 * on it to make sure nobody unmounts until the snapshot creation is done.
233 * The reference counter (bd_fsfreeze_count) guarantees that only the last
234 * unfreeze process can unfreeze the frozen filesystem actually when multiple
235 * freeze requests arrive simultaneously. It counts up in freeze_bdev() and
236 * count down in thaw_bdev(). When it becomes 0, thaw_bdev() will unfreeze
237 * actually.
238 */
239int freeze_bdev(struct block_device *bdev)
240{
241 struct super_block *sb;
242 int error = 0;
243
244 mutex_lock(&bdev->bd_fsfreeze_mutex);
245 if (++bdev->bd_fsfreeze_count > 1)
246 goto done;
247
248 sb = get_active_super(bdev);
249 if (!sb)
250 goto sync;
251 if (sb->s_op->freeze_super)
252 error = sb->s_op->freeze_super(sb);
253 else
254 error = freeze_super(sb);
255 deactivate_super(sb);
256
257 if (error) {
258 bdev->bd_fsfreeze_count--;
259 goto done;
260 }
261 bdev->bd_fsfreeze_sb = sb;
262
263sync:
264 sync_blockdev(bdev);
265done:
266 mutex_unlock(&bdev->bd_fsfreeze_mutex);
267 return error;
268}
269EXPORT_SYMBOL(freeze_bdev);
270
271/**
272 * thaw_bdev -- unlock filesystem
273 * @bdev: blockdevice to unlock
274 *
275 * Unlocks the filesystem and marks it writeable again after freeze_bdev().
276 */
277int thaw_bdev(struct block_device *bdev)
278{
279 struct super_block *sb;
280 int error = -EINVAL;
281
282 mutex_lock(&bdev->bd_fsfreeze_mutex);
283 if (!bdev->bd_fsfreeze_count)
284 goto out;
285
286 error = 0;
287 if (--bdev->bd_fsfreeze_count > 0)
288 goto out;
289
290 sb = bdev->bd_fsfreeze_sb;
291 if (!sb)
292 goto out;
293
294 if (sb->s_op->thaw_super)
295 error = sb->s_op->thaw_super(sb);
296 else
297 error = thaw_super(sb);
298 if (error)
299 bdev->bd_fsfreeze_count++;
300 else
301 bdev->bd_fsfreeze_sb = NULL;
302out:
303 mutex_unlock(&bdev->bd_fsfreeze_mutex);
304 return error;
305}
306EXPORT_SYMBOL(thaw_bdev);
307
308/**
309 * bdev_read_page() - Start reading a page from a block device
310 * @bdev: The device to read the page from
311 * @sector: The offset on the device to read the page to (need not be aligned)
312 * @page: The page to read
313 *
314 * On entry, the page should be locked. It will be unlocked when the page
315 * has been read. If the block driver implements rw_page synchronously,
316 * that will be true on exit from this function, but it need not be.
317 *
318 * Errors returned by this function are usually "soft", eg out of memory, or
319 * queue full; callers should try a different route to read this page rather
320 * than propagate an error back up the stack.
321 *
322 * Return: negative errno if an error occurs, 0 if submission was successful.
323 */
324int bdev_read_page(struct block_device *bdev, sector_t sector,
325 struct page *page)
326{
327 const struct block_device_operations *ops = bdev->bd_disk->fops;
328 int result = -EOPNOTSUPP;
329
330 if (!ops->rw_page || bdev_get_integrity(bdev))
331 return result;
332
333 result = blk_queue_enter(bdev_get_queue(bdev), 0);
334 if (result)
335 return result;
336 result = ops->rw_page(bdev, sector + get_start_sect(bdev), page,
337 REQ_OP_READ);
338 blk_queue_exit(bdev_get_queue(bdev));
339 return result;
340}
341
342/**
343 * bdev_write_page() - Start writing a page to a block device
344 * @bdev: The device to write the page to
345 * @sector: The offset on the device to write the page to (need not be aligned)
346 * @page: The page to write
347 * @wbc: The writeback_control for the write
348 *
349 * On entry, the page should be locked and not currently under writeback.
350 * On exit, if the write started successfully, the page will be unlocked and
351 * under writeback. If the write failed already (eg the driver failed to
352 * queue the page to the device), the page will still be locked. If the
353 * caller is a ->writepage implementation, it will need to unlock the page.
354 *
355 * Errors returned by this function are usually "soft", eg out of memory, or
356 * queue full; callers should try a different route to write this page rather
357 * than propagate an error back up the stack.
358 *
359 * Return: negative errno if an error occurs, 0 if submission was successful.
360 */
361int bdev_write_page(struct block_device *bdev, sector_t sector,
362 struct page *page, struct writeback_control *wbc)
363{
364 int result;
365 const struct block_device_operations *ops = bdev->bd_disk->fops;
366
367 if (!ops->rw_page || bdev_get_integrity(bdev))
368 return -EOPNOTSUPP;
369 result = blk_queue_enter(bdev_get_queue(bdev), 0);
370 if (result)
371 return result;
372
373 set_page_writeback(page);
374 result = ops->rw_page(bdev, sector + get_start_sect(bdev), page,
375 REQ_OP_WRITE);
376 if (result) {
377 end_page_writeback(page);
378 } else {
379 clean_page_buffers(page);
380 unlock_page(page);
381 }
382 blk_queue_exit(bdev_get_queue(bdev));
383 return result;
384}
385
386/*
387 * pseudo-fs
388 */
389
390static __cacheline_aligned_in_smp DEFINE_SPINLOCK(bdev_lock);
391static struct kmem_cache * bdev_cachep __read_mostly;
392
393static struct inode *bdev_alloc_inode(struct super_block *sb)
394{
395 struct bdev_inode *ei = alloc_inode_sb(sb, bdev_cachep, GFP_KERNEL);
396
397 if (!ei)
398 return NULL;
399 memset(&ei->bdev, 0, sizeof(ei->bdev));
400 return &ei->vfs_inode;
401}
402
403static void bdev_free_inode(struct inode *inode)
404{
405 struct block_device *bdev = I_BDEV(inode);
406
407 free_percpu(bdev->bd_stats);
408 kfree(bdev->bd_meta_info);
409
410 if (!bdev_is_partition(bdev)) {
411 if (bdev->bd_disk && bdev->bd_disk->bdi)
412 bdi_put(bdev->bd_disk->bdi);
413 kfree(bdev->bd_disk);
414 }
415
416 if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR)
417 blk_free_ext_minor(MINOR(bdev->bd_dev));
418
419 kmem_cache_free(bdev_cachep, BDEV_I(inode));
420}
421
422static void init_once(void *data)
423{
424 struct bdev_inode *ei = data;
425
426 inode_init_once(&ei->vfs_inode);
427}
428
429static void bdev_evict_inode(struct inode *inode)
430{
431 truncate_inode_pages_final(&inode->i_data);
432 invalidate_inode_buffers(inode); /* is it needed here? */
433 clear_inode(inode);
434}
435
436static const struct super_operations bdev_sops = {
437 .statfs = simple_statfs,
438 .alloc_inode = bdev_alloc_inode,
439 .free_inode = bdev_free_inode,
440 .drop_inode = generic_delete_inode,
441 .evict_inode = bdev_evict_inode,
442};
443
444static int bd_init_fs_context(struct fs_context *fc)
445{
446 struct pseudo_fs_context *ctx = init_pseudo(fc, BDEVFS_MAGIC);
447 if (!ctx)
448 return -ENOMEM;
449 fc->s_iflags |= SB_I_CGROUPWB;
450 ctx->ops = &bdev_sops;
451 return 0;
452}
453
454static struct file_system_type bd_type = {
455 .name = "bdev",
456 .init_fs_context = bd_init_fs_context,
457 .kill_sb = kill_anon_super,
458};
459
460struct super_block *blockdev_superblock __read_mostly;
461EXPORT_SYMBOL_GPL(blockdev_superblock);
462
463void __init bdev_cache_init(void)
464{
465 int err;
466 static struct vfsmount *bd_mnt;
467
468 bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode),
469 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
470 SLAB_MEM_SPREAD|SLAB_ACCOUNT|SLAB_PANIC),
471 init_once);
472 err = register_filesystem(&bd_type);
473 if (err)
474 panic("Cannot register bdev pseudo-fs");
475 bd_mnt = kern_mount(&bd_type);
476 if (IS_ERR(bd_mnt))
477 panic("Cannot create bdev pseudo-fs");
478 blockdev_superblock = bd_mnt->mnt_sb; /* For writeback */
479}
480
481struct block_device *bdev_alloc(struct gendisk *disk, u8 partno)
482{
483 struct block_device *bdev;
484 struct inode *inode;
485
486 inode = new_inode(blockdev_superblock);
487 if (!inode)
488 return NULL;
489 inode->i_mode = S_IFBLK;
490 inode->i_rdev = 0;
491 inode->i_data.a_ops = &def_blk_aops;
492 mapping_set_gfp_mask(&inode->i_data, GFP_USER);
493
494 bdev = I_BDEV(inode);
495 mutex_init(&bdev->bd_fsfreeze_mutex);
496 spin_lock_init(&bdev->bd_size_lock);
497 bdev->bd_partno = partno;
498 bdev->bd_inode = inode;
499 bdev->bd_queue = disk->queue;
500 bdev->bd_stats = alloc_percpu(struct disk_stats);
501 if (!bdev->bd_stats) {
502 iput(inode);
503 return NULL;
504 }
505 bdev->bd_disk = disk;
506 return bdev;
507}
508
509void bdev_add(struct block_device *bdev, dev_t dev)
510{
511 bdev->bd_dev = dev;
512 bdev->bd_inode->i_rdev = dev;
513 bdev->bd_inode->i_ino = dev;
514 insert_inode_hash(bdev->bd_inode);
515}
516
517long nr_blockdev_pages(void)
518{
519 struct inode *inode;
520 long ret = 0;
521
522 spin_lock(&blockdev_superblock->s_inode_list_lock);
523 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list)
524 ret += inode->i_mapping->nrpages;
525 spin_unlock(&blockdev_superblock->s_inode_list_lock);
526
527 return ret;
528}
529
530/**
531 * bd_may_claim - test whether a block device can be claimed
532 * @bdev: block device of interest
533 * @whole: whole block device containing @bdev, may equal @bdev
534 * @holder: holder trying to claim @bdev
535 *
536 * Test whether @bdev can be claimed by @holder.
537 *
538 * CONTEXT:
539 * spin_lock(&bdev_lock).
540 *
541 * RETURNS:
542 * %true if @bdev can be claimed, %false otherwise.
543 */
544static bool bd_may_claim(struct block_device *bdev, struct block_device *whole,
545 void *holder)
546{
547 if (bdev->bd_holder == holder)
548 return true; /* already a holder */
549 else if (bdev->bd_holder != NULL)
550 return false; /* held by someone else */
551 else if (whole == bdev)
552 return true; /* is a whole device which isn't held */
553
554 else if (whole->bd_holder == bd_may_claim)
555 return true; /* is a partition of a device that is being partitioned */
556 else if (whole->bd_holder != NULL)
557 return false; /* is a partition of a held device */
558 else
559 return true; /* is a partition of an un-held device */
560}
561
562/**
563 * bd_prepare_to_claim - claim a block device
564 * @bdev: block device of interest
565 * @holder: holder trying to claim @bdev
566 *
567 * Claim @bdev. This function fails if @bdev is already claimed by another
568 * holder and waits if another claiming is in progress. return, the caller
569 * has ownership of bd_claiming and bd_holder[s].
570 *
571 * RETURNS:
572 * 0 if @bdev can be claimed, -EBUSY otherwise.
573 */
574int bd_prepare_to_claim(struct block_device *bdev, void *holder)
575{
576 struct block_device *whole = bdev_whole(bdev);
577
578 if (WARN_ON_ONCE(!holder))
579 return -EINVAL;
580retry:
581 spin_lock(&bdev_lock);
582 /* if someone else claimed, fail */
583 if (!bd_may_claim(bdev, whole, holder)) {
584 spin_unlock(&bdev_lock);
585 return -EBUSY;
586 }
587
588 /* if claiming is already in progress, wait for it to finish */
589 if (whole->bd_claiming) {
590 wait_queue_head_t *wq = bit_waitqueue(&whole->bd_claiming, 0);
591 DEFINE_WAIT(wait);
592
593 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
594 spin_unlock(&bdev_lock);
595 schedule();
596 finish_wait(wq, &wait);
597 goto retry;
598 }
599
600 /* yay, all mine */
601 whole->bd_claiming = holder;
602 spin_unlock(&bdev_lock);
603 return 0;
604}
605EXPORT_SYMBOL_GPL(bd_prepare_to_claim); /* only for the loop driver */
606
607static void bd_clear_claiming(struct block_device *whole, void *holder)
608{
609 lockdep_assert_held(&bdev_lock);
610 /* tell others that we're done */
611 BUG_ON(whole->bd_claiming != holder);
612 whole->bd_claiming = NULL;
613 wake_up_bit(&whole->bd_claiming, 0);
614}
615
616/**
617 * bd_finish_claiming - finish claiming of a block device
618 * @bdev: block device of interest
619 * @holder: holder that has claimed @bdev
620 *
621 * Finish exclusive open of a block device. Mark the device as exlusively
622 * open by the holder and wake up all waiters for exclusive open to finish.
623 */
624static void bd_finish_claiming(struct block_device *bdev, void *holder)
625{
626 struct block_device *whole = bdev_whole(bdev);
627
628 spin_lock(&bdev_lock);
629 BUG_ON(!bd_may_claim(bdev, whole, holder));
630 /*
631 * Note that for a whole device bd_holders will be incremented twice,
632 * and bd_holder will be set to bd_may_claim before being set to holder
633 */
634 whole->bd_holders++;
635 whole->bd_holder = bd_may_claim;
636 bdev->bd_holders++;
637 bdev->bd_holder = holder;
638 bd_clear_claiming(whole, holder);
639 spin_unlock(&bdev_lock);
640}
641
642/**
643 * bd_abort_claiming - abort claiming of a block device
644 * @bdev: block device of interest
645 * @holder: holder that has claimed @bdev
646 *
647 * Abort claiming of a block device when the exclusive open failed. This can be
648 * also used when exclusive open is not actually desired and we just needed
649 * to block other exclusive openers for a while.
650 */
651void bd_abort_claiming(struct block_device *bdev, void *holder)
652{
653 spin_lock(&bdev_lock);
654 bd_clear_claiming(bdev_whole(bdev), holder);
655 spin_unlock(&bdev_lock);
656}
657EXPORT_SYMBOL(bd_abort_claiming);
658
659static void blkdev_flush_mapping(struct block_device *bdev)
660{
661 WARN_ON_ONCE(bdev->bd_holders);
662 sync_blockdev(bdev);
663 kill_bdev(bdev);
664 bdev_write_inode(bdev);
665}
666
667static int blkdev_get_whole(struct block_device *bdev, fmode_t mode)
668{
669 struct gendisk *disk = bdev->bd_disk;
670 int ret;
671
672 if (disk->fops->open) {
673 ret = disk->fops->open(bdev, mode);
674 if (ret) {
675 /* avoid ghost partitions on a removed medium */
676 if (ret == -ENOMEDIUM &&
677 test_bit(GD_NEED_PART_SCAN, &disk->state))
678 bdev_disk_changed(disk, true);
679 return ret;
680 }
681 }
682
683 if (!atomic_read(&bdev->bd_openers))
684 set_init_blocksize(bdev);
685 if (test_bit(GD_NEED_PART_SCAN, &disk->state))
686 bdev_disk_changed(disk, false);
687 atomic_inc(&bdev->bd_openers);
688 return 0;
689}
690
691static void blkdev_put_whole(struct block_device *bdev, fmode_t mode)
692{
693 if (atomic_dec_and_test(&bdev->bd_openers))
694 blkdev_flush_mapping(bdev);
695 if (bdev->bd_disk->fops->release)
696 bdev->bd_disk->fops->release(bdev->bd_disk, mode);
697}
698
699static int blkdev_get_part(struct block_device *part, fmode_t mode)
700{
701 struct gendisk *disk = part->bd_disk;
702 int ret;
703
704 if (atomic_read(&part->bd_openers))
705 goto done;
706
707 ret = blkdev_get_whole(bdev_whole(part), mode);
708 if (ret)
709 return ret;
710
711 ret = -ENXIO;
712 if (!bdev_nr_sectors(part))
713 goto out_blkdev_put;
714
715 disk->open_partitions++;
716 set_init_blocksize(part);
717done:
718 atomic_inc(&part->bd_openers);
719 return 0;
720
721out_blkdev_put:
722 blkdev_put_whole(bdev_whole(part), mode);
723 return ret;
724}
725
726static void blkdev_put_part(struct block_device *part, fmode_t mode)
727{
728 struct block_device *whole = bdev_whole(part);
729
730 if (!atomic_dec_and_test(&part->bd_openers))
731 return;
732 blkdev_flush_mapping(part);
733 whole->bd_disk->open_partitions--;
734 blkdev_put_whole(whole, mode);
735}
736
737struct block_device *blkdev_get_no_open(dev_t dev)
738{
739 struct block_device *bdev;
740 struct inode *inode;
741
742 inode = ilookup(blockdev_superblock, dev);
743 if (!inode && IS_ENABLED(CONFIG_BLOCK_LEGACY_AUTOLOAD)) {
744 blk_request_module(dev);
745 inode = ilookup(blockdev_superblock, dev);
746 if (inode)
747 pr_warn_ratelimited(
748"block device autoloading is deprecated and will be removed.\n");
749 }
750 if (!inode)
751 return NULL;
752
753 /* switch from the inode reference to a device mode one: */
754 bdev = &BDEV_I(inode)->bdev;
755 if (!kobject_get_unless_zero(&bdev->bd_device.kobj))
756 bdev = NULL;
757 iput(inode);
758 return bdev;
759}
760
761void blkdev_put_no_open(struct block_device *bdev)
762{
763 put_device(&bdev->bd_device);
764}
765
766/**
767 * blkdev_get_by_dev - open a block device by device number
768 * @dev: device number of block device to open
769 * @mode: FMODE_* mask
770 * @holder: exclusive holder identifier
771 *
772 * Open the block device described by device number @dev. If @mode includes
773 * %FMODE_EXCL, the block device is opened with exclusive access. Specifying
774 * %FMODE_EXCL with a %NULL @holder is invalid. Exclusive opens may nest for
775 * the same @holder.
776 *
777 * Use this interface ONLY if you really do not have anything better - i.e. when
778 * you are behind a truly sucky interface and all you are given is a device
779 * number. Everything else should use blkdev_get_by_path().
780 *
781 * CONTEXT:
782 * Might sleep.
783 *
784 * RETURNS:
785 * Reference to the block_device on success, ERR_PTR(-errno) on failure.
786 */
787struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder)
788{
789 bool unblock_events = true;
790 struct block_device *bdev;
791 struct gendisk *disk;
792 int ret;
793
794 ret = devcgroup_check_permission(DEVCG_DEV_BLOCK,
795 MAJOR(dev), MINOR(dev),
796 ((mode & FMODE_READ) ? DEVCG_ACC_READ : 0) |
797 ((mode & FMODE_WRITE) ? DEVCG_ACC_WRITE : 0));
798 if (ret)
799 return ERR_PTR(ret);
800
801 bdev = blkdev_get_no_open(dev);
802 if (!bdev)
803 return ERR_PTR(-ENXIO);
804 disk = bdev->bd_disk;
805
806 if (mode & FMODE_EXCL) {
807 ret = bd_prepare_to_claim(bdev, holder);
808 if (ret)
809 goto put_blkdev;
810 }
811
812 disk_block_events(disk);
813
814 mutex_lock(&disk->open_mutex);
815 ret = -ENXIO;
816 if (!disk_live(disk))
817 goto abort_claiming;
818 if (!try_module_get(disk->fops->owner))
819 goto abort_claiming;
820 if (bdev_is_partition(bdev))
821 ret = blkdev_get_part(bdev, mode);
822 else
823 ret = blkdev_get_whole(bdev, mode);
824 if (ret)
825 goto put_module;
826 if (mode & FMODE_EXCL) {
827 bd_finish_claiming(bdev, holder);
828
829 /*
830 * Block event polling for write claims if requested. Any write
831 * holder makes the write_holder state stick until all are
832 * released. This is good enough and tracking individual
833 * writeable reference is too fragile given the way @mode is
834 * used in blkdev_get/put().
835 */
836 if ((mode & FMODE_WRITE) && !bdev->bd_write_holder &&
837 (disk->event_flags & DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE)) {
838 bdev->bd_write_holder = true;
839 unblock_events = false;
840 }
841 }
842 mutex_unlock(&disk->open_mutex);
843
844 if (unblock_events)
845 disk_unblock_events(disk);
846 return bdev;
847put_module:
848 module_put(disk->fops->owner);
849abort_claiming:
850 if (mode & FMODE_EXCL)
851 bd_abort_claiming(bdev, holder);
852 mutex_unlock(&disk->open_mutex);
853 disk_unblock_events(disk);
854put_blkdev:
855 blkdev_put_no_open(bdev);
856 return ERR_PTR(ret);
857}
858EXPORT_SYMBOL(blkdev_get_by_dev);
859
860/**
861 * blkdev_get_by_path - open a block device by name
862 * @path: path to the block device to open
863 * @mode: FMODE_* mask
864 * @holder: exclusive holder identifier
865 *
866 * Open the block device described by the device file at @path. If @mode
867 * includes %FMODE_EXCL, the block device is opened with exclusive access.
868 * Specifying %FMODE_EXCL with a %NULL @holder is invalid. Exclusive opens may
869 * nest for the same @holder.
870 *
871 * CONTEXT:
872 * Might sleep.
873 *
874 * RETURNS:
875 * Reference to the block_device on success, ERR_PTR(-errno) on failure.
876 */
877struct block_device *blkdev_get_by_path(const char *path, fmode_t mode,
878 void *holder)
879{
880 struct block_device *bdev;
881 dev_t dev;
882 int error;
883
884 error = lookup_bdev(path, &dev);
885 if (error)
886 return ERR_PTR(error);
887
888 bdev = blkdev_get_by_dev(dev, mode, holder);
889 if (!IS_ERR(bdev) && (mode & FMODE_WRITE) && bdev_read_only(bdev)) {
890 blkdev_put(bdev, mode);
891 return ERR_PTR(-EACCES);
892 }
893
894 return bdev;
895}
896EXPORT_SYMBOL(blkdev_get_by_path);
897
898void blkdev_put(struct block_device *bdev, fmode_t mode)
899{
900 struct gendisk *disk = bdev->bd_disk;
901
902 /*
903 * Sync early if it looks like we're the last one. If someone else
904 * opens the block device between now and the decrement of bd_openers
905 * then we did a sync that we didn't need to, but that's not the end
906 * of the world and we want to avoid long (could be several minute)
907 * syncs while holding the mutex.
908 */
909 if (atomic_read(&bdev->bd_openers) == 1)
910 sync_blockdev(bdev);
911
912 mutex_lock(&disk->open_mutex);
913 if (mode & FMODE_EXCL) {
914 struct block_device *whole = bdev_whole(bdev);
915 bool bdev_free;
916
917 /*
918 * Release a claim on the device. The holder fields
919 * are protected with bdev_lock. open_mutex is to
920 * synchronize disk_holder unlinking.
921 */
922 spin_lock(&bdev_lock);
923
924 WARN_ON_ONCE(--bdev->bd_holders < 0);
925 WARN_ON_ONCE(--whole->bd_holders < 0);
926
927 if ((bdev_free = !bdev->bd_holders))
928 bdev->bd_holder = NULL;
929 if (!whole->bd_holders)
930 whole->bd_holder = NULL;
931
932 spin_unlock(&bdev_lock);
933
934 /*
935 * If this was the last claim, remove holder link and
936 * unblock evpoll if it was a write holder.
937 */
938 if (bdev_free && bdev->bd_write_holder) {
939 disk_unblock_events(disk);
940 bdev->bd_write_holder = false;
941 }
942 }
943
944 /*
945 * Trigger event checking and tell drivers to flush MEDIA_CHANGE
946 * event. This is to ensure detection of media removal commanded
947 * from userland - e.g. eject(1).
948 */
949 disk_flush_events(disk, DISK_EVENT_MEDIA_CHANGE);
950
951 if (bdev_is_partition(bdev))
952 blkdev_put_part(bdev, mode);
953 else
954 blkdev_put_whole(bdev, mode);
955 mutex_unlock(&disk->open_mutex);
956
957 module_put(disk->fops->owner);
958 blkdev_put_no_open(bdev);
959}
960EXPORT_SYMBOL(blkdev_put);
961
962/**
963 * lookup_bdev() - Look up a struct block_device by name.
964 * @pathname: Name of the block device in the filesystem.
965 * @dev: Pointer to the block device's dev_t, if found.
966 *
967 * Lookup the block device's dev_t at @pathname in the current
968 * namespace if possible and return it in @dev.
969 *
970 * Context: May sleep.
971 * Return: 0 if succeeded, negative errno otherwise.
972 */
973int lookup_bdev(const char *pathname, dev_t *dev)
974{
975 struct inode *inode;
976 struct path path;
977 int error;
978
979 if (!pathname || !*pathname)
980 return -EINVAL;
981
982 error = kern_path(pathname, LOOKUP_FOLLOW, &path);
983 if (error)
984 return error;
985
986 inode = d_backing_inode(path.dentry);
987 error = -ENOTBLK;
988 if (!S_ISBLK(inode->i_mode))
989 goto out_path_put;
990 error = -EACCES;
991 if (!may_open_dev(&path))
992 goto out_path_put;
993
994 *dev = inode->i_rdev;
995 error = 0;
996out_path_put:
997 path_put(&path);
998 return error;
999}
1000EXPORT_SYMBOL(lookup_bdev);
1001
1002int __invalidate_device(struct block_device *bdev, bool kill_dirty)
1003{
1004 struct super_block *sb = get_super(bdev);
1005 int res = 0;
1006
1007 if (sb) {
1008 /*
1009 * no need to lock the super, get_super holds the
1010 * read mutex so the filesystem cannot go away
1011 * under us (->put_super runs with the write lock
1012 * hold).
1013 */
1014 shrink_dcache_sb(sb);
1015 res = invalidate_inodes(sb, kill_dirty);
1016 drop_super(sb);
1017 }
1018 invalidate_bdev(bdev);
1019 return res;
1020}
1021EXPORT_SYMBOL(__invalidate_device);
1022
1023void sync_bdevs(bool wait)
1024{
1025 struct inode *inode, *old_inode = NULL;
1026
1027 spin_lock(&blockdev_superblock->s_inode_list_lock);
1028 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) {
1029 struct address_space *mapping = inode->i_mapping;
1030 struct block_device *bdev;
1031
1032 spin_lock(&inode->i_lock);
1033 if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) ||
1034 mapping->nrpages == 0) {
1035 spin_unlock(&inode->i_lock);
1036 continue;
1037 }
1038 __iget(inode);
1039 spin_unlock(&inode->i_lock);
1040 spin_unlock(&blockdev_superblock->s_inode_list_lock);
1041 /*
1042 * We hold a reference to 'inode' so it couldn't have been
1043 * removed from s_inodes list while we dropped the
1044 * s_inode_list_lock We cannot iput the inode now as we can
1045 * be holding the last reference and we cannot iput it under
1046 * s_inode_list_lock. So we keep the reference and iput it
1047 * later.
1048 */
1049 iput(old_inode);
1050 old_inode = inode;
1051 bdev = I_BDEV(inode);
1052
1053 mutex_lock(&bdev->bd_disk->open_mutex);
1054 if (!atomic_read(&bdev->bd_openers)) {
1055 ; /* skip */
1056 } else if (wait) {
1057 /*
1058 * We keep the error status of individual mapping so
1059 * that applications can catch the writeback error using
1060 * fsync(2). See filemap_fdatawait_keep_errors() for
1061 * details.
1062 */
1063 filemap_fdatawait_keep_errors(inode->i_mapping);
1064 } else {
1065 filemap_fdatawrite(inode->i_mapping);
1066 }
1067 mutex_unlock(&bdev->bd_disk->open_mutex);
1068
1069 spin_lock(&blockdev_superblock->s_inode_list_lock);
1070 }
1071 spin_unlock(&blockdev_superblock->s_inode_list_lock);
1072 iput(old_inode);
1073}