projects/linux·blob v7.2 · refs

2325 lines · 58.3 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright 1993 by Theodore Ts'o.
4 */
5#include <linux/module.h>
6#include <linux/moduleparam.h>
7#include <linux/sched.h>
8#include <linux/fs.h>
9#include <linux/pagemap.h>
10#include <linux/file.h>
11#include <linux/stat.h>
12#include <linux/errno.h>
13#include <linux/major.h>
14#include <linux/wait.h>
15#include <linux/blkpg.h>
16#include <linux/init.h>
17#include <linux/swap.h>
18#include <linux/slab.h>
19#include <linux/compat.h>
20#include <linux/suspend.h>
21#include <linux/freezer.h>
22#include <linux/mutex.h>
23#include <linux/writeback.h>
24#include <linux/completion.h>
25#include <linux/highmem.h>
26#include <linux/splice.h>
27#include <linux/sysfs.h>
28#include <linux/miscdevice.h>
29#include <linux/falloc.h>
30#include <linux/uio.h>
31#include <linux/ioprio.h>
32#include <linux/blk-cgroup.h>
33#include <linux/sched/mm.h>
34#include <linux/statfs.h>
35#include <linux/uaccess.h>
36#include <linux/blk-mq.h>
37#include <linux/spinlock.h>
38#include <uapi/linux/loop.h>
39
40/* Possible states of device */
41enum {
42 Lo_unbound,
43 Lo_bound,
44 Lo_rundown,
45 Lo_deleting,
46};
47
48struct loop_device {
49 int lo_number;
50 loff_t lo_offset;
51 loff_t lo_sizelimit;
52 int lo_flags;
53 char lo_file_name[LO_NAME_SIZE];
54
55 struct file *lo_backing_file;
56 unsigned int lo_min_dio_size;
57 struct block_device *lo_device;
58
59 gfp_t old_gfp_mask;
60
61 spinlock_t lo_lock;
62 int lo_state;
63 spinlock_t lo_work_lock;
64 struct workqueue_struct *workqueue;
65 struct work_struct rootcg_work;
66 struct list_head rootcg_cmd_list;
67 struct list_head idle_worker_list;
68 struct rb_root worker_tree;
69 struct timer_list timer;
70 bool sysfs_inited;
71
72 struct request_queue *lo_queue;
73 struct blk_mq_tag_set tag_set;
74 struct gendisk *lo_disk;
75 struct mutex lo_mutex;
76 bool idr_visible;
77};
78
79struct loop_cmd {
80 struct list_head list_entry;
81 bool use_aio; /* use AIO interface to handle I/O */
82 atomic_t ref; /* only for aio */
83 long ret;
84 struct kiocb iocb;
85 struct bio_vec *bvec;
86 struct cgroup_subsys_state *blkcg_css;
87 struct cgroup_subsys_state *memcg_css;
88};
89
90#define LOOP_IDLE_WORKER_TIMEOUT (60 * HZ)
91#define LOOP_DEFAULT_HW_Q_DEPTH 128
92
93static DEFINE_IDR(loop_index_idr);
94static DEFINE_MUTEX(loop_ctl_mutex);
95static DEFINE_MUTEX(loop_validate_mutex);
96
97/**
98 * loop_global_lock_killable() - take locks for safe loop_validate_file() test
99 *
100 * @lo: struct loop_device
101 * @global: true if @lo is about to bind another "struct loop_device", false otherwise
102 *
103 * Returns 0 on success, -EINTR otherwise.
104 *
105 * Since loop_validate_file() traverses on other "struct loop_device" if
106 * is_loop_device() is true, we need a global lock for serializing concurrent
107 * loop_configure()/loop_change_fd()/__loop_clr_fd() calls.
108 */
109static int loop_global_lock_killable(struct loop_device *lo, bool global)
110{
111 int err;
112
113 if (global) {
114 err = mutex_lock_killable(&loop_validate_mutex);
115 if (err)
116 return err;
117 }
118 err = mutex_lock_killable(&lo->lo_mutex);
119 if (err && global)
120 mutex_unlock(&loop_validate_mutex);
121 return err;
122}
123
124/**
125 * loop_global_unlock() - release locks taken by loop_global_lock_killable()
126 *
127 * @lo: struct loop_device
128 * @global: true if @lo was about to bind another "struct loop_device", false otherwise
129 */
130static void loop_global_unlock(struct loop_device *lo, bool global)
131{
132 mutex_unlock(&lo->lo_mutex);
133 if (global)
134 mutex_unlock(&loop_validate_mutex);
135}
136
137static int max_part;
138static int part_shift;
139
140static loff_t lo_calculate_size(struct loop_device *lo, struct file *file)
141{
142 loff_t loopsize;
143 int ret;
144
145 if (S_ISBLK(file_inode(file)->i_mode)) {
146 loopsize = i_size_read(file->f_mapping->host);
147 } else {
148 struct kstat stat;
149
150 /*
151 * Get the accurate file size. This provides better results than
152 * cached inode data, particularly for network filesystems where
153 * metadata may be stale.
154 */
155 ret = vfs_getattr_nosec(&file->f_path, &stat, STATX_SIZE, 0);
156 if (ret)
157 return 0;
158
159 loopsize = stat.size;
160 }
161
162 if (lo->lo_offset > 0)
163 loopsize -= lo->lo_offset;
164 /* offset is beyond i_size, weird but possible */
165 if (loopsize < 0)
166 return 0;
167 if (lo->lo_sizelimit > 0 && lo->lo_sizelimit < loopsize)
168 loopsize = lo->lo_sizelimit;
169 /*
170 * Unfortunately, if we want to do I/O on the device,
171 * the number of 512-byte sectors has to fit into a sector_t.
172 */
173 return loopsize >> 9;
174}
175
176/*
177 * We support direct I/O only if lo_offset is aligned with the logical I/O size
178 * of backing device, and the logical block size of loop is bigger than that of
179 * the backing device.
180 */
181static bool lo_can_use_dio(struct loop_device *lo)
182{
183 if (!(lo->lo_backing_file->f_mode & FMODE_CAN_ODIRECT))
184 return false;
185 if (queue_logical_block_size(lo->lo_queue) < lo->lo_min_dio_size)
186 return false;
187 if (lo->lo_offset & (lo->lo_min_dio_size - 1))
188 return false;
189 return true;
190}
191
192/*
193 * Direct I/O can be enabled either by using an O_DIRECT file descriptor, or by
194 * passing in the LO_FLAGS_DIRECT_IO flag from userspace. It will be silently
195 * disabled when the device block size is too small or the offset is unaligned.
196 *
197 * loop_get_status will always report the effective LO_FLAGS_DIRECT_IO flag and
198 * not the originally passed in one.
199 */
200static inline void loop_update_dio(struct loop_device *lo)
201{
202 lockdep_assert_held(&lo->lo_mutex);
203 WARN_ON_ONCE(lo->lo_state == Lo_bound &&
204 lo->lo_queue->mq_freeze_depth == 0);
205
206 if ((lo->lo_flags & LO_FLAGS_DIRECT_IO) && !lo_can_use_dio(lo))
207 lo->lo_flags &= ~LO_FLAGS_DIRECT_IO;
208}
209
210/**
211 * loop_set_size() - sets device size and notifies userspace
212 * @lo: struct loop_device to set the size for
213 * @size: new size of the loop device
214 *
215 * Callers must validate that the size passed into this function fits into
216 * a sector_t, eg using loop_validate_size()
217 */
218static void loop_set_size(struct loop_device *lo, loff_t size)
219{
220 if (!set_capacity_and_notify(lo->lo_disk, size))
221 kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE);
222}
223
224static void loop_clear_limits(struct loop_device *lo, int mode)
225{
226 struct queue_limits lim = queue_limits_start_update(lo->lo_queue);
227
228 if (mode & FALLOC_FL_ZERO_RANGE)
229 lim.max_write_zeroes_sectors = 0;
230
231 if (mode & FALLOC_FL_PUNCH_HOLE) {
232 lim.max_hw_discard_sectors = 0;
233 lim.discard_granularity = 0;
234 }
235
236 /*
237 * XXX: this updates the queue limits without freezing the queue, which
238 * is against the locking protocol and dangerous. But we can't just
239 * freeze the queue as we're inside the ->queue_rq method here. So this
240 * should move out into a workqueue unless we get the file operations to
241 * advertise if they support specific fallocate operations.
242 */
243 queue_limits_commit_update(lo->lo_queue, &lim);
244}
245
246static int lo_fallocate(struct loop_device *lo, struct request *rq, loff_t pos,
247 int mode)
248{
249 /*
250 * We use fallocate to manipulate the space mappings used by the image
251 * a.k.a. discard/zerorange.
252 */
253 struct file *file = lo->lo_backing_file;
254 int ret;
255
256 mode |= FALLOC_FL_KEEP_SIZE;
257
258 if (!bdev_max_discard_sectors(lo->lo_device))
259 return -EOPNOTSUPP;
260
261 ret = file->f_op->fallocate(file, mode, pos, blk_rq_bytes(rq));
262 if (unlikely(ret && ret != -EINVAL && ret != -EOPNOTSUPP))
263 return -EIO;
264
265 /*
266 * We initially configure the limits in a hope that fallocate is
267 * supported and clear them here if that turns out not to be true.
268 */
269 if (unlikely(ret == -EOPNOTSUPP))
270 loop_clear_limits(lo, mode);
271
272 return ret;
273}
274
275static int lo_req_flush(struct loop_device *lo, struct request *rq)
276{
277 int ret = vfs_fsync(lo->lo_backing_file, 0);
278 if (unlikely(ret && ret != -EINVAL))
279 ret = -EIO;
280
281 return ret;
282}
283
284static void lo_complete_rq(struct request *rq)
285{
286 struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
287 blk_status_t ret = BLK_STS_OK;
288
289 if (cmd->ret < 0 || cmd->ret == blk_rq_bytes(rq) ||
290 req_op(rq) != REQ_OP_READ) {
291 if (cmd->ret < 0)
292 ret = errno_to_blk_status(cmd->ret);
293 goto end_io;
294 }
295
296 /*
297 * Short READ - if we got some data, advance our request and
298 * retry it. If we got no data, end the rest with EIO.
299 */
300 if (cmd->ret) {
301 blk_update_request(rq, BLK_STS_OK, cmd->ret);
302 cmd->ret = 0;
303 blk_mq_requeue_request(rq, true);
304 } else {
305 struct bio *bio = rq->bio;
306
307 while (bio) {
308 zero_fill_bio(bio);
309 bio = bio->bi_next;
310 }
311
312 ret = BLK_STS_IOERR;
313end_io:
314 blk_mq_end_request(rq, ret);
315 }
316}
317
318static void lo_rw_aio_do_completion(struct loop_cmd *cmd)
319{
320 struct request *rq = blk_mq_rq_from_pdu(cmd);
321
322 if (!atomic_dec_and_test(&cmd->ref))
323 return;
324 kfree(cmd->bvec);
325 cmd->bvec = NULL;
326 if (req_op(rq) == REQ_OP_WRITE)
327 kiocb_end_write(&cmd->iocb);
328 if (likely(!blk_should_fake_timeout(rq->q)))
329 blk_mq_complete_request(rq);
330}
331
332static void lo_rw_aio_complete(struct kiocb *iocb, long ret)
333{
334 struct loop_cmd *cmd = container_of(iocb, struct loop_cmd, iocb);
335
336 cmd->ret = ret;
337 lo_rw_aio_do_completion(cmd);
338}
339
340static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd,
341 loff_t pos, int rw)
342{
343 struct iov_iter iter;
344 struct req_iterator rq_iter;
345 struct request *rq = blk_mq_rq_from_pdu(cmd);
346 struct file *file = lo->lo_backing_file;
347 unsigned int nr_bvec;
348 int ret;
349
350 nr_bvec = blk_rq_nr_bvec(rq);
351
352 if (rq->bio != rq->biotail) {
353 struct bio_vec tmp, *bvec;
354
355 cmd->bvec = kmalloc_objs(*cmd->bvec, nr_bvec, GFP_NOIO);
356 if (!cmd->bvec)
357 return -EIO;
358
359 /*
360 * The bios of the request may be started from the middle of
361 * the 'bvec' because of bio splitting, so we can't directly
362 * copy bio->bi_iov_vec to new bvec. The rq_for_each_bvec
363 * API will take care of all details for us.
364 */
365 bvec = cmd->bvec;
366 rq_for_each_bvec(tmp, rq, rq_iter) {
367 *bvec = tmp;
368 bvec++;
369 }
370 iov_iter_bvec(&iter, rw, cmd->bvec, nr_bvec, blk_rq_bytes(rq));
371 iter.iov_offset = 0;
372 } else {
373 /*
374 * Same here, this bio may be started from the middle of the
375 * 'bvec' because of bio splitting, so offset from the bvec
376 * must be passed to iov iterator
377 */
378 iov_iter_bvec(&iter, rw,
379 __bvec_iter_bvec(rq->bio->bi_io_vec, rq->bio->bi_iter),
380 nr_bvec, blk_rq_bytes(rq));
381 iter.iov_offset = rq->bio->bi_iter.bi_bvec_done;
382 }
383 atomic_set(&cmd->ref, 2);
384
385 cmd->iocb.ki_pos = pos;
386 cmd->iocb.ki_filp = file;
387 cmd->iocb.ki_ioprio = req_get_ioprio(rq);
388 if (cmd->use_aio) {
389 cmd->iocb.ki_complete = lo_rw_aio_complete;
390 cmd->iocb.ki_flags = IOCB_DIRECT;
391 } else {
392 cmd->iocb.ki_complete = NULL;
393 cmd->iocb.ki_flags = 0;
394 }
395
396 if (rw == ITER_SOURCE) {
397 kiocb_start_write(&cmd->iocb);
398 ret = file->f_op->write_iter(&cmd->iocb, &iter);
399 } else
400 ret = file->f_op->read_iter(&cmd->iocb, &iter);
401
402 lo_rw_aio_do_completion(cmd);
403
404 if (ret != -EIOCBQUEUED)
405 lo_rw_aio_complete(&cmd->iocb, ret);
406 return -EIOCBQUEUED;
407}
408
409static int do_req_filebacked(struct loop_device *lo, struct request *rq)
410{
411 struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
412 loff_t pos = ((loff_t) blk_rq_pos(rq) << 9) + lo->lo_offset;
413
414 switch (req_op(rq)) {
415 case REQ_OP_FLUSH:
416 return lo_req_flush(lo, rq);
417 case REQ_OP_WRITE_ZEROES:
418 /*
419 * If the caller doesn't want deallocation, call zeroout to
420 * write zeroes the range. Otherwise, punch them out.
421 */
422 return lo_fallocate(lo, rq, pos,
423 (rq->cmd_flags & REQ_NOUNMAP) ?
424 FALLOC_FL_ZERO_RANGE :
425 FALLOC_FL_PUNCH_HOLE);
426 case REQ_OP_DISCARD:
427 return lo_fallocate(lo, rq, pos, FALLOC_FL_PUNCH_HOLE);
428 case REQ_OP_WRITE:
429 return lo_rw_aio(lo, cmd, pos, ITER_SOURCE);
430 case REQ_OP_READ:
431 return lo_rw_aio(lo, cmd, pos, ITER_DEST);
432 default:
433 WARN_ON_ONCE(1);
434 return -EIO;
435 }
436}
437
438static void loop_reread_partitions(struct loop_device *lo)
439{
440 int rc;
441
442 mutex_lock(&lo->lo_disk->open_mutex);
443 rc = bdev_disk_changed(lo->lo_disk, false);
444 mutex_unlock(&lo->lo_disk->open_mutex);
445 if (rc)
446 pr_warn("%s: partition scan of loop%d (%s) failed (rc=%d)\n",
447 __func__, lo->lo_number, lo->lo_file_name, rc);
448}
449
450static unsigned int loop_query_min_dio_size(struct loop_device *lo)
451{
452 struct file *file = lo->lo_backing_file;
453 struct block_device *sb_bdev = file->f_mapping->host->i_sb->s_bdev;
454 struct kstat st;
455
456 /*
457 * Use the minimal dio alignment of the file system if provided.
458 */
459 if (!vfs_getattr(&file->f_path, &st, STATX_DIOALIGN, 0) &&
460 (st.result_mask & STATX_DIOALIGN))
461 return st.dio_offset_align;
462
463 /*
464 * In a perfect world this wouldn't be needed, but as of Linux 6.13 only
465 * a handful of file systems support the STATX_DIOALIGN flag.
466 */
467 if (sb_bdev)
468 return bdev_logical_block_size(sb_bdev);
469 return SECTOR_SIZE;
470}
471
472static inline int is_loop_device(struct file *file)
473{
474 struct inode *i = file->f_mapping->host;
475
476 return i && S_ISBLK(i->i_mode) && imajor(i) == LOOP_MAJOR;
477}
478
479static int loop_validate_file(struct file *file, struct block_device *bdev)
480{
481 struct inode *inode = file->f_mapping->host;
482 struct file *f = file;
483
484 /* Avoid recursion */
485 while (is_loop_device(f)) {
486 struct loop_device *l;
487
488 lockdep_assert_held(&loop_validate_mutex);
489 if (f->f_mapping->host->i_rdev == bdev->bd_dev)
490 return -EBADF;
491
492 l = I_BDEV(f->f_mapping->host)->bd_disk->private_data;
493 if (l->lo_state != Lo_bound)
494 return -EINVAL;
495 /* Order wrt setting lo->lo_backing_file in loop_configure(). */
496 rmb();
497 f = l->lo_backing_file;
498 }
499 if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode))
500 return -EINVAL;
501 return 0;
502}
503
504static void loop_assign_backing_file(struct loop_device *lo, struct file *file)
505{
506 lo->lo_backing_file = file;
507 lo->old_gfp_mask = mapping_gfp_mask(file->f_mapping);
508 mapping_set_gfp_mask(file->f_mapping,
509 lo->old_gfp_mask & ~(__GFP_IO | __GFP_FS));
510 if (lo->lo_backing_file->f_flags & O_DIRECT)
511 lo->lo_flags |= LO_FLAGS_DIRECT_IO;
512 lo->lo_min_dio_size = loop_query_min_dio_size(lo);
513}
514
515static int loop_check_backing_file(struct file *file)
516{
517 if (!file->f_op->read_iter)
518 return -EINVAL;
519
520 if ((file->f_mode & FMODE_WRITE) && !file->f_op->write_iter)
521 return -EINVAL;
522
523 return 0;
524}
525
526/*
527 * loop_change_fd switched the backing store of a loopback device to
528 * a new file. This is useful for operating system installers to free up
529 * the original file and in High Availability environments to switch to
530 * an alternative location for the content in case of server meltdown.
531 * This can only work if the loop device is used read-only, and if the
532 * new backing store is the same size and type as the old backing store.
533 */
534static int loop_change_fd(struct loop_device *lo, struct block_device *bdev,
535 unsigned int arg)
536{
537 struct file *file = fget(arg);
538 struct file *old_file;
539 unsigned int memflags;
540 int error;
541 bool partscan;
542 bool is_loop;
543
544 if (!file)
545 return -EBADF;
546
547 error = loop_check_backing_file(file);
548 if (error) {
549 fput(file);
550 return error;
551 }
552
553 /* suppress uevents while reconfiguring the device */
554 dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 1);
555
556 is_loop = is_loop_device(file);
557 error = loop_global_lock_killable(lo, is_loop);
558 if (error)
559 goto out_putf;
560 error = -ENXIO;
561 if (lo->lo_state != Lo_bound)
562 goto out_err;
563
564 /* the loop device has to be read-only */
565 error = -EINVAL;
566 if (!(lo->lo_flags & LO_FLAGS_READ_ONLY))
567 goto out_err;
568
569 error = loop_validate_file(file, bdev);
570 if (error)
571 goto out_err;
572
573 old_file = lo->lo_backing_file;
574
575 error = -EINVAL;
576
577 /* size of the new backing store needs to be the same */
578 if (lo_calculate_size(lo, file) != lo_calculate_size(lo, old_file))
579 goto out_err;
580
581 /*
582 * We might switch to direct I/O mode for the loop device, write back
583 * all dirty data the page cache now that so that the individual I/O
584 * operations don't have to do that.
585 */
586 vfs_fsync(file, 0);
587
588 /* and ... switch */
589 disk_force_media_change(lo->lo_disk);
590 memflags = blk_mq_freeze_queue(lo->lo_queue);
591 mapping_set_gfp_mask(old_file->f_mapping, lo->old_gfp_mask);
592 loop_assign_backing_file(lo, file);
593 loop_update_dio(lo);
594 blk_mq_unfreeze_queue(lo->lo_queue, memflags);
595 partscan = lo->lo_flags & LO_FLAGS_PARTSCAN;
596 loop_global_unlock(lo, is_loop);
597
598 /*
599 * Flush loop_validate_file() before fput(), for l->lo_backing_file
600 * might be pointing at old_file which might be the last reference.
601 */
602 if (!is_loop) {
603 mutex_lock(&loop_validate_mutex);
604 mutex_unlock(&loop_validate_mutex);
605 }
606 /*
607 * We must drop file reference outside of lo_mutex as dropping
608 * the file ref can take open_mutex which creates circular locking
609 * dependency.
610 */
611 fput(old_file);
612 dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 0);
613 if (partscan)
614 loop_reread_partitions(lo);
615
616 error = 0;
617done:
618 kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE);
619 return error;
620
621out_err:
622 loop_global_unlock(lo, is_loop);
623out_putf:
624 fput(file);
625 dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 0);
626 goto done;
627}
628
629/* loop sysfs attributes */
630
631static ssize_t loop_attr_show(struct device *dev, char *page,
632 ssize_t (*callback)(struct loop_device *, char *))
633{
634 struct gendisk *disk = dev_to_disk(dev);
635 struct loop_device *lo = disk->private_data;
636
637 return callback(lo, page);
638}
639
640#define LOOP_ATTR_RO(_name) \
641static ssize_t loop_attr_##_name##_show(struct loop_device *, char *); \
642static ssize_t loop_attr_do_show_##_name(struct device *d, \
643 struct device_attribute *attr, char *b) \
644{ \
645 return loop_attr_show(d, b, loop_attr_##_name##_show); \
646} \
647static struct device_attribute loop_attr_##_name = \
648 __ATTR(_name, 0444, loop_attr_do_show_##_name, NULL);
649
650static ssize_t loop_attr_backing_file_show(struct loop_device *lo, char *buf)
651{
652 ssize_t ret;
653 char *p = NULL;
654
655 spin_lock_irq(&lo->lo_lock);
656 if (lo->lo_backing_file)
657 p = file_path(lo->lo_backing_file, buf, PAGE_SIZE - 1);
658 spin_unlock_irq(&lo->lo_lock);
659
660 if (IS_ERR_OR_NULL(p))
661 ret = PTR_ERR(p);
662 else {
663 ret = strlen(p);
664 memmove(buf, p, ret);
665 buf[ret++] = '\n';
666 buf[ret] = 0;
667 }
668
669 return ret;
670}
671
672static ssize_t loop_attr_offset_show(struct loop_device *lo, char *buf)
673{
674 return sysfs_emit(buf, "%llu\n", (unsigned long long)lo->lo_offset);
675}
676
677static ssize_t loop_attr_sizelimit_show(struct loop_device *lo, char *buf)
678{
679 return sysfs_emit(buf, "%llu\n", (unsigned long long)lo->lo_sizelimit);
680}
681
682static ssize_t loop_attr_autoclear_show(struct loop_device *lo, char *buf)
683{
684 int autoclear = (lo->lo_flags & LO_FLAGS_AUTOCLEAR);
685
686 return sysfs_emit(buf, "%s\n", autoclear ? "1" : "0");
687}
688
689static ssize_t loop_attr_partscan_show(struct loop_device *lo, char *buf)
690{
691 int partscan = (lo->lo_flags & LO_FLAGS_PARTSCAN);
692
693 return sysfs_emit(buf, "%s\n", partscan ? "1" : "0");
694}
695
696static ssize_t loop_attr_dio_show(struct loop_device *lo, char *buf)
697{
698 int dio = (lo->lo_flags & LO_FLAGS_DIRECT_IO);
699
700 return sysfs_emit(buf, "%s\n", dio ? "1" : "0");
701}
702
703LOOP_ATTR_RO(backing_file);
704LOOP_ATTR_RO(offset);
705LOOP_ATTR_RO(sizelimit);
706LOOP_ATTR_RO(autoclear);
707LOOP_ATTR_RO(partscan);
708LOOP_ATTR_RO(dio);
709
710static struct attribute *loop_attrs[] = {
711 &loop_attr_backing_file.attr,
712 &loop_attr_offset.attr,
713 &loop_attr_sizelimit.attr,
714 &loop_attr_autoclear.attr,
715 &loop_attr_partscan.attr,
716 &loop_attr_dio.attr,
717 NULL,
718};
719
720static struct attribute_group loop_attribute_group = {
721 .name = "loop",
722 .attrs= loop_attrs,
723};
724
725static void loop_sysfs_init(struct loop_device *lo)
726{
727 lo->sysfs_inited = !sysfs_create_group(&disk_to_dev(lo->lo_disk)->kobj,
728 &loop_attribute_group);
729}
730
731static void loop_sysfs_exit(struct loop_device *lo)
732{
733 if (lo->sysfs_inited)
734 sysfs_remove_group(&disk_to_dev(lo->lo_disk)->kobj,
735 &loop_attribute_group);
736}
737
738static void loop_get_discard_config(struct loop_device *lo,
739 u32 *granularity, u32 *max_discard_sectors)
740{
741 struct file *file = lo->lo_backing_file;
742 struct inode *inode = file->f_mapping->host;
743 struct kstatfs sbuf;
744
745 /*
746 * If the backing device is a block device, mirror its zeroing
747 * capability. Set the discard sectors to the block device's zeroing
748 * capabilities because loop discards result in blkdev_issue_zeroout(),
749 * not blkdev_issue_discard(). This maintains consistent behavior with
750 * file-backed loop devices: discarded regions read back as zero.
751 */
752 if (S_ISBLK(inode->i_mode)) {
753 struct block_device *bdev = I_BDEV(inode);
754
755 *max_discard_sectors = bdev_write_zeroes_sectors(bdev);
756 *granularity = bdev_discard_granularity(bdev);
757
758 /*
759 * We use punch hole to reclaim the free space used by the
760 * image a.k.a. discard.
761 */
762 } else if (file->f_op->fallocate && !vfs_statfs(&file->f_path, &sbuf)) {
763 *max_discard_sectors = UINT_MAX >> 9;
764 *granularity = sbuf.f_bsize;
765 }
766}
767
768struct loop_worker {
769 struct rb_node rb_node;
770 struct work_struct work;
771 struct list_head cmd_list;
772 struct list_head idle_list;
773 struct loop_device *lo;
774 struct cgroup_subsys_state *blkcg_css;
775 unsigned long last_ran_at;
776};
777
778static void loop_workfn(struct work_struct *work);
779
780#ifdef CONFIG_BLK_CGROUP
781static inline int queue_on_root_worker(struct cgroup_subsys_state *css)
782{
783 return !css || css == blkcg_root_css;
784}
785#else
786static inline int queue_on_root_worker(struct cgroup_subsys_state *css)
787{
788 return !css;
789}
790#endif
791
792static void loop_queue_work(struct loop_device *lo, struct loop_cmd *cmd)
793{
794 struct rb_node **node, *parent = NULL;
795 struct loop_worker *cur_worker, *worker = NULL;
796 struct work_struct *work;
797 struct list_head *cmd_list;
798
799 spin_lock_irq(&lo->lo_work_lock);
800
801 if (queue_on_root_worker(cmd->blkcg_css))
802 goto queue_work;
803
804 node = &lo->worker_tree.rb_node;
805
806 while (*node) {
807 parent = *node;
808 cur_worker = container_of(*node, struct loop_worker, rb_node);
809 if (cur_worker->blkcg_css == cmd->blkcg_css) {
810 worker = cur_worker;
811 break;
812 } else if ((long)cur_worker->blkcg_css < (long)cmd->blkcg_css) {
813 node = &(*node)->rb_left;
814 } else {
815 node = &(*node)->rb_right;
816 }
817 }
818 if (worker)
819 goto queue_work;
820
821 worker = kzalloc_obj(struct loop_worker, GFP_NOWAIT);
822 /*
823 * In the event we cannot allocate a worker, just queue on the
824 * rootcg worker and issue the I/O as the rootcg
825 */
826 if (!worker) {
827 cmd->blkcg_css = NULL;
828 if (cmd->memcg_css)
829 css_put(cmd->memcg_css);
830 cmd->memcg_css = NULL;
831 goto queue_work;
832 }
833
834 worker->blkcg_css = cmd->blkcg_css;
835 css_get(worker->blkcg_css);
836 INIT_WORK(&worker->work, loop_workfn);
837 INIT_LIST_HEAD(&worker->cmd_list);
838 INIT_LIST_HEAD(&worker->idle_list);
839 worker->lo = lo;
840 rb_link_node(&worker->rb_node, parent, node);
841 rb_insert_color(&worker->rb_node, &lo->worker_tree);
842queue_work:
843 if (worker) {
844 /*
845 * We need to remove from the idle list here while
846 * holding the lock so that the idle timer doesn't
847 * free the worker
848 */
849 if (!list_empty(&worker->idle_list))
850 list_del_init(&worker->idle_list);
851 work = &worker->work;
852 cmd_list = &worker->cmd_list;
853 } else {
854 work = &lo->rootcg_work;
855 cmd_list = &lo->rootcg_cmd_list;
856 }
857 list_add_tail(&cmd->list_entry, cmd_list);
858 queue_work(lo->workqueue, work);
859 spin_unlock_irq(&lo->lo_work_lock);
860}
861
862static void loop_set_timer(struct loop_device *lo)
863{
864 timer_reduce(&lo->timer, jiffies + LOOP_IDLE_WORKER_TIMEOUT);
865}
866
867static void loop_free_idle_workers(struct loop_device *lo, bool delete_all)
868{
869 struct loop_worker *pos, *worker;
870
871 spin_lock_irq(&lo->lo_work_lock);
872 list_for_each_entry_safe(worker, pos, &lo->idle_worker_list,
873 idle_list) {
874 if (!delete_all &&
875 time_is_after_jiffies(worker->last_ran_at +
876 LOOP_IDLE_WORKER_TIMEOUT))
877 break;
878 list_del(&worker->idle_list);
879 rb_erase(&worker->rb_node, &lo->worker_tree);
880 css_put(worker->blkcg_css);
881 kfree(worker);
882 }
883 if (!list_empty(&lo->idle_worker_list))
884 loop_set_timer(lo);
885 spin_unlock_irq(&lo->lo_work_lock);
886}
887
888static void loop_free_idle_workers_timer(struct timer_list *timer)
889{
890 struct loop_device *lo = container_of(timer, struct loop_device, timer);
891
892 return loop_free_idle_workers(lo, false);
893}
894
895/**
896 * loop_set_status_from_info - configure device from loop_info
897 * @lo: struct loop_device to configure
898 * @info: struct loop_info64 to configure the device with
899 *
900 * Configures the loop device parameters according to the passed
901 * in loop_info64 configuration.
902 */
903static int
904loop_set_status_from_info(struct loop_device *lo,
905 const struct loop_info64 *info)
906{
907 if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE)
908 return -EINVAL;
909
910 switch (info->lo_encrypt_type) {
911 case LO_CRYPT_NONE:
912 break;
913 case LO_CRYPT_XOR:
914 pr_warn("support for the xor transformation has been removed.\n");
915 return -EINVAL;
916 case LO_CRYPT_CRYPTOAPI:
917 pr_warn("support for cryptoloop has been removed. Use dm-crypt instead.\n");
918 return -EINVAL;
919 default:
920 return -EINVAL;
921 }
922
923 /* Avoid assigning overflow values */
924 if (info->lo_offset > LLONG_MAX || info->lo_sizelimit > LLONG_MAX)
925 return -EOVERFLOW;
926
927 lo->lo_offset = info->lo_offset;
928 lo->lo_sizelimit = info->lo_sizelimit;
929
930 memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE);
931 lo->lo_file_name[LO_NAME_SIZE-1] = 0;
932 return 0;
933}
934
935static unsigned int loop_default_blocksize(struct loop_device *lo)
936{
937 /* In case of direct I/O, match underlying minimum I/O size */
938 if (lo->lo_flags & LO_FLAGS_DIRECT_IO)
939 return lo->lo_min_dio_size;
940 return SECTOR_SIZE;
941}
942
943static void loop_update_limits(struct loop_device *lo, struct queue_limits *lim,
944 unsigned int bsize)
945{
946 struct file *file = lo->lo_backing_file;
947 struct inode *inode = file->f_mapping->host;
948 struct block_device *backing_bdev = NULL;
949 u32 granularity = 0, max_discard_sectors = 0;
950
951 if (S_ISBLK(inode->i_mode))
952 backing_bdev = I_BDEV(inode);
953 else if (inode->i_sb->s_bdev)
954 backing_bdev = inode->i_sb->s_bdev;
955
956 if (!bsize)
957 bsize = loop_default_blocksize(lo);
958
959 loop_get_discard_config(lo, &granularity, &max_discard_sectors);
960
961 lim->logical_block_size = bsize;
962 lim->physical_block_size = bsize;
963 lim->io_min = bsize;
964 lim->features &= ~(BLK_FEAT_WRITE_CACHE | BLK_FEAT_ROTATIONAL);
965 if (file->f_op->fsync && !(lo->lo_flags & LO_FLAGS_READ_ONLY))
966 lim->features |= BLK_FEAT_WRITE_CACHE;
967 if (backing_bdev && bdev_rot(backing_bdev))
968 lim->features |= BLK_FEAT_ROTATIONAL;
969 lim->max_hw_discard_sectors = max_discard_sectors;
970 lim->max_write_zeroes_sectors = max_discard_sectors;
971 if (max_discard_sectors)
972 lim->discard_granularity = granularity;
973 else
974 lim->discard_granularity = 0;
975}
976
977static int loop_configure(struct loop_device *lo, blk_mode_t mode,
978 struct block_device *bdev,
979 const struct loop_config *config)
980{
981 struct file *file = fget(config->fd);
982 struct queue_limits lim;
983 int error;
984 loff_t size;
985 bool partscan;
986 bool is_loop;
987
988 if (!file)
989 return -EBADF;
990
991 error = loop_check_backing_file(file);
992 if (error) {
993 fput(file);
994 return error;
995 }
996
997 is_loop = is_loop_device(file);
998
999 /* This is safe, since we have a reference from open(). */
1000 __module_get(THIS_MODULE);
1001
1002 /*
1003 * If we don't hold exclusive handle for the device, upgrade to it
1004 * here to avoid changing device under exclusive owner.
1005 */
1006 if (!(mode & BLK_OPEN_EXCL)) {
1007 error = bd_prepare_to_claim(bdev, loop_configure, NULL);
1008 if (error)
1009 goto out_putf;
1010 }
1011
1012 error = loop_global_lock_killable(lo, is_loop);
1013 if (error)
1014 goto out_bdev;
1015
1016 error = -EBUSY;
1017 if (lo->lo_state != Lo_unbound)
1018 goto out_unlock;
1019
1020 error = loop_validate_file(file, bdev);
1021 if (error)
1022 goto out_unlock;
1023
1024 if ((config->info.lo_flags & ~LOOP_CONFIGURE_SETTABLE_FLAGS) != 0) {
1025 error = -EINVAL;
1026 goto out_unlock;
1027 }
1028
1029 error = loop_set_status_from_info(lo, &config->info);
1030 if (error)
1031 goto out_unlock;
1032 lo->lo_flags = config->info.lo_flags;
1033
1034 if (!(file->f_mode & FMODE_WRITE) || !(mode & BLK_OPEN_WRITE) ||
1035 !file->f_op->write_iter)
1036 lo->lo_flags |= LO_FLAGS_READ_ONLY;
1037
1038 if (!lo->workqueue) {
1039 lo->workqueue = alloc_workqueue("loop%d",
1040 WQ_UNBOUND | WQ_FREEZABLE,
1041 0, lo->lo_number);
1042 if (!lo->workqueue) {
1043 error = -ENOMEM;
1044 goto out_unlock;
1045 }
1046 }
1047
1048 /* suppress uevents while reconfiguring the device */
1049 dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 1);
1050
1051 disk_force_media_change(lo->lo_disk);
1052 set_disk_ro(lo->lo_disk, (lo->lo_flags & LO_FLAGS_READ_ONLY) != 0);
1053
1054 lo->lo_device = bdev;
1055 loop_assign_backing_file(lo, file);
1056
1057 lim = queue_limits_start_update(lo->lo_queue);
1058 loop_update_limits(lo, &lim, config->block_size);
1059 /* No need to freeze the queue as the device isn't bound yet. */
1060 error = queue_limits_commit_update(lo->lo_queue, &lim);
1061 if (error)
1062 goto out_unlock;
1063
1064 /*
1065 * We might switch to direct I/O mode for the loop device, write back
1066 * all dirty data the page cache now that so that the individual I/O
1067 * operations don't have to do that.
1068 */
1069 vfs_fsync(file, 0);
1070
1071 loop_update_dio(lo);
1072 loop_sysfs_init(lo);
1073
1074 size = lo_calculate_size(lo, file);
1075 loop_set_size(lo, size);
1076
1077 /* Order wrt reading lo_state in loop_validate_file(). */
1078 wmb();
1079
1080 WRITE_ONCE(lo->lo_state, Lo_bound);
1081 if (part_shift)
1082 lo->lo_flags |= LO_FLAGS_PARTSCAN;
1083 partscan = lo->lo_flags & LO_FLAGS_PARTSCAN;
1084 if (partscan)
1085 clear_bit(GD_SUPPRESS_PART_SCAN, &lo->lo_disk->state);
1086
1087 dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 0);
1088 kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE);
1089
1090 loop_global_unlock(lo, is_loop);
1091 if (partscan)
1092 loop_reread_partitions(lo);
1093
1094 if (!(mode & BLK_OPEN_EXCL))
1095 bd_abort_claiming(bdev, loop_configure);
1096
1097 return 0;
1098
1099out_unlock:
1100 loop_global_unlock(lo, is_loop);
1101out_bdev:
1102 if (!(mode & BLK_OPEN_EXCL))
1103 bd_abort_claiming(bdev, loop_configure);
1104out_putf:
1105 fput(file);
1106 /* This is safe: open() is still holding a reference. */
1107 module_put(THIS_MODULE);
1108 return error;
1109}
1110
1111static void __loop_clr_fd(struct loop_device *lo)
1112{
1113 struct queue_limits lim;
1114 struct file *filp;
1115 gfp_t gfp = lo->old_gfp_mask;
1116 int err;
1117
1118 spin_lock_irq(&lo->lo_lock);
1119 filp = lo->lo_backing_file;
1120 lo->lo_backing_file = NULL;
1121 spin_unlock_irq(&lo->lo_lock);
1122
1123 lo->lo_device = NULL;
1124 lo->lo_offset = 0;
1125 lo->lo_sizelimit = 0;
1126 memset(lo->lo_file_name, 0, LO_NAME_SIZE);
1127
1128 /*
1129 * Reset the block size to the default.
1130 *
1131 * No queue freezing needed because this is called from the final
1132 * ->release call only, so there can't be any outstanding I/O.
1133 */
1134 lim = queue_limits_start_update(lo->lo_queue);
1135 lim.logical_block_size = SECTOR_SIZE;
1136 lim.physical_block_size = SECTOR_SIZE;
1137 lim.io_min = SECTOR_SIZE;
1138 queue_limits_commit_update(lo->lo_queue, &lim);
1139
1140 invalidate_disk(lo->lo_disk);
1141 loop_sysfs_exit(lo);
1142 /* let user-space know about this change */
1143 kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE);
1144 mapping_set_gfp_mask(filp->f_mapping, gfp);
1145 /* This is safe: open() is still holding a reference. */
1146 module_put(THIS_MODULE);
1147
1148 disk_force_media_change(lo->lo_disk);
1149
1150 /*
1151 * Remove all partitions, including partitions added manually with
1152 * BLKPG, which may exist even if LO_FLAGS_PARTSCAN is not set.
1153 *
1154 * open_mutex has been held already in release path, so don't acquire
1155 * it here.
1156 */
1157 err = bdev_disk_changed(lo->lo_disk, false);
1158 if (err)
1159 pr_warn("%s: partition scan of loop%d failed (rc=%d)\n",
1160 __func__, lo->lo_number, err);
1161 /* Device is gone, no point in returning error */
1162
1163 /*
1164 * lo->lo_state is set to Lo_unbound here after removing partitions has
1165 * finished. There cannot be anybody else entering __loop_clr_fd() as
1166 * Lo_rundown state protects us from all the other places trying to
1167 * change the 'lo' device.
1168 */
1169 lo->lo_flags = 0;
1170 if (!part_shift)
1171 set_bit(GD_SUPPRESS_PART_SCAN, &lo->lo_disk->state);
1172 mutex_lock(&lo->lo_mutex);
1173 WRITE_ONCE(lo->lo_state, Lo_unbound);
1174 mutex_unlock(&lo->lo_mutex);
1175
1176 /*
1177 * Need not hold lo_mutex to fput backing file. Calling fput holding
1178 * lo_mutex triggers a circular lock dependency possibility warning as
1179 * fput can take open_mutex which is usually taken before lo_mutex.
1180 */
1181 fput(filp);
1182}
1183
1184static int loop_clr_fd(struct loop_device *lo)
1185{
1186 int err;
1187
1188 /*
1189 * Since lo_ioctl() is called without locks held, it is possible that
1190 * loop_configure()/loop_change_fd() and loop_clr_fd() run in parallel.
1191 *
1192 * Therefore, use global lock when setting Lo_rundown state in order to
1193 * make sure that loop_validate_file() will fail if the "struct file"
1194 * which loop_configure()/loop_change_fd() found via fget() was this
1195 * loop device.
1196 */
1197 err = loop_global_lock_killable(lo, true);
1198 if (err)
1199 return err;
1200 if (lo->lo_state != Lo_bound) {
1201 loop_global_unlock(lo, true);
1202 return -ENXIO;
1203 }
1204 /*
1205 * Mark the device for removing the backing device on last close.
1206 * If we are the only opener, also switch the state to roundown here to
1207 * prevent new openers from coming in.
1208 */
1209
1210 lo->lo_flags |= LO_FLAGS_AUTOCLEAR;
1211 if (disk_openers(lo->lo_disk) == 1)
1212 WRITE_ONCE(lo->lo_state, Lo_rundown);
1213 loop_global_unlock(lo, true);
1214
1215 return 0;
1216}
1217
1218static int
1219loop_set_status(struct loop_device *lo, const struct loop_info64 *info)
1220{
1221 int err;
1222 bool partscan = false;
1223 bool size_changed = false;
1224 unsigned int memflags;
1225
1226 err = mutex_lock_killable(&lo->lo_mutex);
1227 if (err)
1228 return err;
1229 if (lo->lo_state != Lo_bound) {
1230 err = -ENXIO;
1231 goto out_unlock;
1232 }
1233
1234 if (lo->lo_offset != info->lo_offset ||
1235 lo->lo_sizelimit != info->lo_sizelimit) {
1236 size_changed = true;
1237 sync_blockdev(lo->lo_device);
1238 invalidate_bdev(lo->lo_device);
1239 }
1240
1241 /* I/O needs to be drained before changing lo_offset or lo_sizelimit */
1242 memflags = blk_mq_freeze_queue(lo->lo_queue);
1243
1244 err = loop_set_status_from_info(lo, info);
1245 if (err)
1246 goto out_unfreeze;
1247
1248 partscan = !(lo->lo_flags & LO_FLAGS_PARTSCAN) &&
1249 (info->lo_flags & LO_FLAGS_PARTSCAN);
1250
1251 lo->lo_flags &= ~LOOP_SET_STATUS_CLEARABLE_FLAGS;
1252 lo->lo_flags |= (info->lo_flags & LOOP_SET_STATUS_SETTABLE_FLAGS);
1253
1254 /* update the direct I/O flag if lo_offset changed */
1255 loop_update_dio(lo);
1256
1257out_unfreeze:
1258 blk_mq_unfreeze_queue(lo->lo_queue, memflags);
1259 if (partscan)
1260 clear_bit(GD_SUPPRESS_PART_SCAN, &lo->lo_disk->state);
1261 if (!err && size_changed) {
1262 loff_t new_size = lo_calculate_size(lo, lo->lo_backing_file);
1263 loop_set_size(lo, new_size);
1264 }
1265out_unlock:
1266 mutex_unlock(&lo->lo_mutex);
1267 if (partscan)
1268 loop_reread_partitions(lo);
1269
1270 return err;
1271}
1272
1273static int
1274loop_get_status(struct loop_device *lo, struct loop_info64 *info)
1275{
1276 struct path path;
1277 struct kstat stat;
1278 int ret;
1279
1280 ret = mutex_lock_killable(&lo->lo_mutex);
1281 if (ret)
1282 return ret;
1283 if (lo->lo_state != Lo_bound) {
1284 mutex_unlock(&lo->lo_mutex);
1285 return -ENXIO;
1286 }
1287
1288 memset(info, 0, sizeof(*info));
1289 info->lo_number = lo->lo_number;
1290 info->lo_offset = lo->lo_offset;
1291 info->lo_sizelimit = lo->lo_sizelimit;
1292 info->lo_flags = lo->lo_flags;
1293 memcpy(info->lo_file_name, lo->lo_file_name, LO_NAME_SIZE);
1294
1295 /* Drop lo_mutex while we call into the filesystem. */
1296 path = lo->lo_backing_file->f_path;
1297 path_get(&path);
1298 mutex_unlock(&lo->lo_mutex);
1299 ret = vfs_getattr(&path, &stat, STATX_INO, AT_STATX_SYNC_AS_STAT);
1300 if (!ret) {
1301 info->lo_device = huge_encode_dev(stat.dev);
1302 info->lo_inode = stat.ino;
1303 info->lo_rdevice = huge_encode_dev(stat.rdev);
1304 }
1305 path_put(&path);
1306 return ret;
1307}
1308
1309static void
1310loop_info64_from_old(const struct loop_info *info, struct loop_info64 *info64)
1311{
1312 memset(info64, 0, sizeof(*info64));
1313 info64->lo_number = info->lo_number;
1314 info64->lo_device = info->lo_device;
1315 info64->lo_inode = info->lo_inode;
1316 info64->lo_rdevice = info->lo_rdevice;
1317 info64->lo_offset = info->lo_offset;
1318 info64->lo_sizelimit = 0;
1319 info64->lo_flags = info->lo_flags;
1320 memcpy(info64->lo_file_name, info->lo_name, LO_NAME_SIZE);
1321}
1322
1323static int
1324loop_info64_to_old(const struct loop_info64 *info64, struct loop_info *info)
1325{
1326 memset(info, 0, sizeof(*info));
1327 info->lo_number = info64->lo_number;
1328 info->lo_device = info64->lo_device;
1329 info->lo_inode = info64->lo_inode;
1330 info->lo_rdevice = info64->lo_rdevice;
1331 info->lo_offset = info64->lo_offset;
1332 info->lo_flags = info64->lo_flags;
1333 memcpy(info->lo_name, info64->lo_file_name, LO_NAME_SIZE);
1334
1335 /* error in case values were truncated */
1336 if (info->lo_device != info64->lo_device ||
1337 info->lo_rdevice != info64->lo_rdevice ||
1338 info->lo_inode != info64->lo_inode ||
1339 info->lo_offset != info64->lo_offset)
1340 return -EOVERFLOW;
1341
1342 return 0;
1343}
1344
1345static int
1346loop_set_status_old(struct loop_device *lo, const struct loop_info __user *arg)
1347{
1348 struct loop_info info;
1349 struct loop_info64 info64;
1350
1351 if (copy_from_user(&info, arg, sizeof (struct loop_info)))
1352 return -EFAULT;
1353 loop_info64_from_old(&info, &info64);
1354 return loop_set_status(lo, &info64);
1355}
1356
1357static int
1358loop_set_status64(struct loop_device *lo, const struct loop_info64 __user *arg)
1359{
1360 struct loop_info64 info64;
1361
1362 if (copy_from_user(&info64, arg, sizeof (struct loop_info64)))
1363 return -EFAULT;
1364 return loop_set_status(lo, &info64);
1365}
1366
1367static int
1368loop_get_status_old(struct loop_device *lo, struct loop_info __user *arg) {
1369 struct loop_info info;
1370 struct loop_info64 info64;
1371 int err;
1372
1373 if (!arg)
1374 return -EINVAL;
1375 err = loop_get_status(lo, &info64);
1376 if (!err)
1377 err = loop_info64_to_old(&info64, &info);
1378 if (!err && copy_to_user(arg, &info, sizeof(info)))
1379 err = -EFAULT;
1380
1381 return err;
1382}
1383
1384static int
1385loop_get_status64(struct loop_device *lo, struct loop_info64 __user *arg) {
1386 struct loop_info64 info64;
1387 int err;
1388
1389 if (!arg)
1390 return -EINVAL;
1391 err = loop_get_status(lo, &info64);
1392 if (!err && copy_to_user(arg, &info64, sizeof(info64)))
1393 err = -EFAULT;
1394
1395 return err;
1396}
1397
1398static int loop_set_capacity(struct loop_device *lo)
1399{
1400 loff_t size;
1401
1402 if (unlikely(lo->lo_state != Lo_bound))
1403 return -ENXIO;
1404
1405 size = lo_calculate_size(lo, lo->lo_backing_file);
1406 loop_set_size(lo, size);
1407
1408 return 0;
1409}
1410
1411static int loop_set_dio(struct loop_device *lo, unsigned long arg)
1412{
1413 bool use_dio = !!arg;
1414 unsigned int memflags;
1415
1416 if (lo->lo_state != Lo_bound)
1417 return -ENXIO;
1418 if (use_dio == !!(lo->lo_flags & LO_FLAGS_DIRECT_IO))
1419 return 0;
1420
1421 if (use_dio) {
1422 if (!lo_can_use_dio(lo))
1423 return -EINVAL;
1424 /* flush dirty pages before starting to use direct I/O */
1425 vfs_fsync(lo->lo_backing_file, 0);
1426 }
1427
1428 memflags = blk_mq_freeze_queue(lo->lo_queue);
1429 if (use_dio)
1430 lo->lo_flags |= LO_FLAGS_DIRECT_IO;
1431 else
1432 lo->lo_flags &= ~LO_FLAGS_DIRECT_IO;
1433 blk_mq_unfreeze_queue(lo->lo_queue, memflags);
1434 return 0;
1435}
1436
1437static int loop_set_block_size(struct loop_device *lo, blk_mode_t mode,
1438 struct block_device *bdev, unsigned long arg)
1439{
1440 struct queue_limits lim;
1441 unsigned int memflags;
1442 int err = 0;
1443
1444 /*
1445 * If we don't hold exclusive handle for the device, upgrade to it
1446 * here to avoid changing device under exclusive owner.
1447 */
1448 if (!(mode & BLK_OPEN_EXCL)) {
1449 err = bd_prepare_to_claim(bdev, loop_set_block_size, NULL);
1450 if (err)
1451 return err;
1452 }
1453
1454 err = mutex_lock_killable(&lo->lo_mutex);
1455 if (err)
1456 goto abort_claim;
1457
1458 if (lo->lo_state != Lo_bound) {
1459 err = -ENXIO;
1460 goto unlock;
1461 }
1462
1463 if (lo->lo_queue->limits.logical_block_size == arg)
1464 goto unlock;
1465
1466 sync_blockdev(lo->lo_device);
1467 invalidate_bdev(lo->lo_device);
1468
1469 lim = queue_limits_start_update(lo->lo_queue);
1470 loop_update_limits(lo, &lim, arg);
1471
1472 memflags = blk_mq_freeze_queue(lo->lo_queue);
1473 err = queue_limits_commit_update(lo->lo_queue, &lim);
1474 loop_update_dio(lo);
1475 blk_mq_unfreeze_queue(lo->lo_queue, memflags);
1476
1477unlock:
1478 mutex_unlock(&lo->lo_mutex);
1479abort_claim:
1480 if (!(mode & BLK_OPEN_EXCL))
1481 bd_abort_claiming(bdev, loop_set_block_size);
1482 return err;
1483}
1484
1485static int lo_simple_ioctl(struct loop_device *lo, unsigned int cmd,
1486 unsigned long arg)
1487{
1488 int err;
1489
1490 err = mutex_lock_killable(&lo->lo_mutex);
1491 if (err)
1492 return err;
1493 switch (cmd) {
1494 case LOOP_SET_CAPACITY:
1495 err = loop_set_capacity(lo);
1496 break;
1497 case LOOP_SET_DIRECT_IO:
1498 err = loop_set_dio(lo, arg);
1499 break;
1500 default:
1501 err = -EINVAL;
1502 }
1503 mutex_unlock(&lo->lo_mutex);
1504 return err;
1505}
1506
1507static int lo_ioctl(struct block_device *bdev, blk_mode_t mode,
1508 unsigned int cmd, unsigned long arg)
1509{
1510 struct loop_device *lo = bdev->bd_disk->private_data;
1511 void __user *argp = (void __user *) arg;
1512 int err;
1513
1514 switch (cmd) {
1515 case LOOP_SET_FD: {
1516 /*
1517 * Legacy case - pass in a zeroed out struct loop_config with
1518 * only the file descriptor set , which corresponds with the
1519 * default parameters we'd have used otherwise.
1520 */
1521 struct loop_config config;
1522
1523 memset(&config, 0, sizeof(config));
1524 config.fd = arg;
1525
1526 return loop_configure(lo, mode, bdev, &config);
1527 }
1528 case LOOP_CONFIGURE: {
1529 struct loop_config config;
1530
1531 if (copy_from_user(&config, argp, sizeof(config)))
1532 return -EFAULT;
1533
1534 return loop_configure(lo, mode, bdev, &config);
1535 }
1536 case LOOP_CHANGE_FD:
1537 return loop_change_fd(lo, bdev, arg);
1538 case LOOP_CLR_FD:
1539 return loop_clr_fd(lo);
1540 case LOOP_SET_STATUS:
1541 err = -EPERM;
1542 if ((mode & BLK_OPEN_WRITE) || capable(CAP_SYS_ADMIN))
1543 err = loop_set_status_old(lo, argp);
1544 break;
1545 case LOOP_GET_STATUS:
1546 return loop_get_status_old(lo, argp);
1547 case LOOP_SET_STATUS64:
1548 err = -EPERM;
1549 if ((mode & BLK_OPEN_WRITE) || capable(CAP_SYS_ADMIN))
1550 err = loop_set_status64(lo, argp);
1551 break;
1552 case LOOP_GET_STATUS64:
1553 return loop_get_status64(lo, argp);
1554 case LOOP_SET_BLOCK_SIZE:
1555 if (!(mode & BLK_OPEN_WRITE) && !capable(CAP_SYS_ADMIN))
1556 return -EPERM;
1557 return loop_set_block_size(lo, mode, bdev, arg);
1558 case LOOP_SET_CAPACITY:
1559 case LOOP_SET_DIRECT_IO:
1560 if (!(mode & BLK_OPEN_WRITE) && !capable(CAP_SYS_ADMIN))
1561 return -EPERM;
1562 fallthrough;
1563 default:
1564 err = lo_simple_ioctl(lo, cmd, arg);
1565 break;
1566 }
1567
1568 return err;
1569}
1570
1571#ifdef CONFIG_COMPAT
1572struct compat_loop_info {
1573 compat_int_t lo_number; /* ioctl r/o */
1574 compat_dev_t lo_device; /* ioctl r/o */
1575 compat_ulong_t lo_inode; /* ioctl r/o */
1576 compat_dev_t lo_rdevice; /* ioctl r/o */
1577 compat_int_t lo_offset;
1578 compat_int_t lo_encrypt_type; /* obsolete, ignored */
1579 compat_int_t lo_encrypt_key_size; /* ioctl w/o */
1580 compat_int_t lo_flags; /* ioctl r/o */
1581 char lo_name[LO_NAME_SIZE];
1582 unsigned char lo_encrypt_key[LO_KEY_SIZE]; /* ioctl w/o */
1583 compat_ulong_t lo_init[2];
1584 char reserved[4];
1585};
1586
1587/*
1588 * Transfer 32-bit compatibility structure in userspace to 64-bit loop info
1589 * - noinlined to reduce stack space usage in main part of driver
1590 */
1591static noinline int
1592loop_info64_from_compat(const struct compat_loop_info __user *arg,
1593 struct loop_info64 *info64)
1594{
1595 struct compat_loop_info info;
1596
1597 if (copy_from_user(&info, arg, sizeof(info)))
1598 return -EFAULT;
1599
1600 memset(info64, 0, sizeof(*info64));
1601 info64->lo_number = info.lo_number;
1602 info64->lo_device = info.lo_device;
1603 info64->lo_inode = info.lo_inode;
1604 info64->lo_rdevice = info.lo_rdevice;
1605 info64->lo_offset = info.lo_offset;
1606 info64->lo_sizelimit = 0;
1607 info64->lo_flags = info.lo_flags;
1608 memcpy(info64->lo_file_name, info.lo_name, LO_NAME_SIZE);
1609 return 0;
1610}
1611
1612/*
1613 * Transfer 64-bit loop info to 32-bit compatibility structure in userspace
1614 * - noinlined to reduce stack space usage in main part of driver
1615 */
1616static noinline int
1617loop_info64_to_compat(const struct loop_info64 *info64,
1618 struct compat_loop_info __user *arg)
1619{
1620 struct compat_loop_info info;
1621
1622 memset(&info, 0, sizeof(info));
1623 info.lo_number = info64->lo_number;
1624 info.lo_device = info64->lo_device;
1625 info.lo_inode = info64->lo_inode;
1626 info.lo_rdevice = info64->lo_rdevice;
1627 info.lo_offset = info64->lo_offset;
1628 info.lo_flags = info64->lo_flags;
1629 memcpy(info.lo_name, info64->lo_file_name, LO_NAME_SIZE);
1630
1631 /* error in case values were truncated */
1632 if (info.lo_device != info64->lo_device ||
1633 info.lo_rdevice != info64->lo_rdevice ||
1634 info.lo_inode != info64->lo_inode ||
1635 info.lo_offset != info64->lo_offset)
1636 return -EOVERFLOW;
1637
1638 if (copy_to_user(arg, &info, sizeof(info)))
1639 return -EFAULT;
1640 return 0;
1641}
1642
1643static int
1644loop_set_status_compat(struct loop_device *lo,
1645 const struct compat_loop_info __user *arg)
1646{
1647 struct loop_info64 info64;
1648 int ret;
1649
1650 ret = loop_info64_from_compat(arg, &info64);
1651 if (ret < 0)
1652 return ret;
1653 return loop_set_status(lo, &info64);
1654}
1655
1656static int
1657loop_get_status_compat(struct loop_device *lo,
1658 struct compat_loop_info __user *arg)
1659{
1660 struct loop_info64 info64;
1661 int err;
1662
1663 if (!arg)
1664 return -EINVAL;
1665 err = loop_get_status(lo, &info64);
1666 if (!err)
1667 err = loop_info64_to_compat(&info64, arg);
1668 return err;
1669}
1670
1671static int lo_compat_ioctl(struct block_device *bdev, blk_mode_t mode,
1672 unsigned int cmd, unsigned long arg)
1673{
1674 struct loop_device *lo = bdev->bd_disk->private_data;
1675 int err;
1676
1677 switch(cmd) {
1678 case LOOP_SET_STATUS:
1679 err = loop_set_status_compat(lo,
1680 (const struct compat_loop_info __user *)arg);
1681 break;
1682 case LOOP_GET_STATUS:
1683 err = loop_get_status_compat(lo,
1684 (struct compat_loop_info __user *)arg);
1685 break;
1686 case LOOP_SET_CAPACITY:
1687 case LOOP_CLR_FD:
1688 case LOOP_GET_STATUS64:
1689 case LOOP_SET_STATUS64:
1690 case LOOP_CONFIGURE:
1691 arg = (unsigned long) compat_ptr(arg);
1692 fallthrough;
1693 case LOOP_SET_FD:
1694 case LOOP_CHANGE_FD:
1695 case LOOP_SET_BLOCK_SIZE:
1696 case LOOP_SET_DIRECT_IO:
1697 err = lo_ioctl(bdev, mode, cmd, arg);
1698 break;
1699 default:
1700 err = -ENOIOCTLCMD;
1701 break;
1702 }
1703 return err;
1704}
1705#endif
1706
1707static int lo_open(struct gendisk *disk, blk_mode_t mode)
1708{
1709 struct loop_device *lo = disk->private_data;
1710 int err;
1711
1712 err = mutex_lock_killable(&lo->lo_mutex);
1713 if (err)
1714 return err;
1715
1716 if (lo->lo_state == Lo_deleting || lo->lo_state == Lo_rundown)
1717 err = -ENXIO;
1718 mutex_unlock(&lo->lo_mutex);
1719 return err;
1720}
1721
1722static void lo_release(struct gendisk *disk)
1723{
1724 struct loop_device *lo = disk->private_data;
1725 bool need_clear = false;
1726
1727 if (disk_openers(disk) > 0)
1728 return;
1729 /*
1730 * Clear the backing device information if this is the last close of
1731 * a device that's been marked for auto clear, or on which LOOP_CLR_FD
1732 * has been called.
1733 */
1734
1735 mutex_lock(&lo->lo_mutex);
1736 if (lo->lo_state == Lo_bound && (lo->lo_flags & LO_FLAGS_AUTOCLEAR))
1737 WRITE_ONCE(lo->lo_state, Lo_rundown);
1738
1739 need_clear = (lo->lo_state == Lo_rundown);
1740 mutex_unlock(&lo->lo_mutex);
1741
1742 if (need_clear)
1743 __loop_clr_fd(lo);
1744}
1745
1746static void lo_free_disk(struct gendisk *disk)
1747{
1748 struct loop_device *lo = disk->private_data;
1749
1750 if (lo->workqueue)
1751 destroy_workqueue(lo->workqueue);
1752 loop_free_idle_workers(lo, true);
1753 timer_shutdown_sync(&lo->timer);
1754 mutex_destroy(&lo->lo_mutex);
1755 kfree(lo);
1756}
1757
1758static const struct block_device_operations lo_fops = {
1759 .owner = THIS_MODULE,
1760 .open = lo_open,
1761 .release = lo_release,
1762 .ioctl = lo_ioctl,
1763#ifdef CONFIG_COMPAT
1764 .compat_ioctl = lo_compat_ioctl,
1765#endif
1766 .free_disk = lo_free_disk,
1767};
1768
1769/*
1770 * And now the modules code and kernel interface.
1771 */
1772
1773/*
1774 * If max_loop is specified, create that many devices upfront.
1775 * This also becomes a hard limit. If max_loop is not specified,
1776 * the default isn't a hard limit (as before commit 85c50197716c
1777 * changed the default value from 0 for max_loop=0 reasons), just
1778 * create CONFIG_BLK_DEV_LOOP_MIN_COUNT loop devices at module
1779 * init time. Loop devices can be requested on-demand with the
1780 * /dev/loop-control interface, or be instantiated by accessing
1781 * a 'dead' device node.
1782 */
1783static int max_loop = CONFIG_BLK_DEV_LOOP_MIN_COUNT;
1784
1785#ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD
1786static bool max_loop_specified;
1787
1788static int max_loop_param_set_int(const char *val,
1789 const struct kernel_param *kp)
1790{
1791 int ret;
1792
1793 ret = param_set_int(val, kp);
1794 if (ret < 0)
1795 return ret;
1796
1797 max_loop_specified = true;
1798 return 0;
1799}
1800
1801static const struct kernel_param_ops max_loop_param_ops = {
1802 .set = max_loop_param_set_int,
1803 .get = param_get_int,
1804};
1805
1806module_param_cb(max_loop, &max_loop_param_ops, &max_loop, 0444);
1807MODULE_PARM_DESC(max_loop, "Maximum number of loop devices");
1808#else
1809module_param(max_loop, int, 0444);
1810MODULE_PARM_DESC(max_loop, "Initial number of loop devices");
1811#endif
1812
1813module_param(max_part, int, 0444);
1814MODULE_PARM_DESC(max_part, "Maximum number of partitions per loop device");
1815
1816static int hw_queue_depth = LOOP_DEFAULT_HW_Q_DEPTH;
1817
1818static int loop_set_hw_queue_depth(const char *s, const struct kernel_param *p)
1819{
1820 int qd, ret;
1821
1822 ret = kstrtoint(s, 0, &qd);
1823 if (ret < 0)
1824 return ret;
1825 if (qd < 1)
1826 return -EINVAL;
1827 hw_queue_depth = qd;
1828 return 0;
1829}
1830
1831static const struct kernel_param_ops loop_hw_qdepth_param_ops = {
1832 .set = loop_set_hw_queue_depth,
1833 .get = param_get_int,
1834};
1835
1836device_param_cb(hw_queue_depth, &loop_hw_qdepth_param_ops, &hw_queue_depth, 0444);
1837MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: " __stringify(LOOP_DEFAULT_HW_Q_DEPTH));
1838
1839MODULE_DESCRIPTION("Loopback device support");
1840MODULE_LICENSE("GPL");
1841MODULE_ALIAS_BLOCKDEV_MAJOR(LOOP_MAJOR);
1842
1843static blk_status_t loop_queue_rq(struct blk_mq_hw_ctx *hctx,
1844 const struct blk_mq_queue_data *bd)
1845{
1846 struct request *rq = bd->rq;
1847 struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
1848 struct loop_device *lo = rq->q->queuedata;
1849
1850 blk_mq_start_request(rq);
1851
1852 if (data_race(READ_ONCE(lo->lo_state)) != Lo_bound)
1853 return BLK_STS_IOERR;
1854
1855 switch (req_op(rq)) {
1856 case REQ_OP_FLUSH:
1857 case REQ_OP_DISCARD:
1858 case REQ_OP_WRITE_ZEROES:
1859 cmd->use_aio = false;
1860 break;
1861 default:
1862 cmd->use_aio = lo->lo_flags & LO_FLAGS_DIRECT_IO;
1863 break;
1864 }
1865
1866 /* always use the first bio's css */
1867 cmd->blkcg_css = NULL;
1868 cmd->memcg_css = NULL;
1869#ifdef CONFIG_BLK_CGROUP
1870 if (rq->bio) {
1871 cmd->blkcg_css = bio_blkcg_css(rq->bio);
1872#ifdef CONFIG_MEMCG
1873 if (cmd->blkcg_css) {
1874 cmd->memcg_css =
1875 cgroup_get_e_css(cmd->blkcg_css->cgroup,
1876 &memory_cgrp_subsys);
1877 }
1878#endif
1879 }
1880#endif
1881 loop_queue_work(lo, cmd);
1882
1883 return BLK_STS_OK;
1884}
1885
1886static void loop_handle_cmd(struct loop_cmd *cmd)
1887{
1888 struct cgroup_subsys_state *cmd_blkcg_css = cmd->blkcg_css;
1889 struct cgroup_subsys_state *cmd_memcg_css = cmd->memcg_css;
1890 struct request *rq = blk_mq_rq_from_pdu(cmd);
1891 const bool write = op_is_write(req_op(rq));
1892 struct loop_device *lo = rq->q->queuedata;
1893 int ret = 0;
1894 struct mem_cgroup *old_memcg = NULL;
1895
1896 if (write && (lo->lo_flags & LO_FLAGS_READ_ONLY)) {
1897 ret = -EIO;
1898 goto failed;
1899 }
1900
1901 /* We can block in this context, so ignore REQ_NOWAIT. */
1902 if (rq->cmd_flags & REQ_NOWAIT)
1903 rq->cmd_flags &= ~REQ_NOWAIT;
1904
1905 if (cmd_blkcg_css)
1906 kthread_associate_blkcg(cmd_blkcg_css);
1907 if (cmd_memcg_css)
1908 old_memcg = set_active_memcg(
1909 mem_cgroup_from_css(cmd_memcg_css));
1910
1911 /*
1912 * do_req_filebacked() may call blk_mq_complete_request() synchronously
1913 * or asynchronously if using aio. Hence, do not touch 'cmd' after
1914 * do_req_filebacked() has returned unless we are sure that 'cmd' has
1915 * not yet been completed.
1916 */
1917 ret = do_req_filebacked(lo, rq);
1918
1919 if (cmd_blkcg_css)
1920 kthread_associate_blkcg(NULL);
1921
1922 if (cmd_memcg_css) {
1923 set_active_memcg(old_memcg);
1924 css_put(cmd_memcg_css);
1925 }
1926 failed:
1927 /* complete non-aio request */
1928 if (ret != -EIOCBQUEUED) {
1929 if (ret == -EOPNOTSUPP)
1930 cmd->ret = ret;
1931 else
1932 cmd->ret = ret ? -EIO : 0;
1933 if (likely(!blk_should_fake_timeout(rq->q)))
1934 blk_mq_complete_request(rq);
1935 }
1936}
1937
1938static void loop_process_work(struct loop_worker *worker,
1939 struct list_head *cmd_list, struct loop_device *lo)
1940{
1941 int orig_flags = current->flags;
1942 struct loop_cmd *cmd;
1943
1944 current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO;
1945 spin_lock_irq(&lo->lo_work_lock);
1946 while (!list_empty(cmd_list)) {
1947 cmd = container_of(
1948 cmd_list->next, struct loop_cmd, list_entry);
1949 list_del(cmd_list->next);
1950 spin_unlock_irq(&lo->lo_work_lock);
1951
1952 loop_handle_cmd(cmd);
1953 cond_resched();
1954
1955 spin_lock_irq(&lo->lo_work_lock);
1956 }
1957
1958 /*
1959 * We only add to the idle list if there are no pending cmds
1960 * *and* the worker will not run again which ensures that it
1961 * is safe to free any worker on the idle list
1962 */
1963 if (worker && !work_pending(&worker->work)) {
1964 worker->last_ran_at = jiffies;
1965 list_add_tail(&worker->idle_list, &lo->idle_worker_list);
1966 loop_set_timer(lo);
1967 }
1968 spin_unlock_irq(&lo->lo_work_lock);
1969 current->flags = orig_flags;
1970}
1971
1972static void loop_workfn(struct work_struct *work)
1973{
1974 struct loop_worker *worker =
1975 container_of(work, struct loop_worker, work);
1976 loop_process_work(worker, &worker->cmd_list, worker->lo);
1977}
1978
1979static void loop_rootcg_workfn(struct work_struct *work)
1980{
1981 struct loop_device *lo =
1982 container_of(work, struct loop_device, rootcg_work);
1983 loop_process_work(NULL, &lo->rootcg_cmd_list, lo);
1984}
1985
1986static const struct blk_mq_ops loop_mq_ops = {
1987 .queue_rq = loop_queue_rq,
1988 .complete = lo_complete_rq,
1989};
1990
1991static int loop_add(int i)
1992{
1993 struct queue_limits lim = {
1994 /*
1995 * Random number picked from the historic block max_sectors cap.
1996 */
1997 .max_hw_sectors = 2560u,
1998 };
1999 struct loop_device *lo;
2000 struct gendisk *disk;
2001 int err;
2002
2003 err = -ENOMEM;
2004 lo = kzalloc_obj(*lo);
2005 if (!lo)
2006 goto out;
2007 lo->worker_tree = RB_ROOT;
2008 INIT_LIST_HEAD(&lo->idle_worker_list);
2009 timer_setup(&lo->timer, loop_free_idle_workers_timer, TIMER_DEFERRABLE);
2010 WRITE_ONCE(lo->lo_state, Lo_unbound);
2011
2012 err = mutex_lock_killable(&loop_ctl_mutex);
2013 if (err)
2014 goto out_free_dev;
2015
2016 /* allocate id, if @id >= 0, we're requesting that specific id */
2017 if (i >= 0) {
2018 err = idr_alloc(&loop_index_idr, lo, i, i + 1, GFP_KERNEL);
2019 if (err == -ENOSPC)
2020 err = -EEXIST;
2021 } else {
2022 err = idr_alloc(&loop_index_idr, lo, 0, 0, GFP_KERNEL);
2023 }
2024 mutex_unlock(&loop_ctl_mutex);
2025 if (err < 0)
2026 goto out_free_dev;
2027 i = err;
2028
2029 lo->tag_set.ops = &loop_mq_ops;
2030 lo->tag_set.nr_hw_queues = 1;
2031 lo->tag_set.queue_depth = hw_queue_depth;
2032 lo->tag_set.numa_node = NUMA_NO_NODE;
2033 lo->tag_set.cmd_size = sizeof(struct loop_cmd);
2034 lo->tag_set.flags = BLK_MQ_F_STACKING | BLK_MQ_F_NO_SCHED_BY_DEFAULT;
2035 lo->tag_set.driver_data = lo;
2036
2037 err = blk_mq_alloc_tag_set(&lo->tag_set);
2038 if (err)
2039 goto out_free_idr;
2040
2041 disk = lo->lo_disk = blk_mq_alloc_disk(&lo->tag_set, &lim, lo);
2042 if (IS_ERR(disk)) {
2043 err = PTR_ERR(disk);
2044 goto out_cleanup_tags;
2045 }
2046 lo->lo_queue = lo->lo_disk->queue;
2047
2048 /*
2049 * Disable partition scanning by default. The in-kernel partition
2050 * scanning can be requested individually per-device during its
2051 * setup. Userspace can always add and remove partitions from all
2052 * devices. The needed partition minors are allocated from the
2053 * extended minor space, the main loop device numbers will continue
2054 * to match the loop minors, regardless of the number of partitions
2055 * used.
2056 *
2057 * If max_part is given, partition scanning is globally enabled for
2058 * all loop devices. The minors for the main loop devices will be
2059 * multiples of max_part.
2060 *
2061 * Note: Global-for-all-devices, set-only-at-init, read-only module
2062 * parameteters like 'max_loop' and 'max_part' make things needlessly
2063 * complicated, are too static, inflexible and may surprise
2064 * userspace tools. Parameters like this in general should be avoided.
2065 */
2066 if (!part_shift)
2067 set_bit(GD_SUPPRESS_PART_SCAN, &disk->state);
2068 mutex_init(&lo->lo_mutex);
2069 lo->lo_number = i;
2070 spin_lock_init(&lo->lo_lock);
2071 spin_lock_init(&lo->lo_work_lock);
2072 INIT_WORK(&lo->rootcg_work, loop_rootcg_workfn);
2073 INIT_LIST_HEAD(&lo->rootcg_cmd_list);
2074 disk->major = LOOP_MAJOR;
2075 disk->first_minor = i << part_shift;
2076 disk->minors = 1 << part_shift;
2077 disk->fops = &lo_fops;
2078 disk->private_data = lo;
2079 disk->queue = lo->lo_queue;
2080 disk->events = DISK_EVENT_MEDIA_CHANGE;
2081 disk->event_flags = DISK_EVENT_FLAG_UEVENT;
2082 sprintf(disk->disk_name, "loop%d", i);
2083 /* Make this loop device reachable from pathname. */
2084 err = add_disk(disk);
2085 if (err)
2086 goto out_cleanup_disk;
2087
2088 /* Show this loop device. */
2089 mutex_lock(&loop_ctl_mutex);
2090 lo->idr_visible = true;
2091 mutex_unlock(&loop_ctl_mutex);
2092
2093 return i;
2094
2095out_cleanup_disk:
2096 put_disk(disk);
2097out_cleanup_tags:
2098 blk_mq_free_tag_set(&lo->tag_set);
2099out_free_idr:
2100 mutex_lock(&loop_ctl_mutex);
2101 idr_remove(&loop_index_idr, i);
2102 mutex_unlock(&loop_ctl_mutex);
2103out_free_dev:
2104 kfree(lo);
2105out:
2106 return err;
2107}
2108
2109static void loop_remove(struct loop_device *lo)
2110{
2111 /* Make this loop device unreachable from pathname. */
2112 del_gendisk(lo->lo_disk);
2113 blk_mq_free_tag_set(&lo->tag_set);
2114
2115 mutex_lock(&loop_ctl_mutex);
2116 idr_remove(&loop_index_idr, lo->lo_number);
2117 mutex_unlock(&loop_ctl_mutex);
2118
2119 put_disk(lo->lo_disk);
2120}
2121
2122#ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD
2123static void loop_probe(dev_t dev)
2124{
2125 int idx = MINOR(dev) >> part_shift;
2126
2127 if (max_loop_specified && max_loop && idx >= max_loop)
2128 return;
2129 loop_add(idx);
2130}
2131#else
2132#define loop_probe NULL
2133#endif /* !CONFIG_BLOCK_LEGACY_AUTOLOAD */
2134
2135static int loop_control_remove(int idx)
2136{
2137 struct loop_device *lo;
2138 int ret;
2139
2140 if (idx < 0) {
2141 pr_warn_once("deleting an unspecified loop device is not supported.\n");
2142 return -EINVAL;
2143 }
2144
2145 /* Hide this loop device for serialization. */
2146 ret = mutex_lock_killable(&loop_ctl_mutex);
2147 if (ret)
2148 return ret;
2149 lo = idr_find(&loop_index_idr, idx);
2150 if (!lo || !lo->idr_visible)
2151 ret = -ENODEV;
2152 else
2153 lo->idr_visible = false;
2154 mutex_unlock(&loop_ctl_mutex);
2155 if (ret)
2156 return ret;
2157
2158 /* Check whether this loop device can be removed. */
2159 ret = mutex_lock_killable(&lo->lo_mutex);
2160 if (ret)
2161 goto mark_visible;
2162 if (lo->lo_state != Lo_unbound || disk_openers(lo->lo_disk) > 0) {
2163 mutex_unlock(&lo->lo_mutex);
2164 ret = -EBUSY;
2165 goto mark_visible;
2166 }
2167 /* Mark this loop device as no more bound, but not quite unbound yet */
2168 WRITE_ONCE(lo->lo_state, Lo_deleting);
2169 mutex_unlock(&lo->lo_mutex);
2170
2171 loop_remove(lo);
2172 return 0;
2173
2174mark_visible:
2175 /* Show this loop device again. */
2176 mutex_lock(&loop_ctl_mutex);
2177 lo->idr_visible = true;
2178 mutex_unlock(&loop_ctl_mutex);
2179 return ret;
2180}
2181
2182static int loop_control_get_free(int idx)
2183{
2184 struct loop_device *lo;
2185 int id, ret;
2186
2187 ret = mutex_lock_killable(&loop_ctl_mutex);
2188 if (ret)
2189 return ret;
2190 idr_for_each_entry(&loop_index_idr, lo, id) {
2191 /*
2192 * Hitting a race results in creating a new loop device
2193 * which is harmless.
2194 */
2195 if (lo->idr_visible &&
2196 data_race(READ_ONCE(lo->lo_state)) == Lo_unbound)
2197 goto found;
2198 }
2199 mutex_unlock(&loop_ctl_mutex);
2200 return loop_add(-1);
2201found:
2202 mutex_unlock(&loop_ctl_mutex);
2203 return id;
2204}
2205
2206static long loop_control_ioctl(struct file *file, unsigned int cmd,
2207 unsigned long parm)
2208{
2209 switch (cmd) {
2210 case LOOP_CTL_ADD:
2211 return loop_add(parm);
2212 case LOOP_CTL_REMOVE:
2213 return loop_control_remove(parm);
2214 case LOOP_CTL_GET_FREE:
2215 return loop_control_get_free(parm);
2216 default:
2217 return -ENOSYS;
2218 }
2219}
2220
2221static const struct file_operations loop_ctl_fops = {
2222 .open = nonseekable_open,
2223 .unlocked_ioctl = loop_control_ioctl,
2224 .compat_ioctl = loop_control_ioctl,
2225 .owner = THIS_MODULE,
2226 .llseek = noop_llseek,
2227};
2228
2229static struct miscdevice loop_misc = {
2230 .minor = LOOP_CTRL_MINOR,
2231 .name = "loop-control",
2232 .fops = &loop_ctl_fops,
2233};
2234
2235MODULE_ALIAS_MISCDEV(LOOP_CTRL_MINOR);
2236MODULE_ALIAS("devname:loop-control");
2237
2238static int __init loop_init(void)
2239{
2240 int i;
2241 int err;
2242
2243 part_shift = 0;
2244 if (max_part > 0) {
2245 part_shift = fls(max_part);
2246
2247 /*
2248 * Adjust max_part according to part_shift as it is exported
2249 * to user space so that user can decide correct minor number
2250 * if [s]he want to create more devices.
2251 *
2252 * Note that -1 is required because partition 0 is reserved
2253 * for the whole disk.
2254 */
2255 max_part = (1UL << part_shift) - 1;
2256 }
2257
2258 if ((1UL << part_shift) > DISK_MAX_PARTS) {
2259 err = -EINVAL;
2260 goto err_out;
2261 }
2262
2263 if (max_loop > 1UL << (MINORBITS - part_shift)) {
2264 err = -EINVAL;
2265 goto err_out;
2266 }
2267
2268 err = misc_register(&loop_misc);
2269 if (err < 0)
2270 goto err_out;
2271
2272
2273 if (__register_blkdev(LOOP_MAJOR, "loop", loop_probe)) {
2274 err = -EIO;
2275 goto misc_out;
2276 }
2277
2278 /* pre-create number of devices given by config or max_loop */
2279 for (i = 0; i < max_loop; i++)
2280 loop_add(i);
2281
2282 printk(KERN_INFO "loop: module loaded\n");
2283 return 0;
2284
2285misc_out:
2286 misc_deregister(&loop_misc);
2287err_out:
2288 return err;
2289}
2290
2291static void __exit loop_exit(void)
2292{
2293 struct loop_device *lo;
2294 int id;
2295
2296 unregister_blkdev(LOOP_MAJOR, "loop");
2297 misc_deregister(&loop_misc);
2298
2299 /*
2300 * There is no need to use loop_ctl_mutex here, for nobody else can
2301 * access loop_index_idr when this module is unloading (unless forced
2302 * module unloading is requested). If this is not a clean unloading,
2303 * we have no means to avoid kernel crash.
2304 */
2305 idr_for_each_entry(&loop_index_idr, lo, id)
2306 loop_remove(lo);
2307
2308 idr_destroy(&loop_index_idr);
2309}
2310
2311module_init(loop_init);
2312module_exit(loop_exit);
2313
2314#ifndef MODULE
2315static int __init max_loop_setup(char *str)
2316{
2317 max_loop = simple_strtol(str, NULL, 0);
2318#ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD
2319 max_loop_specified = true;
2320#endif
2321 return 1;
2322}
2323
2324__setup("max_loop=", max_loop_setup);
2325#endif