projects/linux·blob v4.1 · refs

1869 lines · 45.1 KiB

1/*
2 * linux/drivers/block/loop.c
3 *
4 * Written by Theodore Ts'o, 3/29/93
5 *
6 * Copyright 1993 by Theodore Ts'o. Redistribution of this file is
7 * permitted under the GNU General Public License.
8 *
9 * DES encryption plus some minor changes by Werner Almesberger, 30-MAY-1993
10 * more DES encryption plus IDEA encryption by Nicholas J. Leon, June 20, 1996
11 *
12 * Modularized and updated for 1.1.16 kernel - Mitch Dsouza 28th May 1994
13 * Adapted for 1.3.59 kernel - Andries Brouwer, 1 Feb 1996
14 *
15 * Fixed do_loop_request() re-entrancy - [email protected] Mar 20, 1997
16 *
17 * Added devfs support - Richard Gooch <[email protected]> 16-Jan-1998
18 *
19 * Handle sparse backing files correctly - Kenn Humborg, Jun 28, 1998
20 *
21 * Loadable modules and other fixes by AK, 1998
22 *
23 * Make real block number available to downstream transfer functions, enables
24 * CBC (and relatives) mode encryption requiring unique IVs per data block.
25 * Reed H. Petty, [email protected]
26 *
27 * Maximum number of loop devices now dynamic via max_loop module parameter.
28 * Russell Kroll <[email protected]> 19990701
29 *
30 * Maximum number of loop devices when compiled-in now selectable by passing
31 * max_loop=<1-255> to the kernel on boot.
32 * Erik I. Bolsø, <[email protected]>, Oct 31, 1999
33 *
34 * Completely rewrite request handling to be make_request_fn style and
35 * non blocking, pushing work to a helper thread. Lots of fixes from
36 * Al Viro too.
37 * Jens Axboe <[email protected]>, Nov 2000
38 *
39 * Support up to 256 loop devices
40 * Heinz Mauelshagen <[email protected]>, Feb 2002
41 *
42 * Support for falling back on the write file operation when the address space
43 * operations write_begin is not available on the backing filesystem.
44 * Anton Altaparmakov, 16 Feb 2005
45 *
46 * Still To Fix:
47 * - Advisory locking is ignored here.
48 * - Should use an own CAP_* category instead of CAP_SYS_ADMIN
49 *
50 */
51
52#include <linux/module.h>
53#include <linux/moduleparam.h>
54#include <linux/sched.h>
55#include <linux/fs.h>
56#include <linux/file.h>
57#include <linux/stat.h>
58#include <linux/errno.h>
59#include <linux/major.h>
60#include <linux/wait.h>
61#include <linux/blkdev.h>
62#include <linux/blkpg.h>
63#include <linux/init.h>
64#include <linux/swap.h>
65#include <linux/slab.h>
66#include <linux/compat.h>
67#include <linux/suspend.h>
68#include <linux/freezer.h>
69#include <linux/mutex.h>
70#include <linux/writeback.h>
71#include <linux/completion.h>
72#include <linux/highmem.h>
73#include <linux/kthread.h>
74#include <linux/splice.h>
75#include <linux/sysfs.h>
76#include <linux/miscdevice.h>
77#include <linux/falloc.h>
78#include <linux/uio.h>
79#include "loop.h"
80
81#include <asm/uaccess.h>
82
83static DEFINE_IDR(loop_index_idr);
84static DEFINE_MUTEX(loop_index_mutex);
85
86static int max_part;
87static int part_shift;
88
89static struct workqueue_struct *loop_wq;
90
91static int transfer_xor(struct loop_device *lo, int cmd,
92 struct page *raw_page, unsigned raw_off,
93 struct page *loop_page, unsigned loop_off,
94 int size, sector_t real_block)
95{
96 char *raw_buf = kmap_atomic(raw_page) + raw_off;
97 char *loop_buf = kmap_atomic(loop_page) + loop_off;
98 char *in, *out, *key;
99 int i, keysize;
100
101 if (cmd == READ) {
102 in = raw_buf;
103 out = loop_buf;
104 } else {
105 in = loop_buf;
106 out = raw_buf;
107 }
108
109 key = lo->lo_encrypt_key;
110 keysize = lo->lo_encrypt_key_size;
111 for (i = 0; i < size; i++)
112 *out++ = *in++ ^ key[(i & 511) % keysize];
113
114 kunmap_atomic(loop_buf);
115 kunmap_atomic(raw_buf);
116 cond_resched();
117 return 0;
118}
119
120static int xor_init(struct loop_device *lo, const struct loop_info64 *info)
121{
122 if (unlikely(info->lo_encrypt_key_size <= 0))
123 return -EINVAL;
124 return 0;
125}
126
127static struct loop_func_table none_funcs = {
128 .number = LO_CRYPT_NONE,
129};
130
131static struct loop_func_table xor_funcs = {
132 .number = LO_CRYPT_XOR,
133 .transfer = transfer_xor,
134 .init = xor_init
135};
136
137/* xfer_funcs[0] is special - its release function is never called */
138static struct loop_func_table *xfer_funcs[MAX_LO_CRYPT] = {
139 &none_funcs,
140 &xor_funcs
141};
142
143static loff_t get_size(loff_t offset, loff_t sizelimit, struct file *file)
144{
145 loff_t loopsize;
146
147 /* Compute loopsize in bytes */
148 loopsize = i_size_read(file->f_mapping->host);
149 if (offset > 0)
150 loopsize -= offset;
151 /* offset is beyond i_size, weird but possible */
152 if (loopsize < 0)
153 return 0;
154
155 if (sizelimit > 0 && sizelimit < loopsize)
156 loopsize = sizelimit;
157 /*
158 * Unfortunately, if we want to do I/O on the device,
159 * the number of 512-byte sectors has to fit into a sector_t.
160 */
161 return loopsize >> 9;
162}
163
164static loff_t get_loop_size(struct loop_device *lo, struct file *file)
165{
166 return get_size(lo->lo_offset, lo->lo_sizelimit, file);
167}
168
169static int
170figure_loop_size(struct loop_device *lo, loff_t offset, loff_t sizelimit)
171{
172 loff_t size = get_size(offset, sizelimit, lo->lo_backing_file);
173 sector_t x = (sector_t)size;
174 struct block_device *bdev = lo->lo_device;
175
176 if (unlikely((loff_t)x != size))
177 return -EFBIG;
178 if (lo->lo_offset != offset)
179 lo->lo_offset = offset;
180 if (lo->lo_sizelimit != sizelimit)
181 lo->lo_sizelimit = sizelimit;
182 set_capacity(lo->lo_disk, x);
183 bd_set_size(bdev, (loff_t)get_capacity(bdev->bd_disk) << 9);
184 /* let user-space know about the new size */
185 kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
186 return 0;
187}
188
189static inline int
190lo_do_transfer(struct loop_device *lo, int cmd,
191 struct page *rpage, unsigned roffs,
192 struct page *lpage, unsigned loffs,
193 int size, sector_t rblock)
194{
195 int ret;
196
197 ret = lo->transfer(lo, cmd, rpage, roffs, lpage, loffs, size, rblock);
198 if (likely(!ret))
199 return 0;
200
201 printk_ratelimited(KERN_ERR
202 "loop: Transfer error at byte offset %llu, length %i.\n",
203 (unsigned long long)rblock << 9, size);
204 return ret;
205}
206
207static int lo_write_bvec(struct file *file, struct bio_vec *bvec, loff_t *ppos)
208{
209 struct iov_iter i;
210 ssize_t bw;
211
212 iov_iter_bvec(&i, ITER_BVEC, bvec, 1, bvec->bv_len);
213
214 file_start_write(file);
215 bw = vfs_iter_write(file, &i, ppos);
216 file_end_write(file);
217
218 if (likely(bw == bvec->bv_len))
219 return 0;
220
221 printk_ratelimited(KERN_ERR
222 "loop: Write error at byte offset %llu, length %i.\n",
223 (unsigned long long)*ppos, bvec->bv_len);
224 if (bw >= 0)
225 bw = -EIO;
226 return bw;
227}
228
229static int lo_write_simple(struct loop_device *lo, struct request *rq,
230 loff_t pos)
231{
232 struct bio_vec bvec;
233 struct req_iterator iter;
234 int ret = 0;
235
236 rq_for_each_segment(bvec, rq, iter) {
237 ret = lo_write_bvec(lo->lo_backing_file, &bvec, &pos);
238 if (ret < 0)
239 break;
240 cond_resched();
241 }
242
243 return ret;
244}
245
246/*
247 * This is the slow, transforming version that needs to double buffer the
248 * data as it cannot do the transformations in place without having direct
249 * access to the destination pages of the backing file.
250 */
251static int lo_write_transfer(struct loop_device *lo, struct request *rq,
252 loff_t pos)
253{
254 struct bio_vec bvec, b;
255 struct req_iterator iter;
256 struct page *page;
257 int ret = 0;
258
259 page = alloc_page(GFP_NOIO);
260 if (unlikely(!page))
261 return -ENOMEM;
262
263 rq_for_each_segment(bvec, rq, iter) {
264 ret = lo_do_transfer(lo, WRITE, page, 0, bvec.bv_page,
265 bvec.bv_offset, bvec.bv_len, pos >> 9);
266 if (unlikely(ret))
267 break;
268
269 b.bv_page = page;
270 b.bv_offset = 0;
271 b.bv_len = bvec.bv_len;
272 ret = lo_write_bvec(lo->lo_backing_file, &b, &pos);
273 if (ret < 0)
274 break;
275 }
276
277 __free_page(page);
278 return ret;
279}
280
281static int lo_read_simple(struct loop_device *lo, struct request *rq,
282 loff_t pos)
283{
284 struct bio_vec bvec;
285 struct req_iterator iter;
286 struct iov_iter i;
287 ssize_t len;
288
289 rq_for_each_segment(bvec, rq, iter) {
290 iov_iter_bvec(&i, ITER_BVEC, &bvec, 1, bvec.bv_len);
291 len = vfs_iter_read(lo->lo_backing_file, &i, &pos);
292 if (len < 0)
293 return len;
294
295 flush_dcache_page(bvec.bv_page);
296
297 if (len != bvec.bv_len) {
298 struct bio *bio;
299
300 __rq_for_each_bio(bio, rq)
301 zero_fill_bio(bio);
302 break;
303 }
304 cond_resched();
305 }
306
307 return 0;
308}
309
310static int lo_read_transfer(struct loop_device *lo, struct request *rq,
311 loff_t pos)
312{
313 struct bio_vec bvec, b;
314 struct req_iterator iter;
315 struct iov_iter i;
316 struct page *page;
317 ssize_t len;
318 int ret = 0;
319
320 page = alloc_page(GFP_NOIO);
321 if (unlikely(!page))
322 return -ENOMEM;
323
324 rq_for_each_segment(bvec, rq, iter) {
325 loff_t offset = pos;
326
327 b.bv_page = page;
328 b.bv_offset = 0;
329 b.bv_len = bvec.bv_len;
330
331 iov_iter_bvec(&i, ITER_BVEC, &b, 1, b.bv_len);
332 len = vfs_iter_read(lo->lo_backing_file, &i, &pos);
333 if (len < 0) {
334 ret = len;
335 goto out_free_page;
336 }
337
338 ret = lo_do_transfer(lo, READ, page, 0, bvec.bv_page,
339 bvec.bv_offset, len, offset >> 9);
340 if (ret)
341 goto out_free_page;
342
343 flush_dcache_page(bvec.bv_page);
344
345 if (len != bvec.bv_len) {
346 struct bio *bio;
347
348 __rq_for_each_bio(bio, rq)
349 zero_fill_bio(bio);
350 break;
351 }
352 }
353
354 ret = 0;
355out_free_page:
356 __free_page(page);
357 return ret;
358}
359
360static int lo_discard(struct loop_device *lo, struct request *rq, loff_t pos)
361{
362 /*
363 * We use punch hole to reclaim the free space used by the
364 * image a.k.a. discard. However we do not support discard if
365 * encryption is enabled, because it may give an attacker
366 * useful information.
367 */
368 struct file *file = lo->lo_backing_file;
369 int mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE;
370 int ret;
371
372 if ((!file->f_op->fallocate) || lo->lo_encrypt_key_size) {
373 ret = -EOPNOTSUPP;
374 goto out;
375 }
376
377 ret = file->f_op->fallocate(file, mode, pos, blk_rq_bytes(rq));
378 if (unlikely(ret && ret != -EINVAL && ret != -EOPNOTSUPP))
379 ret = -EIO;
380 out:
381 return ret;
382}
383
384static int lo_req_flush(struct loop_device *lo, struct request *rq)
385{
386 struct file *file = lo->lo_backing_file;
387 int ret = vfs_fsync(file, 0);
388 if (unlikely(ret && ret != -EINVAL))
389 ret = -EIO;
390
391 return ret;
392}
393
394static int do_req_filebacked(struct loop_device *lo, struct request *rq)
395{
396 loff_t pos;
397 int ret;
398
399 pos = ((loff_t) blk_rq_pos(rq) << 9) + lo->lo_offset;
400
401 if (rq->cmd_flags & REQ_WRITE) {
402 if (rq->cmd_flags & REQ_FLUSH)
403 ret = lo_req_flush(lo, rq);
404 else if (rq->cmd_flags & REQ_DISCARD)
405 ret = lo_discard(lo, rq, pos);
406 else if (lo->transfer)
407 ret = lo_write_transfer(lo, rq, pos);
408 else
409 ret = lo_write_simple(lo, rq, pos);
410
411 } else {
412 if (lo->transfer)
413 ret = lo_read_transfer(lo, rq, pos);
414 else
415 ret = lo_read_simple(lo, rq, pos);
416 }
417
418 return ret;
419}
420
421struct switch_request {
422 struct file *file;
423 struct completion wait;
424};
425
426/*
427 * Do the actual switch; called from the BIO completion routine
428 */
429static void do_loop_switch(struct loop_device *lo, struct switch_request *p)
430{
431 struct file *file = p->file;
432 struct file *old_file = lo->lo_backing_file;
433 struct address_space *mapping;
434
435 /* if no new file, only flush of queued bios requested */
436 if (!file)
437 return;
438
439 mapping = file->f_mapping;
440 mapping_set_gfp_mask(old_file->f_mapping, lo->old_gfp_mask);
441 lo->lo_backing_file = file;
442 lo->lo_blocksize = S_ISBLK(mapping->host->i_mode) ?
443 mapping->host->i_bdev->bd_block_size : PAGE_SIZE;
444 lo->old_gfp_mask = mapping_gfp_mask(mapping);
445 mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));
446}
447
448/*
449 * loop_switch performs the hard work of switching a backing store.
450 * First it needs to flush existing IO, it does this by sending a magic
451 * BIO down the pipe. The completion of this BIO does the actual switch.
452 */
453static int loop_switch(struct loop_device *lo, struct file *file)
454{
455 struct switch_request w;
456
457 w.file = file;
458
459 /* freeze queue and wait for completion of scheduled requests */
460 blk_mq_freeze_queue(lo->lo_queue);
461
462 /* do the switch action */
463 do_loop_switch(lo, &w);
464
465 /* unfreeze */
466 blk_mq_unfreeze_queue(lo->lo_queue);
467
468 return 0;
469}
470
471/*
472 * Helper to flush the IOs in loop, but keeping loop thread running
473 */
474static int loop_flush(struct loop_device *lo)
475{
476 return loop_switch(lo, NULL);
477}
478
479/*
480 * loop_change_fd switched the backing store of a loopback device to
481 * a new file. This is useful for operating system installers to free up
482 * the original file and in High Availability environments to switch to
483 * an alternative location for the content in case of server meltdown.
484 * This can only work if the loop device is used read-only, and if the
485 * new backing store is the same size and type as the old backing store.
486 */
487static int loop_change_fd(struct loop_device *lo, struct block_device *bdev,
488 unsigned int arg)
489{
490 struct file *file, *old_file;
491 struct inode *inode;
492 int error;
493
494 error = -ENXIO;
495 if (lo->lo_state != Lo_bound)
496 goto out;
497
498 /* the loop device has to be read-only */
499 error = -EINVAL;
500 if (!(lo->lo_flags & LO_FLAGS_READ_ONLY))
501 goto out;
502
503 error = -EBADF;
504 file = fget(arg);
505 if (!file)
506 goto out;
507
508 inode = file->f_mapping->host;
509 old_file = lo->lo_backing_file;
510
511 error = -EINVAL;
512
513 if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode))
514 goto out_putf;
515
516 /* size of the new backing store needs to be the same */
517 if (get_loop_size(lo, file) != get_loop_size(lo, old_file))
518 goto out_putf;
519
520 /* and ... switch */
521 error = loop_switch(lo, file);
522 if (error)
523 goto out_putf;
524
525 fput(old_file);
526 if (lo->lo_flags & LO_FLAGS_PARTSCAN)
527 ioctl_by_bdev(bdev, BLKRRPART, 0);
528 return 0;
529
530 out_putf:
531 fput(file);
532 out:
533 return error;
534}
535
536static inline int is_loop_device(struct file *file)
537{
538 struct inode *i = file->f_mapping->host;
539
540 return i && S_ISBLK(i->i_mode) && MAJOR(i->i_rdev) == LOOP_MAJOR;
541}
542
543/* loop sysfs attributes */
544
545static ssize_t loop_attr_show(struct device *dev, char *page,
546 ssize_t (*callback)(struct loop_device *, char *))
547{
548 struct gendisk *disk = dev_to_disk(dev);
549 struct loop_device *lo = disk->private_data;
550
551 return callback(lo, page);
552}
553
554#define LOOP_ATTR_RO(_name) \
555static ssize_t loop_attr_##_name##_show(struct loop_device *, char *); \
556static ssize_t loop_attr_do_show_##_name(struct device *d, \
557 struct device_attribute *attr, char *b) \
558{ \
559 return loop_attr_show(d, b, loop_attr_##_name##_show); \
560} \
561static struct device_attribute loop_attr_##_name = \
562 __ATTR(_name, S_IRUGO, loop_attr_do_show_##_name, NULL);
563
564static ssize_t loop_attr_backing_file_show(struct loop_device *lo, char *buf)
565{
566 ssize_t ret;
567 char *p = NULL;
568
569 spin_lock_irq(&lo->lo_lock);
570 if (lo->lo_backing_file)
571 p = d_path(&lo->lo_backing_file->f_path, buf, PAGE_SIZE - 1);
572 spin_unlock_irq(&lo->lo_lock);
573
574 if (IS_ERR_OR_NULL(p))
575 ret = PTR_ERR(p);
576 else {
577 ret = strlen(p);
578 memmove(buf, p, ret);
579 buf[ret++] = '\n';
580 buf[ret] = 0;
581 }
582
583 return ret;
584}
585
586static ssize_t loop_attr_offset_show(struct loop_device *lo, char *buf)
587{
588 return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_offset);
589}
590
591static ssize_t loop_attr_sizelimit_show(struct loop_device *lo, char *buf)
592{
593 return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_sizelimit);
594}
595
596static ssize_t loop_attr_autoclear_show(struct loop_device *lo, char *buf)
597{
598 int autoclear = (lo->lo_flags & LO_FLAGS_AUTOCLEAR);
599
600 return sprintf(buf, "%s\n", autoclear ? "1" : "0");
601}
602
603static ssize_t loop_attr_partscan_show(struct loop_device *lo, char *buf)
604{
605 int partscan = (lo->lo_flags & LO_FLAGS_PARTSCAN);
606
607 return sprintf(buf, "%s\n", partscan ? "1" : "0");
608}
609
610LOOP_ATTR_RO(backing_file);
611LOOP_ATTR_RO(offset);
612LOOP_ATTR_RO(sizelimit);
613LOOP_ATTR_RO(autoclear);
614LOOP_ATTR_RO(partscan);
615
616static struct attribute *loop_attrs[] = {
617 &loop_attr_backing_file.attr,
618 &loop_attr_offset.attr,
619 &loop_attr_sizelimit.attr,
620 &loop_attr_autoclear.attr,
621 &loop_attr_partscan.attr,
622 NULL,
623};
624
625static struct attribute_group loop_attribute_group = {
626 .name = "loop",
627 .attrs= loop_attrs,
628};
629
630static int loop_sysfs_init(struct loop_device *lo)
631{
632 return sysfs_create_group(&disk_to_dev(lo->lo_disk)->kobj,
633 &loop_attribute_group);
634}
635
636static void loop_sysfs_exit(struct loop_device *lo)
637{
638 sysfs_remove_group(&disk_to_dev(lo->lo_disk)->kobj,
639 &loop_attribute_group);
640}
641
642static void loop_config_discard(struct loop_device *lo)
643{
644 struct file *file = lo->lo_backing_file;
645 struct inode *inode = file->f_mapping->host;
646 struct request_queue *q = lo->lo_queue;
647
648 /*
649 * We use punch hole to reclaim the free space used by the
650 * image a.k.a. discard. However we do not support discard if
651 * encryption is enabled, because it may give an attacker
652 * useful information.
653 */
654 if ((!file->f_op->fallocate) ||
655 lo->lo_encrypt_key_size) {
656 q->limits.discard_granularity = 0;
657 q->limits.discard_alignment = 0;
658 q->limits.max_discard_sectors = 0;
659 q->limits.discard_zeroes_data = 0;
660 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q);
661 return;
662 }
663
664 q->limits.discard_granularity = inode->i_sb->s_blocksize;
665 q->limits.discard_alignment = 0;
666 q->limits.max_discard_sectors = UINT_MAX >> 9;
667 q->limits.discard_zeroes_data = 1;
668 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
669}
670
671static int loop_set_fd(struct loop_device *lo, fmode_t mode,
672 struct block_device *bdev, unsigned int arg)
673{
674 struct file *file, *f;
675 struct inode *inode;
676 struct address_space *mapping;
677 unsigned lo_blocksize;
678 int lo_flags = 0;
679 int error;
680 loff_t size;
681
682 /* This is safe, since we have a reference from open(). */
683 __module_get(THIS_MODULE);
684
685 error = -EBADF;
686 file = fget(arg);
687 if (!file)
688 goto out;
689
690 error = -EBUSY;
691 if (lo->lo_state != Lo_unbound)
692 goto out_putf;
693
694 /* Avoid recursion */
695 f = file;
696 while (is_loop_device(f)) {
697 struct loop_device *l;
698
699 if (f->f_mapping->host->i_bdev == bdev)
700 goto out_putf;
701
702 l = f->f_mapping->host->i_bdev->bd_disk->private_data;
703 if (l->lo_state == Lo_unbound) {
704 error = -EINVAL;
705 goto out_putf;
706 }
707 f = l->lo_backing_file;
708 }
709
710 mapping = file->f_mapping;
711 inode = mapping->host;
712
713 error = -EINVAL;
714 if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode))
715 goto out_putf;
716
717 if (!(file->f_mode & FMODE_WRITE) || !(mode & FMODE_WRITE) ||
718 !file->f_op->write_iter)
719 lo_flags |= LO_FLAGS_READ_ONLY;
720
721 lo_blocksize = S_ISBLK(inode->i_mode) ?
722 inode->i_bdev->bd_block_size : PAGE_SIZE;
723
724 error = -EFBIG;
725 size = get_loop_size(lo, file);
726 if ((loff_t)(sector_t)size != size)
727 goto out_putf;
728
729 error = 0;
730
731 set_device_ro(bdev, (lo_flags & LO_FLAGS_READ_ONLY) != 0);
732
733 lo->lo_blocksize = lo_blocksize;
734 lo->lo_device = bdev;
735 lo->lo_flags = lo_flags;
736 lo->lo_backing_file = file;
737 lo->transfer = NULL;
738 lo->ioctl = NULL;
739 lo->lo_sizelimit = 0;
740 lo->old_gfp_mask = mapping_gfp_mask(mapping);
741 mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));
742
743 if (!(lo_flags & LO_FLAGS_READ_ONLY) && file->f_op->fsync)
744 blk_queue_flush(lo->lo_queue, REQ_FLUSH);
745
746 set_capacity(lo->lo_disk, size);
747 bd_set_size(bdev, size << 9);
748 loop_sysfs_init(lo);
749 /* let user-space know about the new size */
750 kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
751
752 set_blocksize(bdev, lo_blocksize);
753
754 lo->lo_state = Lo_bound;
755 if (part_shift)
756 lo->lo_flags |= LO_FLAGS_PARTSCAN;
757 if (lo->lo_flags & LO_FLAGS_PARTSCAN)
758 ioctl_by_bdev(bdev, BLKRRPART, 0);
759
760 /* Grab the block_device to prevent its destruction after we
761 * put /dev/loopXX inode. Later in loop_clr_fd() we bdput(bdev).
762 */
763 bdgrab(bdev);
764 return 0;
765
766 out_putf:
767 fput(file);
768 out:
769 /* This is safe: open() is still holding a reference. */
770 module_put(THIS_MODULE);
771 return error;
772}
773
774static int
775loop_release_xfer(struct loop_device *lo)
776{
777 int err = 0;
778 struct loop_func_table *xfer = lo->lo_encryption;
779
780 if (xfer) {
781 if (xfer->release)
782 err = xfer->release(lo);
783 lo->transfer = NULL;
784 lo->lo_encryption = NULL;
785 module_put(xfer->owner);
786 }
787 return err;
788}
789
790static int
791loop_init_xfer(struct loop_device *lo, struct loop_func_table *xfer,
792 const struct loop_info64 *i)
793{
794 int err = 0;
795
796 if (xfer) {
797 struct module *owner = xfer->owner;
798
799 if (!try_module_get(owner))
800 return -EINVAL;
801 if (xfer->init)
802 err = xfer->init(lo, i);
803 if (err)
804 module_put(owner);
805 else
806 lo->lo_encryption = xfer;
807 }
808 return err;
809}
810
811static int loop_clr_fd(struct loop_device *lo)
812{
813 struct file *filp = lo->lo_backing_file;
814 gfp_t gfp = lo->old_gfp_mask;
815 struct block_device *bdev = lo->lo_device;
816
817 if (lo->lo_state != Lo_bound)
818 return -ENXIO;
819
820 /*
821 * If we've explicitly asked to tear down the loop device,
822 * and it has an elevated reference count, set it for auto-teardown when
823 * the last reference goes away. This stops $!~#$@ udev from
824 * preventing teardown because it decided that it needs to run blkid on
825 * the loopback device whenever they appear. xfstests is notorious for
826 * failing tests because blkid via udev races with a losetup
827 * <dev>/do something like mkfs/losetup -d <dev> causing the losetup -d
828 * command to fail with EBUSY.
829 */
830 if (lo->lo_refcnt > 1) {
831 lo->lo_flags |= LO_FLAGS_AUTOCLEAR;
832 mutex_unlock(&lo->lo_ctl_mutex);
833 return 0;
834 }
835
836 if (filp == NULL)
837 return -EINVAL;
838
839 spin_lock_irq(&lo->lo_lock);
840 lo->lo_state = Lo_rundown;
841 lo->lo_backing_file = NULL;
842 spin_unlock_irq(&lo->lo_lock);
843
844 loop_release_xfer(lo);
845 lo->transfer = NULL;
846 lo->ioctl = NULL;
847 lo->lo_device = NULL;
848 lo->lo_encryption = NULL;
849 lo->lo_offset = 0;
850 lo->lo_sizelimit = 0;
851 lo->lo_encrypt_key_size = 0;
852 memset(lo->lo_encrypt_key, 0, LO_KEY_SIZE);
853 memset(lo->lo_crypt_name, 0, LO_NAME_SIZE);
854 memset(lo->lo_file_name, 0, LO_NAME_SIZE);
855 if (bdev) {
856 bdput(bdev);
857 invalidate_bdev(bdev);
858 }
859 set_capacity(lo->lo_disk, 0);
860 loop_sysfs_exit(lo);
861 if (bdev) {
862 bd_set_size(bdev, 0);
863 /* let user-space know about this change */
864 kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
865 }
866 mapping_set_gfp_mask(filp->f_mapping, gfp);
867 lo->lo_state = Lo_unbound;
868 /* This is safe: open() is still holding a reference. */
869 module_put(THIS_MODULE);
870 if (lo->lo_flags & LO_FLAGS_PARTSCAN && bdev)
871 ioctl_by_bdev(bdev, BLKRRPART, 0);
872 lo->lo_flags = 0;
873 if (!part_shift)
874 lo->lo_disk->flags |= GENHD_FL_NO_PART_SCAN;
875 mutex_unlock(&lo->lo_ctl_mutex);
876 /*
877 * Need not hold lo_ctl_mutex to fput backing file.
878 * Calling fput holding lo_ctl_mutex triggers a circular
879 * lock dependency possibility warning as fput can take
880 * bd_mutex which is usually taken before lo_ctl_mutex.
881 */
882 fput(filp);
883 return 0;
884}
885
886static int
887loop_set_status(struct loop_device *lo, const struct loop_info64 *info)
888{
889 int err;
890 struct loop_func_table *xfer;
891 kuid_t uid = current_uid();
892
893 if (lo->lo_encrypt_key_size &&
894 !uid_eq(lo->lo_key_owner, uid) &&
895 !capable(CAP_SYS_ADMIN))
896 return -EPERM;
897 if (lo->lo_state != Lo_bound)
898 return -ENXIO;
899 if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE)
900 return -EINVAL;
901
902 err = loop_release_xfer(lo);
903 if (err)
904 return err;
905
906 if (info->lo_encrypt_type) {
907 unsigned int type = info->lo_encrypt_type;
908
909 if (type >= MAX_LO_CRYPT)
910 return -EINVAL;
911 xfer = xfer_funcs[type];
912 if (xfer == NULL)
913 return -EINVAL;
914 } else
915 xfer = NULL;
916
917 err = loop_init_xfer(lo, xfer, info);
918 if (err)
919 return err;
920
921 if (lo->lo_offset != info->lo_offset ||
922 lo->lo_sizelimit != info->lo_sizelimit)
923 if (figure_loop_size(lo, info->lo_offset, info->lo_sizelimit))
924 return -EFBIG;
925
926 loop_config_discard(lo);
927
928 memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE);
929 memcpy(lo->lo_crypt_name, info->lo_crypt_name, LO_NAME_SIZE);
930 lo->lo_file_name[LO_NAME_SIZE-1] = 0;
931 lo->lo_crypt_name[LO_NAME_SIZE-1] = 0;
932
933 if (!xfer)
934 xfer = &none_funcs;
935 lo->transfer = xfer->transfer;
936 lo->ioctl = xfer->ioctl;
937
938 if ((lo->lo_flags & LO_FLAGS_AUTOCLEAR) !=
939 (info->lo_flags & LO_FLAGS_AUTOCLEAR))
940 lo->lo_flags ^= LO_FLAGS_AUTOCLEAR;
941
942 if ((info->lo_flags & LO_FLAGS_PARTSCAN) &&
943 !(lo->lo_flags & LO_FLAGS_PARTSCAN)) {
944 lo->lo_flags |= LO_FLAGS_PARTSCAN;
945 lo->lo_disk->flags &= ~GENHD_FL_NO_PART_SCAN;
946 ioctl_by_bdev(lo->lo_device, BLKRRPART, 0);
947 }
948
949 lo->lo_encrypt_key_size = info->lo_encrypt_key_size;
950 lo->lo_init[0] = info->lo_init[0];
951 lo->lo_init[1] = info->lo_init[1];
952 if (info->lo_encrypt_key_size) {
953 memcpy(lo->lo_encrypt_key, info->lo_encrypt_key,
954 info->lo_encrypt_key_size);
955 lo->lo_key_owner = uid;
956 }
957
958 return 0;
959}
960
961static int
962loop_get_status(struct loop_device *lo, struct loop_info64 *info)
963{
964 struct file *file = lo->lo_backing_file;
965 struct kstat stat;
966 int error;
967
968 if (lo->lo_state != Lo_bound)
969 return -ENXIO;
970 error = vfs_getattr(&file->f_path, &stat);
971 if (error)
972 return error;
973 memset(info, 0, sizeof(*info));
974 info->lo_number = lo->lo_number;
975 info->lo_device = huge_encode_dev(stat.dev);
976 info->lo_inode = stat.ino;
977 info->lo_rdevice = huge_encode_dev(lo->lo_device ? stat.rdev : stat.dev);
978 info->lo_offset = lo->lo_offset;
979 info->lo_sizelimit = lo->lo_sizelimit;
980 info->lo_flags = lo->lo_flags;
981 memcpy(info->lo_file_name, lo->lo_file_name, LO_NAME_SIZE);
982 memcpy(info->lo_crypt_name, lo->lo_crypt_name, LO_NAME_SIZE);
983 info->lo_encrypt_type =
984 lo->lo_encryption ? lo->lo_encryption->number : 0;
985 if (lo->lo_encrypt_key_size && capable(CAP_SYS_ADMIN)) {
986 info->lo_encrypt_key_size = lo->lo_encrypt_key_size;
987 memcpy(info->lo_encrypt_key, lo->lo_encrypt_key,
988 lo->lo_encrypt_key_size);
989 }
990 return 0;
991}
992
993static void
994loop_info64_from_old(const struct loop_info *info, struct loop_info64 *info64)
995{
996 memset(info64, 0, sizeof(*info64));
997 info64->lo_number = info->lo_number;
998 info64->lo_device = info->lo_device;
999 info64->lo_inode = info->lo_inode;
1000 info64->lo_rdevice = info->lo_rdevice;
1001 info64->lo_offset = info->lo_offset;
1002 info64->lo_sizelimit = 0;
1003 info64->lo_encrypt_type = info->lo_encrypt_type;
1004 info64->lo_encrypt_key_size = info->lo_encrypt_key_size;
1005 info64->lo_flags = info->lo_flags;
1006 info64->lo_init[0] = info->lo_init[0];
1007 info64->lo_init[1] = info->lo_init[1];
1008 if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
1009 memcpy(info64->lo_crypt_name, info->lo_name, LO_NAME_SIZE);
1010 else
1011 memcpy(info64->lo_file_name, info->lo_name, LO_NAME_SIZE);
1012 memcpy(info64->lo_encrypt_key, info->lo_encrypt_key, LO_KEY_SIZE);
1013}
1014
1015static int
1016loop_info64_to_old(const struct loop_info64 *info64, struct loop_info *info)
1017{
1018 memset(info, 0, sizeof(*info));
1019 info->lo_number = info64->lo_number;
1020 info->lo_device = info64->lo_device;
1021 info->lo_inode = info64->lo_inode;
1022 info->lo_rdevice = info64->lo_rdevice;
1023 info->lo_offset = info64->lo_offset;
1024 info->lo_encrypt_type = info64->lo_encrypt_type;
1025 info->lo_encrypt_key_size = info64->lo_encrypt_key_size;
1026 info->lo_flags = info64->lo_flags;
1027 info->lo_init[0] = info64->lo_init[0];
1028 info->lo_init[1] = info64->lo_init[1];
1029 if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
1030 memcpy(info->lo_name, info64->lo_crypt_name, LO_NAME_SIZE);
1031 else
1032 memcpy(info->lo_name, info64->lo_file_name, LO_NAME_SIZE);
1033 memcpy(info->lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE);
1034
1035 /* error in case values were truncated */
1036 if (info->lo_device != info64->lo_device ||
1037 info->lo_rdevice != info64->lo_rdevice ||
1038 info->lo_inode != info64->lo_inode ||
1039 info->lo_offset != info64->lo_offset)
1040 return -EOVERFLOW;
1041
1042 return 0;
1043}
1044
1045static int
1046loop_set_status_old(struct loop_device *lo, const struct loop_info __user *arg)
1047{
1048 struct loop_info info;
1049 struct loop_info64 info64;
1050
1051 if (copy_from_user(&info, arg, sizeof (struct loop_info)))
1052 return -EFAULT;
1053 loop_info64_from_old(&info, &info64);
1054 return loop_set_status(lo, &info64);
1055}
1056
1057static int
1058loop_set_status64(struct loop_device *lo, const struct loop_info64 __user *arg)
1059{
1060 struct loop_info64 info64;
1061
1062 if (copy_from_user(&info64, arg, sizeof (struct loop_info64)))
1063 return -EFAULT;
1064 return loop_set_status(lo, &info64);
1065}
1066
1067static int
1068loop_get_status_old(struct loop_device *lo, struct loop_info __user *arg) {
1069 struct loop_info info;
1070 struct loop_info64 info64;
1071 int err = 0;
1072
1073 if (!arg)
1074 err = -EINVAL;
1075 if (!err)
1076 err = loop_get_status(lo, &info64);
1077 if (!err)
1078 err = loop_info64_to_old(&info64, &info);
1079 if (!err && copy_to_user(arg, &info, sizeof(info)))
1080 err = -EFAULT;
1081
1082 return err;
1083}
1084
1085static int
1086loop_get_status64(struct loop_device *lo, struct loop_info64 __user *arg) {
1087 struct loop_info64 info64;
1088 int err = 0;
1089
1090 if (!arg)
1091 err = -EINVAL;
1092 if (!err)
1093 err = loop_get_status(lo, &info64);
1094 if (!err && copy_to_user(arg, &info64, sizeof(info64)))
1095 err = -EFAULT;
1096
1097 return err;
1098}
1099
1100static int loop_set_capacity(struct loop_device *lo, struct block_device *bdev)
1101{
1102 if (unlikely(lo->lo_state != Lo_bound))
1103 return -ENXIO;
1104
1105 return figure_loop_size(lo, lo->lo_offset, lo->lo_sizelimit);
1106}
1107
1108static int lo_ioctl(struct block_device *bdev, fmode_t mode,
1109 unsigned int cmd, unsigned long arg)
1110{
1111 struct loop_device *lo = bdev->bd_disk->private_data;
1112 int err;
1113
1114 mutex_lock_nested(&lo->lo_ctl_mutex, 1);
1115 switch (cmd) {
1116 case LOOP_SET_FD:
1117 err = loop_set_fd(lo, mode, bdev, arg);
1118 break;
1119 case LOOP_CHANGE_FD:
1120 err = loop_change_fd(lo, bdev, arg);
1121 break;
1122 case LOOP_CLR_FD:
1123 /* loop_clr_fd would have unlocked lo_ctl_mutex on success */
1124 err = loop_clr_fd(lo);
1125 if (!err)
1126 goto out_unlocked;
1127 break;
1128 case LOOP_SET_STATUS:
1129 err = -EPERM;
1130 if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN))
1131 err = loop_set_status_old(lo,
1132 (struct loop_info __user *)arg);
1133 break;
1134 case LOOP_GET_STATUS:
1135 err = loop_get_status_old(lo, (struct loop_info __user *) arg);
1136 break;
1137 case LOOP_SET_STATUS64:
1138 err = -EPERM;
1139 if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN))
1140 err = loop_set_status64(lo,
1141 (struct loop_info64 __user *) arg);
1142 break;
1143 case LOOP_GET_STATUS64:
1144 err = loop_get_status64(lo, (struct loop_info64 __user *) arg);
1145 break;
1146 case LOOP_SET_CAPACITY:
1147 err = -EPERM;
1148 if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN))
1149 err = loop_set_capacity(lo, bdev);
1150 break;
1151 default:
1152 err = lo->ioctl ? lo->ioctl(lo, cmd, arg) : -EINVAL;
1153 }
1154 mutex_unlock(&lo->lo_ctl_mutex);
1155
1156out_unlocked:
1157 return err;
1158}
1159
1160#ifdef CONFIG_COMPAT
1161struct compat_loop_info {
1162 compat_int_t lo_number; /* ioctl r/o */
1163 compat_dev_t lo_device; /* ioctl r/o */
1164 compat_ulong_t lo_inode; /* ioctl r/o */
1165 compat_dev_t lo_rdevice; /* ioctl r/o */
1166 compat_int_t lo_offset;
1167 compat_int_t lo_encrypt_type;
1168 compat_int_t lo_encrypt_key_size; /* ioctl w/o */
1169 compat_int_t lo_flags; /* ioctl r/o */
1170 char lo_name[LO_NAME_SIZE];
1171 unsigned char lo_encrypt_key[LO_KEY_SIZE]; /* ioctl w/o */
1172 compat_ulong_t lo_init[2];
1173 char reserved[4];
1174};
1175
1176/*
1177 * Transfer 32-bit compatibility structure in userspace to 64-bit loop info
1178 * - noinlined to reduce stack space usage in main part of driver
1179 */
1180static noinline int
1181loop_info64_from_compat(const struct compat_loop_info __user *arg,
1182 struct loop_info64 *info64)
1183{
1184 struct compat_loop_info info;
1185
1186 if (copy_from_user(&info, arg, sizeof(info)))
1187 return -EFAULT;
1188
1189 memset(info64, 0, sizeof(*info64));
1190 info64->lo_number = info.lo_number;
1191 info64->lo_device = info.lo_device;
1192 info64->lo_inode = info.lo_inode;
1193 info64->lo_rdevice = info.lo_rdevice;
1194 info64->lo_offset = info.lo_offset;
1195 info64->lo_sizelimit = 0;
1196 info64->lo_encrypt_type = info.lo_encrypt_type;
1197 info64->lo_encrypt_key_size = info.lo_encrypt_key_size;
1198 info64->lo_flags = info.lo_flags;
1199 info64->lo_init[0] = info.lo_init[0];
1200 info64->lo_init[1] = info.lo_init[1];
1201 if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
1202 memcpy(info64->lo_crypt_name, info.lo_name, LO_NAME_SIZE);
1203 else
1204 memcpy(info64->lo_file_name, info.lo_name, LO_NAME_SIZE);
1205 memcpy(info64->lo_encrypt_key, info.lo_encrypt_key, LO_KEY_SIZE);
1206 return 0;
1207}
1208
1209/*
1210 * Transfer 64-bit loop info to 32-bit compatibility structure in userspace
1211 * - noinlined to reduce stack space usage in main part of driver
1212 */
1213static noinline int
1214loop_info64_to_compat(const struct loop_info64 *info64,
1215 struct compat_loop_info __user *arg)
1216{
1217 struct compat_loop_info info;
1218
1219 memset(&info, 0, sizeof(info));
1220 info.lo_number = info64->lo_number;
1221 info.lo_device = info64->lo_device;
1222 info.lo_inode = info64->lo_inode;
1223 info.lo_rdevice = info64->lo_rdevice;
1224 info.lo_offset = info64->lo_offset;
1225 info.lo_encrypt_type = info64->lo_encrypt_type;
1226 info.lo_encrypt_key_size = info64->lo_encrypt_key_size;
1227 info.lo_flags = info64->lo_flags;
1228 info.lo_init[0] = info64->lo_init[0];
1229 info.lo_init[1] = info64->lo_init[1];
1230 if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
1231 memcpy(info.lo_name, info64->lo_crypt_name, LO_NAME_SIZE);
1232 else
1233 memcpy(info.lo_name, info64->lo_file_name, LO_NAME_SIZE);
1234 memcpy(info.lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE);
1235
1236 /* error in case values were truncated */
1237 if (info.lo_device != info64->lo_device ||
1238 info.lo_rdevice != info64->lo_rdevice ||
1239 info.lo_inode != info64->lo_inode ||
1240 info.lo_offset != info64->lo_offset ||
1241 info.lo_init[0] != info64->lo_init[0] ||
1242 info.lo_init[1] != info64->lo_init[1])
1243 return -EOVERFLOW;
1244
1245 if (copy_to_user(arg, &info, sizeof(info)))
1246 return -EFAULT;
1247 return 0;
1248}
1249
1250static int
1251loop_set_status_compat(struct loop_device *lo,
1252 const struct compat_loop_info __user *arg)
1253{
1254 struct loop_info64 info64;
1255 int ret;
1256
1257 ret = loop_info64_from_compat(arg, &info64);
1258 if (ret < 0)
1259 return ret;
1260 return loop_set_status(lo, &info64);
1261}
1262
1263static int
1264loop_get_status_compat(struct loop_device *lo,
1265 struct compat_loop_info __user *arg)
1266{
1267 struct loop_info64 info64;
1268 int err = 0;
1269
1270 if (!arg)
1271 err = -EINVAL;
1272 if (!err)
1273 err = loop_get_status(lo, &info64);
1274 if (!err)
1275 err = loop_info64_to_compat(&info64, arg);
1276 return err;
1277}
1278
1279static int lo_compat_ioctl(struct block_device *bdev, fmode_t mode,
1280 unsigned int cmd, unsigned long arg)
1281{
1282 struct loop_device *lo = bdev->bd_disk->private_data;
1283 int err;
1284
1285 switch(cmd) {
1286 case LOOP_SET_STATUS:
1287 mutex_lock(&lo->lo_ctl_mutex);
1288 err = loop_set_status_compat(
1289 lo, (const struct compat_loop_info __user *) arg);
1290 mutex_unlock(&lo->lo_ctl_mutex);
1291 break;
1292 case LOOP_GET_STATUS:
1293 mutex_lock(&lo->lo_ctl_mutex);
1294 err = loop_get_status_compat(
1295 lo, (struct compat_loop_info __user *) arg);
1296 mutex_unlock(&lo->lo_ctl_mutex);
1297 break;
1298 case LOOP_SET_CAPACITY:
1299 case LOOP_CLR_FD:
1300 case LOOP_GET_STATUS64:
1301 case LOOP_SET_STATUS64:
1302 arg = (unsigned long) compat_ptr(arg);
1303 case LOOP_SET_FD:
1304 case LOOP_CHANGE_FD:
1305 err = lo_ioctl(bdev, mode, cmd, arg);
1306 break;
1307 default:
1308 err = -ENOIOCTLCMD;
1309 break;
1310 }
1311 return err;
1312}
1313#endif
1314
1315static int lo_open(struct block_device *bdev, fmode_t mode)
1316{
1317 struct loop_device *lo;
1318 int err = 0;
1319
1320 mutex_lock(&loop_index_mutex);
1321 lo = bdev->bd_disk->private_data;
1322 if (!lo) {
1323 err = -ENXIO;
1324 goto out;
1325 }
1326
1327 mutex_lock(&lo->lo_ctl_mutex);
1328 lo->lo_refcnt++;
1329 mutex_unlock(&lo->lo_ctl_mutex);
1330out:
1331 mutex_unlock(&loop_index_mutex);
1332 return err;
1333}
1334
1335static void lo_release(struct gendisk *disk, fmode_t mode)
1336{
1337 struct loop_device *lo = disk->private_data;
1338 int err;
1339
1340 mutex_lock(&lo->lo_ctl_mutex);
1341
1342 if (--lo->lo_refcnt)
1343 goto out;
1344
1345 if (lo->lo_flags & LO_FLAGS_AUTOCLEAR) {
1346 /*
1347 * In autoclear mode, stop the loop thread
1348 * and remove configuration after last close.
1349 */
1350 err = loop_clr_fd(lo);
1351 if (!err)
1352 return;
1353 } else {
1354 /*
1355 * Otherwise keep thread (if running) and config,
1356 * but flush possible ongoing bios in thread.
1357 */
1358 loop_flush(lo);
1359 }
1360
1361out:
1362 mutex_unlock(&lo->lo_ctl_mutex);
1363}
1364
1365static const struct block_device_operations lo_fops = {
1366 .owner = THIS_MODULE,
1367 .open = lo_open,
1368 .release = lo_release,
1369 .ioctl = lo_ioctl,
1370#ifdef CONFIG_COMPAT
1371 .compat_ioctl = lo_compat_ioctl,
1372#endif
1373};
1374
1375/*
1376 * And now the modules code and kernel interface.
1377 */
1378static int max_loop;
1379module_param(max_loop, int, S_IRUGO);
1380MODULE_PARM_DESC(max_loop, "Maximum number of loop devices");
1381module_param(max_part, int, S_IRUGO);
1382MODULE_PARM_DESC(max_part, "Maximum number of partitions per loop device");
1383MODULE_LICENSE("GPL");
1384MODULE_ALIAS_BLOCKDEV_MAJOR(LOOP_MAJOR);
1385
1386int loop_register_transfer(struct loop_func_table *funcs)
1387{
1388 unsigned int n = funcs->number;
1389
1390 if (n >= MAX_LO_CRYPT || xfer_funcs[n])
1391 return -EINVAL;
1392 xfer_funcs[n] = funcs;
1393 return 0;
1394}
1395
1396static int unregister_transfer_cb(int id, void *ptr, void *data)
1397{
1398 struct loop_device *lo = ptr;
1399 struct loop_func_table *xfer = data;
1400
1401 mutex_lock(&lo->lo_ctl_mutex);
1402 if (lo->lo_encryption == xfer)
1403 loop_release_xfer(lo);
1404 mutex_unlock(&lo->lo_ctl_mutex);
1405 return 0;
1406}
1407
1408int loop_unregister_transfer(int number)
1409{
1410 unsigned int n = number;
1411 struct loop_func_table *xfer;
1412
1413 if (n == 0 || n >= MAX_LO_CRYPT || (xfer = xfer_funcs[n]) == NULL)
1414 return -EINVAL;
1415
1416 xfer_funcs[n] = NULL;
1417 idr_for_each(&loop_index_idr, &unregister_transfer_cb, xfer);
1418 return 0;
1419}
1420
1421EXPORT_SYMBOL(loop_register_transfer);
1422EXPORT_SYMBOL(loop_unregister_transfer);
1423
1424static int loop_queue_rq(struct blk_mq_hw_ctx *hctx,
1425 const struct blk_mq_queue_data *bd)
1426{
1427 struct loop_cmd *cmd = blk_mq_rq_to_pdu(bd->rq);
1428
1429 blk_mq_start_request(bd->rq);
1430
1431 if (cmd->rq->cmd_flags & REQ_WRITE) {
1432 struct loop_device *lo = cmd->rq->q->queuedata;
1433 bool need_sched = true;
1434
1435 spin_lock_irq(&lo->lo_lock);
1436 if (lo->write_started)
1437 need_sched = false;
1438 else
1439 lo->write_started = true;
1440 list_add_tail(&cmd->list, &lo->write_cmd_head);
1441 spin_unlock_irq(&lo->lo_lock);
1442
1443 if (need_sched)
1444 queue_work(loop_wq, &lo->write_work);
1445 } else {
1446 queue_work(loop_wq, &cmd->read_work);
1447 }
1448
1449 return BLK_MQ_RQ_QUEUE_OK;
1450}
1451
1452static void loop_handle_cmd(struct loop_cmd *cmd)
1453{
1454 const bool write = cmd->rq->cmd_flags & REQ_WRITE;
1455 struct loop_device *lo = cmd->rq->q->queuedata;
1456 int ret = -EIO;
1457
1458 if (lo->lo_state != Lo_bound)
1459 goto failed;
1460
1461 if (write && (lo->lo_flags & LO_FLAGS_READ_ONLY))
1462 goto failed;
1463
1464 ret = do_req_filebacked(lo, cmd->rq);
1465
1466 failed:
1467 if (ret)
1468 cmd->rq->errors = -EIO;
1469 blk_mq_complete_request(cmd->rq);
1470}
1471
1472static void loop_queue_write_work(struct work_struct *work)
1473{
1474 struct loop_device *lo =
1475 container_of(work, struct loop_device, write_work);
1476 LIST_HEAD(cmd_list);
1477
1478 spin_lock_irq(&lo->lo_lock);
1479 repeat:
1480 list_splice_init(&lo->write_cmd_head, &cmd_list);
1481 spin_unlock_irq(&lo->lo_lock);
1482
1483 while (!list_empty(&cmd_list)) {
1484 struct loop_cmd *cmd = list_first_entry(&cmd_list,
1485 struct loop_cmd, list);
1486 list_del_init(&cmd->list);
1487 loop_handle_cmd(cmd);
1488 }
1489
1490 spin_lock_irq(&lo->lo_lock);
1491 if (!list_empty(&lo->write_cmd_head))
1492 goto repeat;
1493 lo->write_started = false;
1494 spin_unlock_irq(&lo->lo_lock);
1495}
1496
1497static void loop_queue_read_work(struct work_struct *work)
1498{
1499 struct loop_cmd *cmd =
1500 container_of(work, struct loop_cmd, read_work);
1501
1502 loop_handle_cmd(cmd);
1503}
1504
1505static int loop_init_request(void *data, struct request *rq,
1506 unsigned int hctx_idx, unsigned int request_idx,
1507 unsigned int numa_node)
1508{
1509 struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
1510
1511 cmd->rq = rq;
1512 INIT_WORK(&cmd->read_work, loop_queue_read_work);
1513
1514 return 0;
1515}
1516
1517static struct blk_mq_ops loop_mq_ops = {
1518 .queue_rq = loop_queue_rq,
1519 .map_queue = blk_mq_map_queue,
1520 .init_request = loop_init_request,
1521};
1522
1523static int loop_add(struct loop_device **l, int i)
1524{
1525 struct loop_device *lo;
1526 struct gendisk *disk;
1527 int err;
1528
1529 err = -ENOMEM;
1530 lo = kzalloc(sizeof(*lo), GFP_KERNEL);
1531 if (!lo)
1532 goto out;
1533
1534 lo->lo_state = Lo_unbound;
1535
1536 /* allocate id, if @id >= 0, we're requesting that specific id */
1537 if (i >= 0) {
1538 err = idr_alloc(&loop_index_idr, lo, i, i + 1, GFP_KERNEL);
1539 if (err == -ENOSPC)
1540 err = -EEXIST;
1541 } else {
1542 err = idr_alloc(&loop_index_idr, lo, 0, 0, GFP_KERNEL);
1543 }
1544 if (err < 0)
1545 goto out_free_dev;
1546 i = err;
1547
1548 err = -ENOMEM;
1549 lo->tag_set.ops = &loop_mq_ops;
1550 lo->tag_set.nr_hw_queues = 1;
1551 lo->tag_set.queue_depth = 128;
1552 lo->tag_set.numa_node = NUMA_NO_NODE;
1553 lo->tag_set.cmd_size = sizeof(struct loop_cmd);
1554 lo->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE;
1555 lo->tag_set.driver_data = lo;
1556
1557 err = blk_mq_alloc_tag_set(&lo->tag_set);
1558 if (err)
1559 goto out_free_idr;
1560
1561 lo->lo_queue = blk_mq_init_queue(&lo->tag_set);
1562 if (IS_ERR_OR_NULL(lo->lo_queue)) {
1563 err = PTR_ERR(lo->lo_queue);
1564 goto out_cleanup_tags;
1565 }
1566 lo->lo_queue->queuedata = lo;
1567
1568 INIT_LIST_HEAD(&lo->write_cmd_head);
1569 INIT_WORK(&lo->write_work, loop_queue_write_work);
1570
1571 disk = lo->lo_disk = alloc_disk(1 << part_shift);
1572 if (!disk)
1573 goto out_free_queue;
1574
1575 /*
1576 * Disable partition scanning by default. The in-kernel partition
1577 * scanning can be requested individually per-device during its
1578 * setup. Userspace can always add and remove partitions from all
1579 * devices. The needed partition minors are allocated from the
1580 * extended minor space, the main loop device numbers will continue
1581 * to match the loop minors, regardless of the number of partitions
1582 * used.
1583 *
1584 * If max_part is given, partition scanning is globally enabled for
1585 * all loop devices. The minors for the main loop devices will be
1586 * multiples of max_part.
1587 *
1588 * Note: Global-for-all-devices, set-only-at-init, read-only module
1589 * parameteters like 'max_loop' and 'max_part' make things needlessly
1590 * complicated, are too static, inflexible and may surprise
1591 * userspace tools. Parameters like this in general should be avoided.
1592 */
1593 if (!part_shift)
1594 disk->flags |= GENHD_FL_NO_PART_SCAN;
1595 disk->flags |= GENHD_FL_EXT_DEVT;
1596 mutex_init(&lo->lo_ctl_mutex);
1597 lo->lo_number = i;
1598 spin_lock_init(&lo->lo_lock);
1599 disk->major = LOOP_MAJOR;
1600 disk->first_minor = i << part_shift;
1601 disk->fops = &lo_fops;
1602 disk->private_data = lo;
1603 disk->queue = lo->lo_queue;
1604 sprintf(disk->disk_name, "loop%d", i);
1605 add_disk(disk);
1606 *l = lo;
1607 return lo->lo_number;
1608
1609out_free_queue:
1610 blk_cleanup_queue(lo->lo_queue);
1611out_cleanup_tags:
1612 blk_mq_free_tag_set(&lo->tag_set);
1613out_free_idr:
1614 idr_remove(&loop_index_idr, i);
1615out_free_dev:
1616 kfree(lo);
1617out:
1618 return err;
1619}
1620
1621static void loop_remove(struct loop_device *lo)
1622{
1623 blk_cleanup_queue(lo->lo_queue);
1624 del_gendisk(lo->lo_disk);
1625 blk_mq_free_tag_set(&lo->tag_set);
1626 put_disk(lo->lo_disk);
1627 kfree(lo);
1628}
1629
1630static int find_free_cb(int id, void *ptr, void *data)
1631{
1632 struct loop_device *lo = ptr;
1633 struct loop_device **l = data;
1634
1635 if (lo->lo_state == Lo_unbound) {
1636 *l = lo;
1637 return 1;
1638 }
1639 return 0;
1640}
1641
1642static int loop_lookup(struct loop_device **l, int i)
1643{
1644 struct loop_device *lo;
1645 int ret = -ENODEV;
1646
1647 if (i < 0) {
1648 int err;
1649
1650 err = idr_for_each(&loop_index_idr, &find_free_cb, &lo);
1651 if (err == 1) {
1652 *l = lo;
1653 ret = lo->lo_number;
1654 }
1655 goto out;
1656 }
1657
1658 /* lookup and return a specific i */
1659 lo = idr_find(&loop_index_idr, i);
1660 if (lo) {
1661 *l = lo;
1662 ret = lo->lo_number;
1663 }
1664out:
1665 return ret;
1666}
1667
1668static struct kobject *loop_probe(dev_t dev, int *part, void *data)
1669{
1670 struct loop_device *lo;
1671 struct kobject *kobj;
1672 int err;
1673
1674 mutex_lock(&loop_index_mutex);
1675 err = loop_lookup(&lo, MINOR(dev) >> part_shift);
1676 if (err < 0)
1677 err = loop_add(&lo, MINOR(dev) >> part_shift);
1678 if (err < 0)
1679 kobj = NULL;
1680 else
1681 kobj = get_disk(lo->lo_disk);
1682 mutex_unlock(&loop_index_mutex);
1683
1684 *part = 0;
1685 return kobj;
1686}
1687
1688static long loop_control_ioctl(struct file *file, unsigned int cmd,
1689 unsigned long parm)
1690{
1691 struct loop_device *lo;
1692 int ret = -ENOSYS;
1693
1694 mutex_lock(&loop_index_mutex);
1695 switch (cmd) {
1696 case LOOP_CTL_ADD:
1697 ret = loop_lookup(&lo, parm);
1698 if (ret >= 0) {
1699 ret = -EEXIST;
1700 break;
1701 }
1702 ret = loop_add(&lo, parm);
1703 break;
1704 case LOOP_CTL_REMOVE:
1705 ret = loop_lookup(&lo, parm);
1706 if (ret < 0)
1707 break;
1708 mutex_lock(&lo->lo_ctl_mutex);
1709 if (lo->lo_state != Lo_unbound) {
1710 ret = -EBUSY;
1711 mutex_unlock(&lo->lo_ctl_mutex);
1712 break;
1713 }
1714 if (lo->lo_refcnt > 0) {
1715 ret = -EBUSY;
1716 mutex_unlock(&lo->lo_ctl_mutex);
1717 break;
1718 }
1719 lo->lo_disk->private_data = NULL;
1720 mutex_unlock(&lo->lo_ctl_mutex);
1721 idr_remove(&loop_index_idr, lo->lo_number);
1722 loop_remove(lo);
1723 break;
1724 case LOOP_CTL_GET_FREE:
1725 ret = loop_lookup(&lo, -1);
1726 if (ret >= 0)
1727 break;
1728 ret = loop_add(&lo, -1);
1729 }
1730 mutex_unlock(&loop_index_mutex);
1731
1732 return ret;
1733}
1734
1735static const struct file_operations loop_ctl_fops = {
1736 .open = nonseekable_open,
1737 .unlocked_ioctl = loop_control_ioctl,
1738 .compat_ioctl = loop_control_ioctl,
1739 .owner = THIS_MODULE,
1740 .llseek = noop_llseek,
1741};
1742
1743static struct miscdevice loop_misc = {
1744 .minor = LOOP_CTRL_MINOR,
1745 .name = "loop-control",
1746 .fops = &loop_ctl_fops,
1747};
1748
1749MODULE_ALIAS_MISCDEV(LOOP_CTRL_MINOR);
1750MODULE_ALIAS("devname:loop-control");
1751
1752static int __init loop_init(void)
1753{
1754 int i, nr;
1755 unsigned long range;
1756 struct loop_device *lo;
1757 int err;
1758
1759 err = misc_register(&loop_misc);
1760 if (err < 0)
1761 return err;
1762
1763 part_shift = 0;
1764 if (max_part > 0) {
1765 part_shift = fls(max_part);
1766
1767 /*
1768 * Adjust max_part according to part_shift as it is exported
1769 * to user space so that user can decide correct minor number
1770 * if [s]he want to create more devices.
1771 *
1772 * Note that -1 is required because partition 0 is reserved
1773 * for the whole disk.
1774 */
1775 max_part = (1UL << part_shift) - 1;
1776 }
1777
1778 if ((1UL << part_shift) > DISK_MAX_PARTS) {
1779 err = -EINVAL;
1780 goto misc_out;
1781 }
1782
1783 if (max_loop > 1UL << (MINORBITS - part_shift)) {
1784 err = -EINVAL;
1785 goto misc_out;
1786 }
1787
1788 /*
1789 * If max_loop is specified, create that many devices upfront.
1790 * This also becomes a hard limit. If max_loop is not specified,
1791 * create CONFIG_BLK_DEV_LOOP_MIN_COUNT loop devices at module
1792 * init time. Loop devices can be requested on-demand with the
1793 * /dev/loop-control interface, or be instantiated by accessing
1794 * a 'dead' device node.
1795 */
1796 if (max_loop) {
1797 nr = max_loop;
1798 range = max_loop << part_shift;
1799 } else {
1800 nr = CONFIG_BLK_DEV_LOOP_MIN_COUNT;
1801 range = 1UL << MINORBITS;
1802 }
1803
1804 if (register_blkdev(LOOP_MAJOR, "loop")) {
1805 err = -EIO;
1806 goto misc_out;
1807 }
1808
1809 loop_wq = alloc_workqueue("kloopd",
1810 WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_UNBOUND, 0);
1811 if (!loop_wq) {
1812 err = -ENOMEM;
1813 goto misc_out;
1814 }
1815
1816 blk_register_region(MKDEV(LOOP_MAJOR, 0), range,
1817 THIS_MODULE, loop_probe, NULL, NULL);
1818
1819 /* pre-create number of devices given by config or max_loop */
1820 mutex_lock(&loop_index_mutex);
1821 for (i = 0; i < nr; i++)
1822 loop_add(&lo, i);
1823 mutex_unlock(&loop_index_mutex);
1824
1825 printk(KERN_INFO "loop: module loaded\n");
1826 return 0;
1827
1828misc_out:
1829 misc_deregister(&loop_misc);
1830 return err;
1831}
1832
1833static int loop_exit_cb(int id, void *ptr, void *data)
1834{
1835 struct loop_device *lo = ptr;
1836
1837 loop_remove(lo);
1838 return 0;
1839}
1840
1841static void __exit loop_exit(void)
1842{
1843 unsigned long range;
1844
1845 range = max_loop ? max_loop << part_shift : 1UL << MINORBITS;
1846
1847 idr_for_each(&loop_index_idr, &loop_exit_cb, NULL);
1848 idr_destroy(&loop_index_idr);
1849
1850 blk_unregister_region(MKDEV(LOOP_MAJOR, 0), range);
1851 unregister_blkdev(LOOP_MAJOR, "loop");
1852
1853 destroy_workqueue(loop_wq);
1854
1855 misc_deregister(&loop_misc);
1856}
1857
1858module_init(loop_init);
1859module_exit(loop_exit);
1860
1861#ifndef MODULE
1862static int __init max_loop_setup(char *str)
1863{
1864 max_loop = simple_strtol(str, NULL, 0);
1865 return 1;
1866}
1867
1868__setup("max_loop=", max_loop_setup);
1869#endif