projects/linux·blob master · refs

1807 lines · 42.6 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2025, Christoph Hellwig.
4 * Copyright (c) 2025, Western Digital Corporation or its affiliates.
5 *
6 * Zoned Loop Device driver - exports a zoned block device using one file per
7 * zone as backing storage.
8 */
9#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10
11#include <linux/module.h>
12#include <linux/blk-mq.h>
13#include <linux/blkzoned.h>
14#include <linux/pagemap.h>
15#include <linux/miscdevice.h>
16#include <linux/falloc.h>
17#include <linux/mutex.h>
18#include <linux/parser.h>
19#include <linux/seq_file.h>
20#include <linux/xattr.h>
21
22/*
23 * Options for adding (and removing) a device.
24 */
25enum {
26 ZLOOP_OPT_ERR = 0,
27 ZLOOP_OPT_ID = (1 << 0),
28 ZLOOP_OPT_CAPACITY = (1 << 1),
29 ZLOOP_OPT_ZONE_SIZE = (1 << 2),
30 ZLOOP_OPT_ZONE_CAPACITY = (1 << 3),
31 ZLOOP_OPT_NR_CONV_ZONES = (1 << 4),
32 ZLOOP_OPT_BASE_DIR = (1 << 5),
33 ZLOOP_OPT_NR_QUEUES = (1 << 6),
34 ZLOOP_OPT_QUEUE_DEPTH = (1 << 7),
35 ZLOOP_OPT_BUFFERED_IO = (1 << 8),
36 ZLOOP_OPT_ZONE_APPEND = (1 << 9),
37 ZLOOP_OPT_ORDERED_ZONE_APPEND = (1 << 10),
38 ZLOOP_OPT_DISCARD_WRITE_CACHE = (1 << 11),
39 ZLOOP_OPT_MAX_OPEN_ZONES = (1 << 12),
40};
41
42static const match_table_t zloop_opt_tokens = {
43 { ZLOOP_OPT_ID, "id=%d" },
44 { ZLOOP_OPT_CAPACITY, "capacity_mb=%u" },
45 { ZLOOP_OPT_ZONE_SIZE, "zone_size_mb=%u" },
46 { ZLOOP_OPT_ZONE_CAPACITY, "zone_capacity_mb=%u" },
47 { ZLOOP_OPT_NR_CONV_ZONES, "conv_zones=%u" },
48 { ZLOOP_OPT_BASE_DIR, "base_dir=%s" },
49 { ZLOOP_OPT_NR_QUEUES, "nr_queues=%u" },
50 { ZLOOP_OPT_QUEUE_DEPTH, "queue_depth=%u" },
51 { ZLOOP_OPT_BUFFERED_IO, "buffered_io" },
52 { ZLOOP_OPT_ZONE_APPEND, "zone_append=%u" },
53 { ZLOOP_OPT_ORDERED_ZONE_APPEND, "ordered_zone_append" },
54 { ZLOOP_OPT_DISCARD_WRITE_CACHE, "discard_write_cache" },
55 { ZLOOP_OPT_MAX_OPEN_ZONES, "max_open_zones=%u" },
56 { ZLOOP_OPT_ERR, NULL }
57};
58
59/* Default values for the "add" operation. */
60#define ZLOOP_DEF_ID -1
61#define ZLOOP_DEF_ZONE_SIZE ((256ULL * SZ_1M) >> SECTOR_SHIFT)
62#define ZLOOP_DEF_NR_ZONES 64
63#define ZLOOP_DEF_NR_CONV_ZONES 8
64#define ZLOOP_DEF_MAX_OPEN_ZONES 0
65#define ZLOOP_DEF_BASE_DIR "/var/local/zloop"
66#define ZLOOP_DEF_NR_QUEUES 1
67#define ZLOOP_DEF_QUEUE_DEPTH 128
68#define ZLOOP_DEF_BUFFERED_IO false
69#define ZLOOP_DEF_ZONE_APPEND true
70#define ZLOOP_DEF_ORDERED_ZONE_APPEND false
71
72/* Arbitrary limit on the zone size (16GB). */
73#define ZLOOP_MAX_ZONE_SIZE_MB 16384
74
75struct zloop_options {
76 unsigned int mask;
77 int id;
78 sector_t capacity;
79 sector_t zone_size;
80 sector_t zone_capacity;
81 unsigned int nr_conv_zones;
82 unsigned int max_open_zones;
83 char *base_dir;
84 unsigned int nr_queues;
85 unsigned int queue_depth;
86 bool buffered_io;
87 bool zone_append;
88 bool ordered_zone_append;
89 bool discard_write_cache;
90};
91
92/*
93 * Device states.
94 */
95enum {
96 Zlo_creating = 0,
97 Zlo_live,
98 Zlo_deleting,
99};
100
101enum zloop_zone_flags {
102 ZLOOP_ZONE_CONV = 0,
103 ZLOOP_ZONE_SEQ_ERROR,
104};
105
106/*
107 * Zone descriptor.
108 * Locking order: z.lock -> z.wp_lock -> zlo.open_zones_lock
109 */
110struct zloop_zone {
111 struct list_head open_zone_entry;
112 struct file *file;
113
114 unsigned long flags;
115 struct mutex lock;
116 spinlock_t wp_lock;
117 enum blk_zone_cond cond;
118 sector_t start;
119 sector_t wp;
120
121 gfp_t old_gfp_mask;
122};
123
124struct zloop_device {
125 unsigned int id;
126 unsigned int state;
127
128 struct blk_mq_tag_set tag_set;
129 struct gendisk *disk;
130
131 struct workqueue_struct *workqueue;
132 bool buffered_io;
133 bool zone_append;
134 bool ordered_zone_append;
135 bool discard_write_cache;
136
137 const char *base_dir;
138 struct file *data_dir;
139
140 unsigned int zone_shift;
141 sector_t zone_size;
142 sector_t zone_capacity;
143 unsigned int nr_zones;
144 unsigned int nr_conv_zones;
145 unsigned int max_open_zones;
146 unsigned int block_size;
147 unsigned int dio_mem_align;
148
149 spinlock_t open_zones_lock;
150 struct list_head open_zones_lru_list;
151 unsigned int nr_open_zones;
152
153 struct zloop_zone zones[] __counted_by(nr_zones);
154};
155
156struct zloop_cmd {
157 struct work_struct work;
158 atomic_t ref;
159 sector_t sector;
160 sector_t nr_sectors;
161 long ret;
162 struct kiocb iocb;
163 struct bio_vec *bvec;
164};
165
166static DEFINE_IDR(zloop_index_idr);
167static DEFINE_MUTEX(zloop_ctl_mutex);
168
169static unsigned int rq_zone_no(struct request *rq)
170{
171 struct zloop_device *zlo = rq->q->queuedata;
172
173 return blk_rq_pos(rq) >> zlo->zone_shift;
174}
175
176/*
177 * Open an already open zone. This is mostly a no-op, except for the imp open ->
178 * exp open condition change that may happen. We also move a zone at the tail of
179 * the list of open zones so that if we need to
180 * implicitly close one open zone, we can do so in LRU order.
181 */
182static inline void zloop_lru_rotate_open_zone(struct zloop_device *zlo,
183 struct zloop_zone *zone)
184{
185 if (zlo->max_open_zones) {
186 spin_lock(&zlo->open_zones_lock);
187 list_move_tail(&zone->open_zone_entry,
188 &zlo->open_zones_lru_list);
189 spin_unlock(&zlo->open_zones_lock);
190 }
191}
192
193static inline void zloop_lru_remove_open_zone(struct zloop_device *zlo,
194 struct zloop_zone *zone)
195{
196 if (zone->cond == BLK_ZONE_COND_IMP_OPEN ||
197 zone->cond == BLK_ZONE_COND_EXP_OPEN) {
198 spin_lock(&zlo->open_zones_lock);
199 list_del_init(&zone->open_zone_entry);
200 zlo->nr_open_zones--;
201 spin_unlock(&zlo->open_zones_lock);
202 }
203}
204
205static inline bool zloop_can_open_zone(struct zloop_device *zlo)
206{
207 return !zlo->max_open_zones || zlo->nr_open_zones < zlo->max_open_zones;
208}
209
210/*
211 * If we have reached the maximum open zones limit, attempt to close an
212 * implicitly open zone (if we have any) so that we can implicitly open another
213 * zone without exceeding the maximum number of open zones.
214 */
215static bool zloop_close_imp_open_zone(struct zloop_device *zlo)
216{
217 struct zloop_zone *zone;
218
219 lockdep_assert_held(&zlo->open_zones_lock);
220
221 if (zloop_can_open_zone(zlo))
222 return true;
223
224 list_for_each_entry(zone, &zlo->open_zones_lru_list, open_zone_entry) {
225 if (zone->cond == BLK_ZONE_COND_IMP_OPEN) {
226 zone->cond = BLK_ZONE_COND_CLOSED;
227 list_del_init(&zone->open_zone_entry);
228 zlo->nr_open_zones--;
229 return true;
230 }
231 }
232
233 return false;
234}
235
236static bool zloop_open_closed_or_empty_zone(struct zloop_device *zlo,
237 struct zloop_zone *zone,
238 bool explicit)
239{
240 spin_lock(&zlo->open_zones_lock);
241
242 if (explicit) {
243 /*
244 * Explicit open: we cannot allow this if we have reached the
245 * maximum open zones limit.
246 */
247 if (!zloop_can_open_zone(zlo))
248 goto fail;
249 zone->cond = BLK_ZONE_COND_EXP_OPEN;
250 } else {
251 /*
252 * Implicit open case: if we have reached the maximum open zones
253 * limit, try to close an implicitly open zone first.
254 */
255 if (!zloop_close_imp_open_zone(zlo))
256 goto fail;
257 zone->cond = BLK_ZONE_COND_IMP_OPEN;
258 }
259
260 zlo->nr_open_zones++;
261 list_add_tail(&zone->open_zone_entry,
262 &zlo->open_zones_lru_list);
263
264 spin_unlock(&zlo->open_zones_lock);
265
266 return true;
267
268fail:
269 spin_unlock(&zlo->open_zones_lock);
270
271 return false;
272}
273
274static bool zloop_do_open_zone(struct zloop_device *zlo,
275 struct zloop_zone *zone, bool explicit)
276{
277 switch (zone->cond) {
278 case BLK_ZONE_COND_IMP_OPEN:
279 case BLK_ZONE_COND_EXP_OPEN:
280 if (explicit)
281 zone->cond = BLK_ZONE_COND_EXP_OPEN;
282 zloop_lru_rotate_open_zone(zlo, zone);
283 return true;
284 case BLK_ZONE_COND_EMPTY:
285 case BLK_ZONE_COND_CLOSED:
286 return zloop_open_closed_or_empty_zone(zlo, zone, explicit);
287 default:
288 return false;
289 }
290}
291
292static void zloop_mark_full(struct zloop_device *zlo, struct zloop_zone *zone)
293{
294 lockdep_assert_held(&zone->wp_lock);
295
296 zloop_lru_remove_open_zone(zlo, zone);
297 zone->cond = BLK_ZONE_COND_FULL;
298 zone->wp = ULLONG_MAX;
299}
300
301static void zloop_mark_empty(struct zloop_device *zlo, struct zloop_zone *zone)
302{
303 lockdep_assert_held(&zone->wp_lock);
304
305 zloop_lru_remove_open_zone(zlo, zone);
306 zone->cond = BLK_ZONE_COND_EMPTY;
307 zone->wp = zone->start;
308}
309
310static int zloop_update_seq_zone(struct zloop_device *zlo, unsigned int zone_no)
311{
312 struct zloop_zone *zone = &zlo->zones[zone_no];
313 struct kstat stat;
314 sector_t file_sectors;
315 int ret;
316
317 lockdep_assert_held(&zone->lock);
318
319 ret = vfs_getattr(&zone->file->f_path, &stat, STATX_SIZE, 0);
320 if (ret < 0) {
321 pr_err("Failed to get zone %u file stat (err=%d)\n",
322 zone_no, ret);
323 set_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags);
324 return ret;
325 }
326
327 file_sectors = stat.size >> SECTOR_SHIFT;
328 if (file_sectors > zlo->zone_capacity) {
329 pr_err("Zone %u file too large (%llu sectors > %llu)\n",
330 zone_no, file_sectors, zlo->zone_capacity);
331 return -EINVAL;
332 }
333
334 if (!IS_ALIGNED(stat.size, zlo->block_size)) {
335 pr_err("Zone %u file size (%llu) not aligned to block size %u\n",
336 zone_no, stat.size, zlo->block_size);
337 return -EINVAL;
338 }
339
340 spin_lock(&zone->wp_lock);
341 if (!file_sectors) {
342 zloop_mark_empty(zlo, zone);
343 } else if (file_sectors == zlo->zone_capacity) {
344 zloop_mark_full(zlo, zone);
345 } else {
346 if (zone->cond != BLK_ZONE_COND_IMP_OPEN &&
347 zone->cond != BLK_ZONE_COND_EXP_OPEN)
348 zone->cond = BLK_ZONE_COND_CLOSED;
349 zone->wp = zone->start + file_sectors;
350 }
351 spin_unlock(&zone->wp_lock);
352
353 return 0;
354}
355
356static int zloop_open_zone(struct zloop_device *zlo, unsigned int zone_no)
357{
358 struct zloop_zone *zone = &zlo->zones[zone_no];
359 int ret = 0;
360
361 if (test_bit(ZLOOP_ZONE_CONV, &zone->flags))
362 return -EIO;
363
364 mutex_lock(&zone->lock);
365
366 if (test_and_clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags)) {
367 ret = zloop_update_seq_zone(zlo, zone_no);
368 if (ret)
369 goto unlock;
370 }
371
372 if (!zloop_do_open_zone(zlo, zone, true))
373 ret = -EIO;
374
375unlock:
376 mutex_unlock(&zone->lock);
377
378 return ret;
379}
380
381static int zloop_close_zone(struct zloop_device *zlo, unsigned int zone_no)
382{
383 struct zloop_zone *zone = &zlo->zones[zone_no];
384 int ret = 0;
385
386 if (test_bit(ZLOOP_ZONE_CONV, &zone->flags))
387 return -EIO;
388
389 mutex_lock(&zone->lock);
390
391 if (test_and_clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags)) {
392 ret = zloop_update_seq_zone(zlo, zone_no);
393 if (ret)
394 goto unlock;
395 }
396
397 switch (zone->cond) {
398 case BLK_ZONE_COND_CLOSED:
399 break;
400 case BLK_ZONE_COND_IMP_OPEN:
401 case BLK_ZONE_COND_EXP_OPEN:
402 spin_lock(&zone->wp_lock);
403 zloop_lru_remove_open_zone(zlo, zone);
404 if (zone->wp == zone->start)
405 zone->cond = BLK_ZONE_COND_EMPTY;
406 else
407 zone->cond = BLK_ZONE_COND_CLOSED;
408 spin_unlock(&zone->wp_lock);
409 break;
410 case BLK_ZONE_COND_EMPTY:
411 case BLK_ZONE_COND_FULL:
412 default:
413 ret = -EIO;
414 break;
415 }
416
417unlock:
418 mutex_unlock(&zone->lock);
419
420 return ret;
421}
422
423static int zloop_reset_zone(struct zloop_device *zlo, unsigned int zone_no)
424{
425 struct zloop_zone *zone = &zlo->zones[zone_no];
426 int ret = 0;
427
428 if (test_bit(ZLOOP_ZONE_CONV, &zone->flags))
429 return -EIO;
430
431 mutex_lock(&zone->lock);
432
433 if (!test_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags) &&
434 zone->cond == BLK_ZONE_COND_EMPTY)
435 goto unlock;
436
437 if (vfs_truncate(&zone->file->f_path, 0)) {
438 set_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags);
439 ret = -EIO;
440 goto unlock;
441 }
442
443 spin_lock(&zone->wp_lock);
444 zloop_mark_empty(zlo, zone);
445 clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags);
446 spin_unlock(&zone->wp_lock);
447
448unlock:
449 mutex_unlock(&zone->lock);
450
451 return ret;
452}
453
454static int zloop_reset_all_zones(struct zloop_device *zlo)
455{
456 unsigned int i;
457 int ret;
458
459 for (i = zlo->nr_conv_zones; i < zlo->nr_zones; i++) {
460 ret = zloop_reset_zone(zlo, i);
461 if (ret)
462 return ret;
463 }
464
465 return 0;
466}
467
468static int zloop_finish_zone(struct zloop_device *zlo, unsigned int zone_no)
469{
470 struct zloop_zone *zone = &zlo->zones[zone_no];
471 int ret = 0;
472
473 if (test_bit(ZLOOP_ZONE_CONV, &zone->flags))
474 return -EIO;
475
476 mutex_lock(&zone->lock);
477
478 if (!test_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags) &&
479 zone->cond == BLK_ZONE_COND_FULL)
480 goto unlock;
481
482 if (vfs_truncate(&zone->file->f_path,
483 zlo->zone_capacity << SECTOR_SHIFT)) {
484 set_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags);
485 ret = -EIO;
486 goto unlock;
487 }
488
489 spin_lock(&zone->wp_lock);
490 zloop_mark_full(zlo, zone);
491 clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags);
492 spin_unlock(&zone->wp_lock);
493
494 unlock:
495 mutex_unlock(&zone->lock);
496
497 return ret;
498}
499
500static void zloop_put_cmd(struct zloop_cmd *cmd)
501{
502 struct request *rq = blk_mq_rq_from_pdu(cmd);
503
504 if (!atomic_dec_and_test(&cmd->ref))
505 return;
506 kfree(cmd->bvec);
507 cmd->bvec = NULL;
508 if (likely(!blk_should_fake_timeout(rq->q)))
509 blk_mq_complete_request(rq);
510}
511
512static void zloop_rw_complete(struct kiocb *iocb, long ret)
513{
514 struct zloop_cmd *cmd = container_of(iocb, struct zloop_cmd, iocb);
515
516 cmd->ret = ret;
517 zloop_put_cmd(cmd);
518}
519
520static int zloop_do_rw(struct zloop_cmd *cmd)
521{
522 struct request *rq = blk_mq_rq_from_pdu(cmd);
523 int rw = req_op(rq) == REQ_OP_READ ? ITER_DEST : ITER_SOURCE;
524 unsigned int nr_bvec = blk_rq_nr_bvec(rq);
525 struct zloop_device *zlo = rq->q->queuedata;
526 struct zloop_zone *zone = &zlo->zones[rq_zone_no(rq)];
527 struct req_iterator rq_iter;
528 struct iov_iter iter;
529
530 if (rq->bio != rq->biotail) {
531 struct bio_vec tmp, *bvec;
532
533 cmd->bvec = kmalloc_objs(*cmd->bvec, nr_bvec, GFP_NOIO);
534 if (!cmd->bvec)
535 return -EIO;
536
537 /*
538 * The bios of the request may be started from the middle of
539 * the 'bvec' because of bio splitting, so we can't directly
540 * copy bio->bi_iov_vec to new bvec. The rq_for_each_bvec
541 * API will take care of all details for us.
542 */
543 bvec = cmd->bvec;
544 rq_for_each_bvec(tmp, rq, rq_iter) {
545 *bvec = tmp;
546 bvec++;
547 }
548 iov_iter_bvec(&iter, rw, cmd->bvec, nr_bvec, blk_rq_bytes(rq));
549 } else {
550 /*
551 * Same here, this bio may be started from the middle of the
552 * 'bvec' because of bio splitting, so offset from the bvec
553 * must be passed to iov iterator
554 */
555 iov_iter_bvec(&iter, rw,
556 __bvec_iter_bvec(rq->bio->bi_io_vec, rq->bio->bi_iter),
557 nr_bvec, blk_rq_bytes(rq));
558 iter.iov_offset = rq->bio->bi_iter.bi_offset;
559 }
560
561 cmd->iocb.ki_pos = (cmd->sector - zone->start) << SECTOR_SHIFT;
562 cmd->iocb.ki_filp = zone->file;
563 cmd->iocb.ki_complete = zloop_rw_complete;
564 if (!zlo->buffered_io)
565 cmd->iocb.ki_flags = IOCB_DIRECT;
566 cmd->iocb.ki_ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0);
567
568 if (rw == ITER_SOURCE)
569 return zone->file->f_op->write_iter(&cmd->iocb, &iter);
570 return zone->file->f_op->read_iter(&cmd->iocb, &iter);
571}
572
573static int zloop_seq_write_prep(struct zloop_cmd *cmd)
574{
575 struct request *rq = blk_mq_rq_from_pdu(cmd);
576 struct zloop_device *zlo = rq->q->queuedata;
577 unsigned int zone_no = rq_zone_no(rq);
578 sector_t nr_sectors = blk_rq_sectors(rq);
579 bool is_append = req_op(rq) == REQ_OP_ZONE_APPEND;
580 struct zloop_zone *zone = &zlo->zones[zone_no];
581 sector_t zone_end = zone->start + zlo->zone_capacity;
582 int ret = 0;
583
584 spin_lock(&zone->wp_lock);
585
586 /*
587 * Zone append operations always go at the current write pointer, but
588 * regular write operations must already be aligned to the write pointer
589 * when submitted.
590 */
591 if (is_append) {
592 /*
593 * If ordered zone append is in use, we already checked and set
594 * the target sector in zloop_queue_rq().
595 */
596 if (!zlo->ordered_zone_append) {
597 if (zone->cond == BLK_ZONE_COND_FULL ||
598 zone->wp + nr_sectors > zone_end) {
599 ret = -EIO;
600 goto out_unlock;
601 }
602 cmd->sector = zone->wp;
603 }
604 } else {
605 if (cmd->sector != zone->wp) {
606 pr_err("Zone %u: unaligned write: sect %llu, wp %llu\n",
607 zone_no, cmd->sector, zone->wp);
608 ret = -EIO;
609 goto out_unlock;
610 }
611 }
612
613 /* Implicitly open the target zone. */
614 if (!zloop_do_open_zone(zlo, zone, false)) {
615 ret = -EIO;
616 goto out_unlock;
617 }
618
619 /*
620 * Advance the write pointer, unless ordered zone append is in use. If
621 * the write fails, the write pointer position will be corrected when
622 * the next I/O starts execution.
623 */
624 if (!is_append || !zlo->ordered_zone_append) {
625 zone->wp += nr_sectors;
626 if (zone->wp == zone_end)
627 zloop_mark_full(zlo, zone);
628 }
629out_unlock:
630 spin_unlock(&zone->wp_lock);
631 return ret;
632}
633
634static void zloop_rw(struct zloop_cmd *cmd)
635{
636 struct request *rq = blk_mq_rq_from_pdu(cmd);
637 struct zloop_device *zlo = rq->q->queuedata;
638 unsigned int zone_no = rq_zone_no(rq);
639 sector_t nr_sectors = blk_rq_sectors(rq);
640 bool is_append = req_op(rq) == REQ_OP_ZONE_APPEND;
641 bool is_write = req_op(rq) == REQ_OP_WRITE || is_append;
642 struct zloop_zone *zone;
643 int ret = -EIO;
644
645 atomic_set(&cmd->ref, 2);
646 cmd->sector = blk_rq_pos(rq);
647 cmd->nr_sectors = nr_sectors;
648 cmd->ret = 0;
649
650 if (WARN_ON_ONCE(is_append && !zlo->zone_append))
651 goto out;
652
653 /* We should never get an I/O beyond the device capacity. */
654 if (WARN_ON_ONCE(zone_no >= zlo->nr_zones))
655 goto out;
656
657 zone = &zlo->zones[zone_no];
658
659 /*
660 * The block layer should never send requests that are not fully
661 * contained within the zone.
662 */
663 if (WARN_ON_ONCE(cmd->sector + nr_sectors >
664 zone->start + zlo->zone_size))
665 goto out;
666
667 if (test_and_clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags)) {
668 mutex_lock(&zone->lock);
669 ret = zloop_update_seq_zone(zlo, zone_no);
670 mutex_unlock(&zone->lock);
671 if (ret)
672 goto out;
673 }
674
675 if (!test_bit(ZLOOP_ZONE_CONV, &zone->flags) && is_write) {
676 mutex_lock(&zone->lock);
677 ret = zloop_seq_write_prep(cmd);
678 if (!ret)
679 ret = zloop_do_rw(cmd);
680 mutex_unlock(&zone->lock);
681 } else {
682 ret = zloop_do_rw(cmd);
683 }
684out:
685 if (ret != -EIOCBQUEUED)
686 zloop_rw_complete(&cmd->iocb, ret);
687 zloop_put_cmd(cmd);
688}
689
690static inline bool zloop_zone_is_active(struct zloop_zone *zone)
691{
692 switch (zone->cond) {
693 case BLK_ZONE_COND_EXP_OPEN:
694 case BLK_ZONE_COND_IMP_OPEN:
695 case BLK_ZONE_COND_CLOSED:
696 return true;
697 default:
698 return false;
699 }
700}
701
702static int zloop_record_safe_wps(struct zloop_device *zlo)
703{
704 unsigned int i;
705 int ret;
706
707 for (i = 0; i < zlo->nr_zones; i++) {
708 struct zloop_zone *zone = &zlo->zones[i];
709 struct file *file = zone->file;
710
711 if (!zloop_zone_is_active(zone))
712 continue;
713 ret = vfs_setxattr(file_mnt_idmap(file), file_dentry(file),
714 "user.zloop.wp", &zone->wp, sizeof(zone->wp), 0);
715 if (ret) {
716 pr_err("%pg: failed to record write pointer (%d)\n",
717 zlo->disk->part0, ret);
718 return ret;
719 }
720 }
721
722 return 0;
723}
724
725/*
726 * Sync the entire FS containing the zone files instead of walking all files.
727 */
728static int zloop_flush(struct zloop_device *zlo)
729{
730 struct super_block *sb = file_inode(zlo->data_dir)->i_sb;
731 int ret;
732
733 if (zlo->discard_write_cache) {
734 ret = zloop_record_safe_wps(zlo);
735 if (ret)
736 return ret;
737 }
738
739 down_read(&sb->s_umount);
740 ret = sync_filesystem(sb);
741 up_read(&sb->s_umount);
742
743 return ret;
744}
745
746static void zloop_handle_cmd(struct zloop_cmd *cmd)
747{
748 struct request *rq = blk_mq_rq_from_pdu(cmd);
749 struct zloop_device *zlo = rq->q->queuedata;
750
751 /* We can block in this context, so ignore REQ_NOWAIT. */
752 if (rq->cmd_flags & REQ_NOWAIT)
753 rq->cmd_flags &= ~REQ_NOWAIT;
754
755 switch (req_op(rq)) {
756 case REQ_OP_READ:
757 case REQ_OP_WRITE:
758 case REQ_OP_ZONE_APPEND:
759 /*
760 * zloop_rw() always executes asynchronously or completes
761 * directly.
762 */
763 zloop_rw(cmd);
764 return;
765 case REQ_OP_FLUSH:
766 cmd->ret = zloop_flush(zlo);
767 break;
768 case REQ_OP_ZONE_RESET:
769 cmd->ret = zloop_reset_zone(zlo, rq_zone_no(rq));
770 break;
771 case REQ_OP_ZONE_RESET_ALL:
772 cmd->ret = zloop_reset_all_zones(zlo);
773 break;
774 case REQ_OP_ZONE_FINISH:
775 cmd->ret = zloop_finish_zone(zlo, rq_zone_no(rq));
776 break;
777 case REQ_OP_ZONE_OPEN:
778 cmd->ret = zloop_open_zone(zlo, rq_zone_no(rq));
779 break;
780 case REQ_OP_ZONE_CLOSE:
781 cmd->ret = zloop_close_zone(zlo, rq_zone_no(rq));
782 break;
783 default:
784 WARN_ON_ONCE(1);
785 pr_err("Unsupported operation %d\n", req_op(rq));
786 cmd->ret = -EOPNOTSUPP;
787 break;
788 }
789
790 blk_mq_complete_request(rq);
791}
792
793static void zloop_cmd_workfn(struct work_struct *work)
794{
795 struct zloop_cmd *cmd = container_of(work, struct zloop_cmd, work);
796 int orig_flags = current->flags;
797
798 current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO;
799 zloop_handle_cmd(cmd);
800 current->flags = orig_flags;
801}
802
803static void zloop_fill_zero_rq(struct request *rq, unsigned int zero_from)
804{
805 struct req_iterator iter;
806 struct bio_vec bv;
807 unsigned int pos = 0;
808 unsigned int end, off;
809
810 rq_for_each_segment(bv, rq, iter) {
811 end = pos + bv.bv_len;
812 if (end > zero_from) {
813 if (pos < zero_from)
814 off = zero_from - pos;
815 else
816 off = 0;
817 memzero_page(bv.bv_page, bv.bv_offset + off,
818 bv.bv_len - off);
819 }
820 pos = end;
821 }
822}
823
824static void zloop_complete_rq(struct request *rq)
825{
826 struct zloop_cmd *cmd = blk_mq_rq_to_pdu(rq);
827 struct zloop_device *zlo = rq->q->queuedata;
828 unsigned int zone_no = cmd->sector >> zlo->zone_shift;
829 struct zloop_zone *zone = &zlo->zones[zone_no];
830 blk_status_t sts = BLK_STS_OK;
831
832 switch (req_op(rq)) {
833 case REQ_OP_READ:
834 if (cmd->ret < 0)
835 pr_err("Zone %u: failed read sector %llu, %llu sectors\n",
836 zone_no, cmd->sector, cmd->nr_sectors);
837
838 /* Short read. Zero out the unread tail beyond cmd->ret. */
839 if (cmd->ret >= 0 && cmd->ret != blk_rq_bytes(rq))
840 zloop_fill_zero_rq(rq, cmd->ret);
841 break;
842 case REQ_OP_WRITE:
843 case REQ_OP_ZONE_APPEND:
844 if (cmd->ret < 0)
845 pr_err("Zone %u: failed %swrite sector %llu, %llu sectors\n",
846 zone_no,
847 req_op(rq) == REQ_OP_WRITE ? "" : "append ",
848 cmd->sector, cmd->nr_sectors);
849
850 if (cmd->ret >= 0 && cmd->ret != blk_rq_bytes(rq)) {
851 pr_err("Zone %u: partial write %ld/%u B\n",
852 zone_no, cmd->ret, blk_rq_bytes(rq));
853 cmd->ret = -EIO;
854 }
855
856 if (cmd->ret < 0 && !test_bit(ZLOOP_ZONE_CONV, &zone->flags)) {
857 /*
858 * A write to a sequential zone file failed: mark the
859 * zone as having an error. This will be corrected and
860 * cleared when the next IO is submitted.
861 */
862 set_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags);
863 break;
864 }
865 if (req_op(rq) == REQ_OP_ZONE_APPEND)
866 rq->__sector = cmd->sector;
867
868 break;
869 default:
870 break;
871 }
872
873 if (cmd->ret < 0)
874 sts = errno_to_blk_status(cmd->ret);
875 blk_mq_end_request(rq, sts);
876}
877
878static bool zloop_set_zone_append_sector(struct request *rq)
879{
880 struct zloop_device *zlo = rq->q->queuedata;
881 unsigned int zone_no = rq_zone_no(rq);
882 struct zloop_zone *zone = &zlo->zones[zone_no];
883 sector_t zone_end = zone->start + zlo->zone_capacity;
884 sector_t nr_sectors = blk_rq_sectors(rq);
885
886 spin_lock(&zone->wp_lock);
887
888 if (zone->cond == BLK_ZONE_COND_FULL ||
889 zone->wp + nr_sectors > zone_end) {
890 spin_unlock(&zone->wp_lock);
891 return false;
892 }
893
894 rq->__sector = zone->wp;
895 zone->wp += blk_rq_sectors(rq);
896 if (zone->wp >= zone_end)
897 zloop_mark_full(zlo, zone);
898
899 spin_unlock(&zone->wp_lock);
900
901 return true;
902}
903
904static blk_status_t zloop_queue_rq(struct blk_mq_hw_ctx *hctx,
905 const struct blk_mq_queue_data *bd)
906{
907 struct request *rq = bd->rq;
908 struct zloop_cmd *cmd = blk_mq_rq_to_pdu(rq);
909 struct zloop_device *zlo = rq->q->queuedata;
910
911 if (data_race(READ_ONCE(zlo->state)) == Zlo_deleting) {
912 rq->rq_flags |= RQF_QUIET;
913 return BLK_STS_IOERR;
914 }
915
916 /*
917 * If we need to strongly order zone append operations, set the request
918 * sector to the zone write pointer location now instead of when the
919 * command work runs.
920 */
921 if (zlo->ordered_zone_append && req_op(rq) == REQ_OP_ZONE_APPEND) {
922 if (!zloop_set_zone_append_sector(rq))
923 return BLK_STS_IOERR;
924 }
925
926 blk_mq_start_request(rq);
927
928 INIT_WORK(&cmd->work, zloop_cmd_workfn);
929 queue_work(zlo->workqueue, &cmd->work);
930
931 return BLK_STS_OK;
932}
933
934static const struct blk_mq_ops zloop_mq_ops = {
935 .queue_rq = zloop_queue_rq,
936 .complete = zloop_complete_rq,
937};
938
939static int zloop_open(struct gendisk *disk, blk_mode_t mode)
940{
941 struct zloop_device *zlo = disk->private_data;
942 int ret;
943
944 ret = mutex_lock_killable(&zloop_ctl_mutex);
945 if (ret)
946 return ret;
947
948 if (zlo->state != Zlo_live)
949 ret = -ENXIO;
950 mutex_unlock(&zloop_ctl_mutex);
951 return ret;
952}
953
954static int zloop_report_zones(struct gendisk *disk, sector_t sector,
955 unsigned int nr_zones, struct blk_report_zones_args *args)
956{
957 struct zloop_device *zlo = disk->private_data;
958 struct blk_zone blkz = {};
959 unsigned int first, i;
960 int ret;
961
962 first = disk_zone_no(disk, sector);
963 if (first >= zlo->nr_zones)
964 return 0;
965 nr_zones = min(nr_zones, zlo->nr_zones - first);
966
967 for (i = 0; i < nr_zones; i++) {
968 unsigned int zone_no = first + i;
969 struct zloop_zone *zone = &zlo->zones[zone_no];
970
971 mutex_lock(&zone->lock);
972
973 if (test_and_clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags)) {
974 ret = zloop_update_seq_zone(zlo, zone_no);
975 if (ret) {
976 mutex_unlock(&zone->lock);
977 return ret;
978 }
979 }
980
981 blkz.start = zone->start;
982 blkz.len = zlo->zone_size;
983 spin_lock(&zone->wp_lock);
984 blkz.wp = zone->wp;
985 spin_unlock(&zone->wp_lock);
986 blkz.cond = zone->cond;
987 if (test_bit(ZLOOP_ZONE_CONV, &zone->flags)) {
988 blkz.type = BLK_ZONE_TYPE_CONVENTIONAL;
989 blkz.capacity = zlo->zone_size;
990 } else {
991 blkz.type = BLK_ZONE_TYPE_SEQWRITE_REQ;
992 blkz.capacity = zlo->zone_capacity;
993 }
994
995 mutex_unlock(&zone->lock);
996
997 ret = disk_report_zone(disk, &blkz, i, args);
998 if (ret)
999 return ret;
1000 }
1001
1002 return nr_zones;
1003}
1004
1005static void zloop_free_disk(struct gendisk *disk)
1006{
1007 struct zloop_device *zlo = disk->private_data;
1008 unsigned int i;
1009
1010 blk_mq_free_tag_set(&zlo->tag_set);
1011
1012 for (i = 0; i < zlo->nr_zones; i++) {
1013 struct zloop_zone *zone = &zlo->zones[i];
1014
1015 mapping_set_gfp_mask(zone->file->f_mapping,
1016 zone->old_gfp_mask);
1017 fput(zone->file);
1018 }
1019
1020 fput(zlo->data_dir);
1021 destroy_workqueue(zlo->workqueue);
1022 kfree(zlo->base_dir);
1023 kvfree(zlo);
1024}
1025
1026static const struct block_device_operations zloop_fops = {
1027 .owner = THIS_MODULE,
1028 .open = zloop_open,
1029 .report_zones = zloop_report_zones,
1030 .free_disk = zloop_free_disk,
1031};
1032
1033__printf(3, 4)
1034static struct file *zloop_filp_open_fmt(int oflags, umode_t mode,
1035 const char *fmt, ...)
1036{
1037 struct file *file;
1038 va_list ap;
1039 char *p;
1040
1041 va_start(ap, fmt);
1042 p = kvasprintf(GFP_KERNEL, fmt, ap);
1043 va_end(ap);
1044
1045 if (!p)
1046 return ERR_PTR(-ENOMEM);
1047 file = filp_open(p, oflags, mode);
1048 kfree(p);
1049 return file;
1050}
1051
1052static int zloop_get_block_size(struct zloop_device *zlo,
1053 struct zloop_zone *zone)
1054{
1055 struct block_device *sb_bdev = zone->file->f_mapping->host->i_sb->s_bdev;
1056 struct kstat st;
1057
1058 /*
1059 * Use the dio alignment of the file system if provided. The incoming
1060 * request's bio_vec is forwarded to the backing file unchanged, so its
1061 * required memory alignment becomes the device's dma_alignment when
1062 * used for direct-io. The file system reports zeroed alignments if the
1063 * file can't be used for direct-io at all, so fall back to the block
1064 * device limits in that case.
1065 */
1066 if (!vfs_getattr(&zone->file->f_path, &st, STATX_DIOALIGN, 0) &&
1067 (st.result_mask & STATX_DIOALIGN) && st.dio_mem_align) {
1068 zlo->block_size = st.dio_offset_align;
1069 zlo->dio_mem_align = min(st.dio_mem_align - 1, PAGE_SIZE - 1);
1070 } else if (sb_bdev) {
1071 zlo->block_size = bdev_physical_block_size(sb_bdev);
1072 zlo->dio_mem_align = bdev_dma_alignment(sb_bdev);
1073 } else {
1074 zlo->block_size = SECTOR_SIZE;
1075 zlo->dio_mem_align = SECTOR_SIZE - 1;
1076 }
1077
1078 /*
1079 * Prefer the FS block size for the device block size when it is no
1080 * larger than 4K; otherwise keep the direct I/O / physical block size
1081 * selected above.
1082 */
1083 if (file_inode(zone->file)->i_sb->s_blocksize <= SZ_4K)
1084 zlo->block_size = file_inode(zone->file)->i_sb->s_blocksize;
1085
1086 if (zlo->zone_capacity & ((zlo->block_size >> SECTOR_SHIFT) - 1)) {
1087 pr_err("Zone capacity is not aligned to block size %u\n",
1088 zlo->block_size);
1089 return -EINVAL;
1090 }
1091
1092 return 0;
1093}
1094
1095static int zloop_init_zone(struct zloop_device *zlo, struct zloop_options *opts,
1096 unsigned int zone_no, bool restore)
1097{
1098 struct zloop_zone *zone = &zlo->zones[zone_no];
1099 int oflags = O_RDWR;
1100 struct kstat stat;
1101 sector_t file_sectors;
1102 int ret;
1103
1104 mutex_init(&zone->lock);
1105 INIT_LIST_HEAD(&zone->open_zone_entry);
1106 spin_lock_init(&zone->wp_lock);
1107 zone->start = (sector_t)zone_no << zlo->zone_shift;
1108
1109 if (!restore)
1110 oflags |= O_CREAT;
1111
1112 if (!opts->buffered_io)
1113 oflags |= O_DIRECT;
1114
1115 if (zone_no < zlo->nr_conv_zones) {
1116 /* Conventional zone file. */
1117 set_bit(ZLOOP_ZONE_CONV, &zone->flags);
1118 zone->cond = BLK_ZONE_COND_NOT_WP;
1119 zone->wp = U64_MAX;
1120
1121 zone->file = zloop_filp_open_fmt(oflags, 0600, "%s/%u/cnv-%06u",
1122 zlo->base_dir, zlo->id, zone_no);
1123 if (IS_ERR(zone->file)) {
1124 pr_err("Failed to open zone %u file %s/%u/cnv-%06u (err=%ld)",
1125 zone_no, zlo->base_dir, zlo->id, zone_no,
1126 PTR_ERR(zone->file));
1127 return PTR_ERR(zone->file);
1128 }
1129
1130 if (!zlo->block_size) {
1131 ret = zloop_get_block_size(zlo, zone);
1132 if (ret)
1133 return ret;
1134 }
1135
1136 ret = vfs_getattr(&zone->file->f_path, &stat, STATX_SIZE, 0);
1137 if (ret < 0) {
1138 pr_err("Failed to get zone %u file stat\n", zone_no);
1139 return ret;
1140 }
1141 file_sectors = stat.size >> SECTOR_SHIFT;
1142
1143 if (restore && file_sectors != zlo->zone_size) {
1144 pr_err("Invalid conventional zone %u file size (%llu sectors != %llu)\n",
1145 zone_no, file_sectors, zlo->zone_capacity);
1146 return ret;
1147 }
1148
1149 ret = vfs_truncate(&zone->file->f_path,
1150 zlo->zone_size << SECTOR_SHIFT);
1151 if (ret < 0) {
1152 pr_err("Failed to truncate zone %u file (err=%d)\n",
1153 zone_no, ret);
1154 return ret;
1155 }
1156
1157 return 0;
1158 }
1159
1160 /* Sequential zone file. */
1161 zone->file = zloop_filp_open_fmt(oflags, 0600, "%s/%u/seq-%06u",
1162 zlo->base_dir, zlo->id, zone_no);
1163 if (IS_ERR(zone->file)) {
1164 pr_err("Failed to open zone %u file %s/%u/seq-%06u (err=%ld)",
1165 zone_no, zlo->base_dir, zlo->id, zone_no,
1166 PTR_ERR(zone->file));
1167 return PTR_ERR(zone->file);
1168 }
1169
1170 if (!zlo->block_size) {
1171 ret = zloop_get_block_size(zlo, zone);
1172 if (ret)
1173 return ret;
1174 }
1175
1176 zloop_get_block_size(zlo, zone);
1177
1178 mutex_lock(&zone->lock);
1179 ret = zloop_update_seq_zone(zlo, zone_no);
1180 mutex_unlock(&zone->lock);
1181
1182 return ret;
1183}
1184
1185static bool zloop_dev_exists(struct zloop_device *zlo)
1186{
1187 struct file *cnv, *seq;
1188 bool exists;
1189
1190 cnv = zloop_filp_open_fmt(O_RDONLY, 0600, "%s/%u/cnv-%06u",
1191 zlo->base_dir, zlo->id, 0);
1192 seq = zloop_filp_open_fmt(O_RDONLY, 0600, "%s/%u/seq-%06u",
1193 zlo->base_dir, zlo->id, 0);
1194 exists = !IS_ERR(cnv) || !IS_ERR(seq);
1195
1196 if (!IS_ERR(cnv))
1197 fput(cnv);
1198 if (!IS_ERR(seq))
1199 fput(seq);
1200
1201 return exists;
1202}
1203
1204static int zloop_ctl_add(struct zloop_options *opts)
1205{
1206 struct queue_limits lim = {
1207 .max_hw_sectors = SZ_1M >> SECTOR_SHIFT,
1208 .chunk_sectors = opts->zone_size,
1209 .features = BLK_FEAT_ZONED | BLK_FEAT_WRITE_CACHE,
1210
1211 };
1212 unsigned int nr_zones, i, j;
1213 struct zloop_device *zlo;
1214 int ret = -EINVAL;
1215 bool restore;
1216
1217 __module_get(THIS_MODULE);
1218
1219 nr_zones = opts->capacity >> ilog2(opts->zone_size);
1220 if (opts->nr_conv_zones >= nr_zones) {
1221 pr_err("Invalid number of conventional zones %u\n",
1222 opts->nr_conv_zones);
1223 goto out;
1224 }
1225
1226 if (opts->max_open_zones > nr_zones - opts->nr_conv_zones) {
1227 pr_err("Invalid maximum number of open zones %u\n",
1228 opts->max_open_zones);
1229 goto out;
1230 }
1231
1232 zlo = kvzalloc_flex(*zlo, zones, nr_zones);
1233 if (!zlo) {
1234 ret = -ENOMEM;
1235 goto out;
1236 }
1237 WRITE_ONCE(zlo->state, Zlo_creating);
1238 spin_lock_init(&zlo->open_zones_lock);
1239 INIT_LIST_HEAD(&zlo->open_zones_lru_list);
1240
1241 ret = mutex_lock_killable(&zloop_ctl_mutex);
1242 if (ret)
1243 goto out_free_dev;
1244
1245 /* Allocate id, if @opts->id >= 0, we're requesting that specific id */
1246 if (opts->id >= 0) {
1247 ret = idr_alloc(&zloop_index_idr, zlo,
1248 opts->id, opts->id + 1, GFP_KERNEL);
1249 if (ret == -ENOSPC)
1250 ret = -EEXIST;
1251 } else {
1252 ret = idr_alloc(&zloop_index_idr, zlo, 0, 0, GFP_KERNEL);
1253 }
1254 mutex_unlock(&zloop_ctl_mutex);
1255 if (ret < 0)
1256 goto out_free_dev;
1257
1258 zlo->id = ret;
1259 zlo->zone_shift = ilog2(opts->zone_size);
1260 zlo->zone_size = opts->zone_size;
1261 if (opts->zone_capacity)
1262 zlo->zone_capacity = opts->zone_capacity;
1263 else
1264 zlo->zone_capacity = zlo->zone_size;
1265 zlo->nr_zones = nr_zones;
1266 zlo->nr_conv_zones = opts->nr_conv_zones;
1267 zlo->max_open_zones = opts->max_open_zones;
1268 zlo->buffered_io = opts->buffered_io;
1269 zlo->zone_append = opts->zone_append;
1270 if (zlo->zone_append)
1271 zlo->ordered_zone_append = opts->ordered_zone_append;
1272 zlo->discard_write_cache = opts->discard_write_cache;
1273
1274 zlo->workqueue = alloc_workqueue("zloop%d", WQ_UNBOUND | WQ_FREEZABLE,
1275 opts->nr_queues * opts->queue_depth, zlo->id);
1276 if (!zlo->workqueue) {
1277 ret = -ENOMEM;
1278 goto out_free_idr;
1279 }
1280
1281 if (opts->base_dir)
1282 zlo->base_dir = kstrdup(opts->base_dir, GFP_KERNEL);
1283 else
1284 zlo->base_dir = kstrdup(ZLOOP_DEF_BASE_DIR, GFP_KERNEL);
1285 if (!zlo->base_dir) {
1286 ret = -ENOMEM;
1287 goto out_destroy_workqueue;
1288 }
1289
1290 zlo->data_dir = zloop_filp_open_fmt(O_RDONLY | O_DIRECTORY, 0, "%s/%u",
1291 zlo->base_dir, zlo->id);
1292 if (IS_ERR(zlo->data_dir)) {
1293 ret = PTR_ERR(zlo->data_dir);
1294 pr_warn("Failed to open directory %s/%u (err=%d)\n",
1295 zlo->base_dir, zlo->id, ret);
1296 goto out_free_base_dir;
1297 }
1298
1299 /*
1300 * If we already have zone files, we are restoring a device created by a
1301 * previous add operation. In this case, zloop_init_zone() will check
1302 * that the zone files are consistent with the zone configuration given.
1303 */
1304 restore = zloop_dev_exists(zlo);
1305 for (i = 0; i < nr_zones; i++) {
1306 ret = zloop_init_zone(zlo, opts, i, restore);
1307 if (ret)
1308 goto out_close_files;
1309 }
1310
1311 lim.physical_block_size = zlo->block_size;
1312 lim.logical_block_size = zlo->block_size;
1313 /* Direct I/O forwards the request pages to the backing files as-is. */
1314 if (!opts->buffered_io)
1315 lim.dma_alignment = max_t(unsigned int, zlo->dio_mem_align,
1316 SECTOR_SIZE - 1);
1317 if (zlo->zone_append)
1318 lim.max_hw_zone_append_sectors = lim.max_hw_sectors;
1319 lim.max_open_zones = zlo->max_open_zones;
1320
1321 zlo->tag_set.ops = &zloop_mq_ops;
1322 zlo->tag_set.nr_hw_queues = opts->nr_queues;
1323 zlo->tag_set.queue_depth = opts->queue_depth;
1324 zlo->tag_set.numa_node = NUMA_NO_NODE;
1325 zlo->tag_set.cmd_size = sizeof(struct zloop_cmd);
1326 zlo->tag_set.driver_data = zlo;
1327
1328 ret = blk_mq_alloc_tag_set(&zlo->tag_set);
1329 if (ret) {
1330 pr_err("blk_mq_alloc_tag_set failed (err=%d)\n", ret);
1331 goto out_close_files;
1332 }
1333
1334 zlo->disk = blk_mq_alloc_disk(&zlo->tag_set, &lim, zlo);
1335 if (IS_ERR(zlo->disk)) {
1336 pr_err("blk_mq_alloc_disk failed (err=%d)\n", ret);
1337 ret = PTR_ERR(zlo->disk);
1338 goto out_cleanup_tags;
1339 }
1340 zlo->disk->flags = GENHD_FL_NO_PART;
1341 zlo->disk->fops = &zloop_fops;
1342 zlo->disk->private_data = zlo;
1343 sprintf(zlo->disk->disk_name, "zloop%d", zlo->id);
1344 set_capacity(zlo->disk, (u64)lim.chunk_sectors * zlo->nr_zones);
1345
1346 ret = blk_revalidate_disk_zones(zlo->disk);
1347 if (ret)
1348 goto out_cleanup_disk;
1349
1350 ret = add_disk(zlo->disk);
1351 if (ret) {
1352 pr_err("add_disk failed (err=%d)\n", ret);
1353 goto out_cleanup_disk;
1354 }
1355
1356 mutex_lock(&zloop_ctl_mutex);
1357 WRITE_ONCE(zlo->state, Zlo_live);
1358 mutex_unlock(&zloop_ctl_mutex);
1359
1360 pr_info("zloop: device %d, %u zones of %llu MiB, %u B block size\n",
1361 zlo->id, zlo->nr_zones,
1362 ((sector_t)zlo->zone_size << SECTOR_SHIFT) >> 20,
1363 zlo->block_size);
1364 pr_info("zloop%d: using %s%s zone append\n",
1365 zlo->id,
1366 zlo->ordered_zone_append ? "ordered " : "",
1367 zlo->zone_append ? "native" : "emulated");
1368
1369 return 0;
1370
1371out_cleanup_disk:
1372 put_disk(zlo->disk);
1373out_cleanup_tags:
1374 blk_mq_free_tag_set(&zlo->tag_set);
1375out_close_files:
1376 for (j = 0; j < i; j++) {
1377 struct zloop_zone *zone = &zlo->zones[j];
1378
1379 if (!IS_ERR_OR_NULL(zone->file))
1380 fput(zone->file);
1381 }
1382 fput(zlo->data_dir);
1383out_free_base_dir:
1384 kfree(zlo->base_dir);
1385out_destroy_workqueue:
1386 destroy_workqueue(zlo->workqueue);
1387out_free_idr:
1388 mutex_lock(&zloop_ctl_mutex);
1389 idr_remove(&zloop_index_idr, zlo->id);
1390 mutex_unlock(&zloop_ctl_mutex);
1391out_free_dev:
1392 kvfree(zlo);
1393out:
1394 module_put(THIS_MODULE);
1395 if (ret == -ENOENT)
1396 ret = -EINVAL;
1397 return ret;
1398}
1399
1400static void zloop_forget_cache(struct zloop_device *zlo)
1401{
1402 unsigned int i;
1403 int ret;
1404
1405 pr_info("%pg: discarding volatile write cache\n", zlo->disk->part0);
1406
1407 for (i = 0; i < zlo->nr_zones; i++) {
1408 struct zloop_zone *zone = &zlo->zones[i];
1409 struct file *file = zone->file;
1410 sector_t old_wp;
1411
1412 if (!zloop_zone_is_active(zone))
1413 continue;
1414
1415 ret = vfs_getxattr(file_mnt_idmap(file), file_dentry(file),
1416 "user.zloop.wp", &old_wp, sizeof(old_wp));
1417 if (ret == -ENODATA) {
1418 old_wp = 0;
1419 } else if (ret != sizeof(old_wp)) {
1420 pr_err("%pg: failed to retrieve write pointer (%d)\n",
1421 zlo->disk->part0, ret);
1422 continue;
1423 }
1424
1425 if (old_wp > zone->wp)
1426 continue;
1427 /*
1428 * This should not happen, if we recored a full zone, it can't
1429 * be active.
1430 */
1431 if (WARN_ON_ONCE(old_wp == ULLONG_MAX))
1432 continue;
1433
1434 vfs_truncate(&file->f_path,
1435 (old_wp - zone->start) << SECTOR_SHIFT);
1436 }
1437}
1438
1439static int zloop_ctl_remove(struct zloop_options *opts)
1440{
1441 struct zloop_device *zlo;
1442 int ret;
1443
1444 if (!(opts->mask & ZLOOP_OPT_ID)) {
1445 pr_err("No ID specified for remove\n");
1446 return -EINVAL;
1447 }
1448
1449 if (opts->mask & ~ZLOOP_OPT_ID) {
1450 pr_err("Invalid option specified for remove\n");
1451 return -EINVAL;
1452 }
1453
1454 ret = mutex_lock_killable(&zloop_ctl_mutex);
1455 if (ret)
1456 return ret;
1457
1458 zlo = idr_find(&zloop_index_idr, opts->id);
1459 if (!zlo || zlo->state == Zlo_creating) {
1460 ret = -ENODEV;
1461 } else if (zlo->state == Zlo_deleting) {
1462 ret = -EINVAL;
1463 } else {
1464 idr_remove(&zloop_index_idr, zlo->id);
1465 WRITE_ONCE(zlo->state, Zlo_deleting);
1466 }
1467
1468 mutex_unlock(&zloop_ctl_mutex);
1469 if (ret)
1470 return ret;
1471
1472 del_gendisk(zlo->disk);
1473
1474 if (zlo->discard_write_cache)
1475 zloop_forget_cache(zlo);
1476
1477 put_disk(zlo->disk);
1478
1479 pr_info("Removed device %d\n", opts->id);
1480
1481 module_put(THIS_MODULE);
1482
1483 return 0;
1484}
1485
1486static int zloop_parse_options(struct zloop_options *opts, const char *buf)
1487{
1488 substring_t args[MAX_OPT_ARGS];
1489 char *options, *o, *p;
1490 unsigned int token;
1491 int ret = 0;
1492
1493 /* Set defaults. */
1494 opts->mask = 0;
1495 opts->id = ZLOOP_DEF_ID;
1496 opts->capacity = ZLOOP_DEF_ZONE_SIZE * ZLOOP_DEF_NR_ZONES;
1497 opts->zone_size = ZLOOP_DEF_ZONE_SIZE;
1498 opts->nr_conv_zones = ZLOOP_DEF_NR_CONV_ZONES;
1499 opts->max_open_zones = ZLOOP_DEF_MAX_OPEN_ZONES;
1500 opts->nr_queues = ZLOOP_DEF_NR_QUEUES;
1501 opts->queue_depth = ZLOOP_DEF_QUEUE_DEPTH;
1502 opts->buffered_io = ZLOOP_DEF_BUFFERED_IO;
1503 opts->zone_append = ZLOOP_DEF_ZONE_APPEND;
1504 opts->ordered_zone_append = ZLOOP_DEF_ORDERED_ZONE_APPEND;
1505
1506 if (!buf)
1507 return 0;
1508
1509 /* Skip leading spaces before the options. */
1510 while (isspace(*buf))
1511 buf++;
1512
1513 options = o = kstrdup(buf, GFP_KERNEL);
1514 if (!options)
1515 return -ENOMEM;
1516
1517 /* Parse the options, doing only some light invalid value checks. */
1518 while ((p = strsep(&o, ",\n")) != NULL) {
1519 if (!*p)
1520 continue;
1521
1522 token = match_token(p, zloop_opt_tokens, args);
1523 opts->mask |= token;
1524 switch (token) {
1525 case ZLOOP_OPT_ID:
1526 if (match_int(args, &opts->id)) {
1527 ret = -EINVAL;
1528 goto out;
1529 }
1530 break;
1531 case ZLOOP_OPT_CAPACITY:
1532 if (match_uint(args, &token)) {
1533 ret = -EINVAL;
1534 goto out;
1535 }
1536 if (!token) {
1537 pr_err("Invalid capacity\n");
1538 ret = -EINVAL;
1539 goto out;
1540 }
1541 opts->capacity =
1542 ((sector_t)token * SZ_1M) >> SECTOR_SHIFT;
1543 break;
1544 case ZLOOP_OPT_ZONE_SIZE:
1545 if (match_uint(args, &token)) {
1546 ret = -EINVAL;
1547 goto out;
1548 }
1549 if (!token || token > ZLOOP_MAX_ZONE_SIZE_MB ||
1550 !is_power_of_2(token)) {
1551 pr_err("Invalid zone size %u\n", token);
1552 ret = -EINVAL;
1553 goto out;
1554 }
1555 opts->zone_size =
1556 ((sector_t)token * SZ_1M) >> SECTOR_SHIFT;
1557 break;
1558 case ZLOOP_OPT_ZONE_CAPACITY:
1559 if (match_uint(args, &token)) {
1560 ret = -EINVAL;
1561 goto out;
1562 }
1563 if (!token) {
1564 pr_err("Invalid zone capacity\n");
1565 ret = -EINVAL;
1566 goto out;
1567 }
1568 opts->zone_capacity =
1569 ((sector_t)token * SZ_1M) >> SECTOR_SHIFT;
1570 break;
1571 case ZLOOP_OPT_NR_CONV_ZONES:
1572 if (match_uint(args, &token)) {
1573 ret = -EINVAL;
1574 goto out;
1575 }
1576 opts->nr_conv_zones = token;
1577 break;
1578 case ZLOOP_OPT_MAX_OPEN_ZONES:
1579 if (match_uint(args, &token)) {
1580 ret = -EINVAL;
1581 goto out;
1582 }
1583 opts->max_open_zones = token;
1584 break;
1585 case ZLOOP_OPT_BASE_DIR:
1586 p = match_strdup(args);
1587 if (!p) {
1588 ret = -ENOMEM;
1589 goto out;
1590 }
1591 kfree(opts->base_dir);
1592 opts->base_dir = p;
1593 break;
1594 case ZLOOP_OPT_NR_QUEUES:
1595 if (match_uint(args, &token)) {
1596 ret = -EINVAL;
1597 goto out;
1598 }
1599 if (!token) {
1600 pr_err("Invalid number of queues\n");
1601 ret = -EINVAL;
1602 goto out;
1603 }
1604 opts->nr_queues = min(token, num_online_cpus());
1605 break;
1606 case ZLOOP_OPT_QUEUE_DEPTH:
1607 if (match_uint(args, &token)) {
1608 ret = -EINVAL;
1609 goto out;
1610 }
1611 if (!token) {
1612 pr_err("Invalid queue depth\n");
1613 ret = -EINVAL;
1614 goto out;
1615 }
1616 opts->queue_depth = token;
1617 break;
1618 case ZLOOP_OPT_BUFFERED_IO:
1619 opts->buffered_io = true;
1620 break;
1621 case ZLOOP_OPT_ZONE_APPEND:
1622 if (match_uint(args, &token)) {
1623 ret = -EINVAL;
1624 goto out;
1625 }
1626 if (token != 0 && token != 1) {
1627 pr_err("Invalid zone_append value\n");
1628 ret = -EINVAL;
1629 goto out;
1630 }
1631 opts->zone_append = token;
1632 break;
1633 case ZLOOP_OPT_ORDERED_ZONE_APPEND:
1634 opts->ordered_zone_append = true;
1635 break;
1636 case ZLOOP_OPT_DISCARD_WRITE_CACHE:
1637 opts->discard_write_cache = true;
1638 break;
1639 case ZLOOP_OPT_ERR:
1640 default:
1641 pr_warn("unknown parameter or missing value '%s'\n", p);
1642 ret = -EINVAL;
1643 goto out;
1644 }
1645 }
1646
1647 ret = -EINVAL;
1648 if (opts->capacity <= opts->zone_size) {
1649 pr_err("Invalid capacity\n");
1650 goto out;
1651 }
1652
1653 if (opts->zone_capacity > opts->zone_size) {
1654 pr_err("Invalid zone capacity\n");
1655 goto out;
1656 }
1657
1658 ret = 0;
1659out:
1660 kfree(options);
1661 return ret;
1662}
1663
1664enum {
1665 ZLOOP_CTL_ADD,
1666 ZLOOP_CTL_REMOVE,
1667};
1668
1669static struct zloop_ctl_op {
1670 int code;
1671 const char *name;
1672} zloop_ctl_ops[] = {
1673 { ZLOOP_CTL_ADD, "add" },
1674 { ZLOOP_CTL_REMOVE, "remove" },
1675 { -1, NULL },
1676};
1677
1678static ssize_t zloop_ctl_write(struct file *file, const char __user *ubuf,
1679 size_t count, loff_t *pos)
1680{
1681 struct zloop_options opts = { };
1682 struct zloop_ctl_op *op;
1683 const char *buf, *opts_buf;
1684 int i, ret;
1685
1686 if (count > PAGE_SIZE)
1687 return -ENOMEM;
1688
1689 buf = memdup_user_nul(ubuf, count);
1690 if (IS_ERR(buf))
1691 return PTR_ERR(buf);
1692
1693 for (i = 0; i < ARRAY_SIZE(zloop_ctl_ops); i++) {
1694 op = &zloop_ctl_ops[i];
1695 if (!op->name) {
1696 pr_err("Invalid operation\n");
1697 ret = -EINVAL;
1698 goto out;
1699 }
1700 if (!strncmp(buf, op->name, strlen(op->name)))
1701 break;
1702 }
1703
1704 if (count <= strlen(op->name))
1705 opts_buf = NULL;
1706 else
1707 opts_buf = buf + strlen(op->name);
1708
1709 ret = zloop_parse_options(&opts, opts_buf);
1710 if (ret) {
1711 pr_err("Failed to parse options\n");
1712 goto out;
1713 }
1714
1715 switch (op->code) {
1716 case ZLOOP_CTL_ADD:
1717 ret = zloop_ctl_add(&opts);
1718 break;
1719 case ZLOOP_CTL_REMOVE:
1720 ret = zloop_ctl_remove(&opts);
1721 break;
1722 default:
1723 pr_err("Invalid operation\n");
1724 ret = -EINVAL;
1725 goto out;
1726 }
1727
1728out:
1729 kfree(opts.base_dir);
1730 kfree(buf);
1731 return ret ? ret : count;
1732}
1733
1734static int zloop_ctl_show(struct seq_file *seq_file, void *private)
1735{
1736 const struct match_token *tok;
1737 int i;
1738
1739 /* Add operation */
1740 seq_printf(seq_file, "%s ", zloop_ctl_ops[0].name);
1741 for (i = 0; i < ARRAY_SIZE(zloop_opt_tokens); i++) {
1742 tok = &zloop_opt_tokens[i];
1743 if (!tok->pattern)
1744 break;
1745 if (i)
1746 seq_putc(seq_file, ',');
1747 seq_puts(seq_file, tok->pattern);
1748 }
1749 seq_putc(seq_file, '\n');
1750
1751 /* Remove operation */
1752 seq_puts(seq_file, zloop_ctl_ops[1].name);
1753 seq_puts(seq_file, " id=%d\n");
1754
1755 return 0;
1756}
1757
1758static int zloop_ctl_open(struct inode *inode, struct file *file)
1759{
1760 file->private_data = NULL;
1761 return single_open(file, zloop_ctl_show, NULL);
1762}
1763
1764static int zloop_ctl_release(struct inode *inode, struct file *file)
1765{
1766 return single_release(inode, file);
1767}
1768
1769static const struct file_operations zloop_ctl_fops = {
1770 .owner = THIS_MODULE,
1771 .open = zloop_ctl_open,
1772 .release = zloop_ctl_release,
1773 .write = zloop_ctl_write,
1774 .read = seq_read,
1775};
1776
1777static struct miscdevice zloop_misc = {
1778 .minor = MISC_DYNAMIC_MINOR,
1779 .name = "zloop-control",
1780 .fops = &zloop_ctl_fops,
1781};
1782
1783static int __init zloop_init(void)
1784{
1785 int ret;
1786
1787 ret = misc_register(&zloop_misc);
1788 if (ret) {
1789 pr_err("Failed to register misc device: %d\n", ret);
1790 return ret;
1791 }
1792 pr_info("Module loaded\n");
1793
1794 return 0;
1795}
1796
1797static void __exit zloop_exit(void)
1798{
1799 misc_deregister(&zloop_misc);
1800 idr_destroy(&zloop_index_idr);
1801}
1802
1803module_init(zloop_init);
1804module_exit(zloop_exit);
1805
1806MODULE_DESCRIPTION("Zoned loopback device");
1807MODULE_LICENSE("GPL");