projects/linux·blob master · refs

1333 lines · 36.0 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 1991, 1992 Linus Torvalds
4 * Copyright (C) 1994, Karl Keyte: Added support for disk statistics
5 * Elevator latency, (C) 2000 Andrea Arcangeli <[email protected]> SuSE
6 * Queue request tables / lock, selectable elevator, Jens Axboe <[email protected]>
7 * kernel-doc documentation started by NeilBrown <[email protected]>
8 * - July2000
9 * bio rewrite, highmem i/o, etc, Jens Axboe <[email protected]> - may 2001
10 */
11
12/*
13 * This handles all read/write requests to block devices
14 */
15#include <linux/kernel.h>
16#include <linux/module.h>
17#include <linux/bio.h>
18#include <linux/blkdev.h>
19#include <linux/blk-pm.h>
20#include <linux/blk-integrity.h>
21#include <linux/highmem.h>
22#include <linux/mm.h>
23#include <linux/pagemap.h>
24#include <linux/kernel_stat.h>
25#include <linux/string.h>
26#include <linux/init.h>
27#include <linux/completion.h>
28#include <linux/slab.h>
29#include <linux/swap.h>
30#include <linux/writeback.h>
31#include <linux/task_io_accounting_ops.h>
32#include <linux/fault-inject.h>
33#include <linux/list_sort.h>
34#include <linux/delay.h>
35#include <linux/ratelimit.h>
36#include <linux/pm_runtime.h>
37#include <linux/t10-pi.h>
38#include <linux/debugfs.h>
39#include <linux/bpf.h>
40#include <linux/part_stat.h>
41#include <linux/sched/sysctl.h>
42#include <linux/blk-crypto.h>
43
44#define CREATE_TRACE_POINTS
45#include <trace/events/block.h>
46
47#include "blk.h"
48#include "blk-mq-sched.h"
49#include "blk-pm.h"
50#include "blk-cgroup.h"
51#include "blk-throttle.h"
52#include "blk-ioprio.h"
53#include "error-injection.h"
54
55struct dentry *blk_debugfs_root;
56
57EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
58EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
59EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
60EXPORT_TRACEPOINT_SYMBOL_GPL(block_split);
61EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug);
62EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_insert);
63
64static DEFINE_IDA(blk_queue_ida);
65
66/*
67 * For queue allocation
68 */
69static struct kmem_cache *blk_requestq_cachep;
70
71/*
72 * Controlling structure to kblockd
73 */
74static struct workqueue_struct *kblockd_workqueue;
75
76/**
77 * blk_queue_flag_set - atomically set a queue flag
78 * @flag: flag to be set
79 * @q: request queue
80 */
81void blk_queue_flag_set(unsigned int flag, struct request_queue *q)
82{
83 set_bit(flag, &q->queue_flags);
84}
85EXPORT_SYMBOL(blk_queue_flag_set);
86
87/**
88 * blk_queue_flag_clear - atomically clear a queue flag
89 * @flag: flag to be cleared
90 * @q: request queue
91 */
92void blk_queue_flag_clear(unsigned int flag, struct request_queue *q)
93{
94 clear_bit(flag, &q->queue_flags);
95}
96EXPORT_SYMBOL(blk_queue_flag_clear);
97
98#define REQ_OP_NAME(name) [REQ_OP_##name] = #name
99static const char *const blk_op_name[] = {
100 REQ_OP_NAME(READ),
101 REQ_OP_NAME(WRITE),
102 REQ_OP_NAME(FLUSH),
103 REQ_OP_NAME(DISCARD),
104 REQ_OP_NAME(SECURE_ERASE),
105 REQ_OP_NAME(ZONE_RESET),
106 REQ_OP_NAME(ZONE_RESET_ALL),
107 REQ_OP_NAME(ZONE_OPEN),
108 REQ_OP_NAME(ZONE_CLOSE),
109 REQ_OP_NAME(ZONE_FINISH),
110 REQ_OP_NAME(ZONE_APPEND),
111 REQ_OP_NAME(WRITE_ZEROES),
112 REQ_OP_NAME(DRV_IN),
113 REQ_OP_NAME(DRV_OUT),
114};
115#undef REQ_OP_NAME
116
117/**
118 * blk_op_str - Return the string "name" for an operation REQ_OP_name.
119 * @op: a request operation.
120 *
121 * Convert a request operation REQ_OP_name into the string "name". Useful for
122 * debugging and tracing BIOs and requests. For an invalid request operation
123 * code, the string "UNKNOWN" is returned.
124 */
125inline const char *blk_op_str(enum req_op op)
126{
127 const char *op_str = "UNKNOWN";
128
129 if (op < ARRAY_SIZE(blk_op_name) && blk_op_name[op])
130 op_str = blk_op_name[op];
131
132 return op_str;
133}
134EXPORT_SYMBOL_GPL(blk_op_str);
135
136enum req_op str_to_blk_op(const char *op)
137{
138 int i;
139
140 for (i = 0; i < ARRAY_SIZE(blk_op_name); i++)
141 if (blk_op_name[i] && !strcmp(blk_op_name[i], op))
142 return (enum req_op)i;
143 return REQ_OP_LAST;
144}
145
146#define ENT(_tag, _errno, _desc) \
147[BLK_STS_##_tag] = { \
148 .errno = _errno, \
149 .tag = __stringify(_tag), \
150 .name = _desc, \
151}
152static const struct {
153 int errno;
154 const char *tag;
155 const char *name;
156} blk_errors[] = {
157 ENT(OK, 0, ""),
158 ENT(NOTSUPP, -EOPNOTSUPP, "operation not supported"),
159 ENT(TIMEOUT, -ETIMEDOUT, "timeout"),
160 ENT(NOSPC, -ENOSPC, "critical space allocation"),
161 ENT(TRANSPORT, -ENOLINK, "recoverable transport"),
162 ENT(TARGET, -EREMOTEIO, "critical target"),
163 ENT(RESV_CONFLICT, -EBADE, "reservation conflict"),
164 ENT(MEDIUM, -ENODATA, "critical medium"),
165 ENT(PROTECTION, -EILSEQ, "protection"),
166 ENT(RESOURCE, -ENOMEM, "kernel resource"),
167 ENT(DEV_RESOURCE, -EBUSY, "device resource"),
168 ENT(AGAIN, -EAGAIN, "nonblocking retry"),
169 ENT(OFFLINE, -ENODEV, "device offline"),
170
171 /* device mapper special case, should not leak out: */
172 ENT(DM_REQUEUE, -EREMCHG, "dm internal retry"),
173
174 /* zone device specific errors */
175 ENT(ZONE_OPEN_RESOURCE, -ETOOMANYREFS, "open zones exceeded"),
176 ENT(ZONE_ACTIVE_RESOURCE, -EOVERFLOW, "active zones exceeded"),
177
178 /* Command duration limit device-side timeout */
179 ENT(DURATION_LIMIT, -ETIME, "duration limit exceeded"),
180 ENT(INVAL, -EINVAL, "invalid"),
181
182 /* everything else not covered above: */
183 ENT(IOERR, -EIO, "I/O"),
184};
185#undef ENT
186
187blk_status_t errno_to_blk_status(int errno)
188{
189 int i;
190
191 for (i = 0; i < ARRAY_SIZE(blk_errors); i++) {
192 if (blk_errors[i].errno == errno)
193 return (__force blk_status_t)i;
194 }
195
196 return BLK_STS_IOERR;
197}
198EXPORT_SYMBOL_GPL(errno_to_blk_status);
199
200int blk_status_to_errno(blk_status_t status)
201{
202 int idx = (__force int)status;
203
204 if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
205 return -EIO;
206 return blk_errors[idx].errno;
207}
208EXPORT_SYMBOL_GPL(blk_status_to_errno);
209
210const char *blk_status_to_str(blk_status_t status)
211{
212 int idx = (__force int)status;
213
214 if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
215 return "<null>";
216 return blk_errors[idx].name;
217}
218
219const char *blk_status_to_tag(blk_status_t status)
220{
221 int idx = (__force int)status;
222
223 if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors) || !blk_errors[idx].tag))
224 return "<null>";
225 return blk_errors[idx].tag;
226}
227
228blk_status_t tag_to_blk_status(const char *tag)
229{
230 int i;
231
232 for (i = 0; i < ARRAY_SIZE(blk_errors); i++) {
233 if (blk_errors[i].tag &&
234 !strcmp(blk_errors[i].tag, tag))
235 return (__force blk_status_t)i;
236 }
237
238 /*
239 * Return BLK_STS_OK for mismatches as this function is intended to
240 * parse error status values.
241 */
242 return BLK_STS_OK;
243}
244
245/**
246 * blk_sync_queue - cancel any pending callbacks on a queue
247 * @q: the queue
248 *
249 * Description:
250 * The block layer may perform asynchronous callback activity
251 * on a queue, such as calling the unplug function after a timeout.
252 * A block device may call blk_sync_queue to ensure that any
253 * such activity is cancelled, thus allowing it to release resources
254 * that the callbacks might use. The caller must already have made sure
255 * that its ->submit_bio will not re-add plugging prior to calling
256 * this function.
257 *
258 * This function does not cancel any asynchronous activity arising
259 * out of elevator or throttling code. That would require elevator_exit()
260 * and blkcg_exit_queue() to be called with queue lock initialized.
261 *
262 */
263void blk_sync_queue(struct request_queue *q)
264{
265 timer_delete_sync(&q->timeout);
266 cancel_work_sync(&q->timeout_work);
267}
268EXPORT_SYMBOL(blk_sync_queue);
269
270/**
271 * blk_set_pm_only - increment pm_only counter
272 * @q: request queue pointer
273 */
274void blk_set_pm_only(struct request_queue *q)
275{
276 atomic_inc(&q->pm_only);
277}
278EXPORT_SYMBOL_GPL(blk_set_pm_only);
279
280void blk_clear_pm_only(struct request_queue *q)
281{
282 int pm_only;
283
284 pm_only = atomic_dec_return(&q->pm_only);
285 WARN_ON_ONCE(pm_only < 0);
286 if (pm_only == 0)
287 wake_up_all(&q->mq_freeze_wq);
288}
289EXPORT_SYMBOL_GPL(blk_clear_pm_only);
290
291static void blk_free_queue_rcu(struct rcu_head *rcu_head)
292{
293 struct request_queue *q = container_of(rcu_head,
294 struct request_queue, rcu_head);
295
296 percpu_ref_exit(&q->q_usage_counter);
297 kmem_cache_free(blk_requestq_cachep, q);
298}
299
300static void blk_free_queue(struct request_queue *q)
301{
302 blk_free_queue_stats(q->stats);
303 if (queue_is_mq(q))
304 blk_mq_release(q);
305
306 ida_free(&blk_queue_ida, q->id);
307 lockdep_unregister_key(&q->io_lock_cls_key);
308 lockdep_unregister_key(&q->q_lock_cls_key);
309 call_rcu(&q->rcu_head, blk_free_queue_rcu);
310}
311
312/**
313 * blk_put_queue - decrement the request_queue refcount
314 * @q: the request_queue structure to decrement the refcount for
315 *
316 * Decrements the refcount of the request_queue and free it when the refcount
317 * reaches 0.
318 */
319void blk_put_queue(struct request_queue *q)
320{
321 if (refcount_dec_and_test(&q->refs))
322 blk_free_queue(q);
323}
324EXPORT_SYMBOL(blk_put_queue);
325
326bool blk_queue_start_drain(struct request_queue *q)
327{
328 /*
329 * When queue DYING flag is set, we need to block new req
330 * entering queue, so we call blk_freeze_queue_start() to
331 * prevent I/O from crossing blk_queue_enter().
332 */
333 bool freeze = __blk_freeze_queue_start(q, current);
334 if (queue_is_mq(q))
335 blk_mq_wake_waiters(q);
336 /* Make blk_queue_enter() reexamine the DYING flag. */
337 wake_up_all(&q->mq_freeze_wq);
338
339 return freeze;
340}
341
342/**
343 * blk_queue_enter() - try to increase q->q_usage_counter
344 * @q: request queue pointer
345 * @flags: BLK_MQ_REQ_NOWAIT and/or BLK_MQ_REQ_PM
346 */
347int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags)
348{
349 const bool pm = flags & BLK_MQ_REQ_PM;
350
351 while (!blk_try_enter_queue(q, pm)) {
352 if (flags & BLK_MQ_REQ_NOWAIT)
353 return -EAGAIN;
354
355 /*
356 * read pair of barrier in blk_freeze_queue_start(), we need to
357 * order reading __PERCPU_REF_DEAD flag of .q_usage_counter and
358 * reading .mq_freeze_depth or queue dying flag, otherwise the
359 * following wait may never return if the two reads are
360 * reordered.
361 */
362 smp_rmb();
363 wait_event(q->mq_freeze_wq,
364 (!q->mq_freeze_depth &&
365 blk_pm_resume_queue(pm, q)) ||
366 blk_queue_dying(q));
367 if (blk_queue_dying(q))
368 return -ENODEV;
369 }
370
371 rwsem_acquire_read(&q->q_lockdep_map, 0, 0, _RET_IP_);
372 rwsem_release(&q->q_lockdep_map, _RET_IP_);
373 return 0;
374}
375
376int __bio_queue_enter(struct request_queue *q, struct bio *bio)
377{
378 while (!blk_try_enter_queue(q, false)) {
379 struct gendisk *disk = bio->bi_bdev->bd_disk;
380
381 if (bio->bi_opf & REQ_NOWAIT) {
382 if (test_bit(GD_DEAD, &disk->state))
383 goto dead;
384 bio_wouldblock_error(bio);
385 return -EAGAIN;
386 }
387
388 /*
389 * read pair of barrier in blk_freeze_queue_start(), we need to
390 * order reading __PERCPU_REF_DEAD flag of .q_usage_counter and
391 * reading .mq_freeze_depth or queue dying flag, otherwise the
392 * following wait may never return if the two reads are
393 * reordered.
394 */
395 smp_rmb();
396 wait_event(q->mq_freeze_wq,
397 (!q->mq_freeze_depth &&
398 blk_pm_resume_queue(false, q)) ||
399 test_bit(GD_DEAD, &disk->state));
400 if (test_bit(GD_DEAD, &disk->state))
401 goto dead;
402 }
403
404 rwsem_acquire_read(&q->io_lockdep_map, 0, 0, _RET_IP_);
405 rwsem_release(&q->io_lockdep_map, _RET_IP_);
406 return 0;
407dead:
408 bio_io_error(bio);
409 return -ENODEV;
410}
411
412void blk_queue_exit(struct request_queue *q)
413{
414 percpu_ref_put(&q->q_usage_counter);
415}
416
417static void blk_queue_usage_counter_release(struct percpu_ref *ref)
418{
419 struct request_queue *q =
420 container_of(ref, struct request_queue, q_usage_counter);
421
422 wake_up_all(&q->mq_freeze_wq);
423}
424
425static void blk_rq_timed_out_timer(struct timer_list *t)
426{
427 struct request_queue *q = timer_container_of(q, t, timeout);
428
429 kblockd_schedule_work(&q->timeout_work);
430}
431
432static void blk_timeout_work(struct work_struct *work)
433{
434}
435
436struct request_queue *blk_alloc_queue(struct queue_limits *lim, int node_id)
437{
438 struct request_queue *q;
439 int error;
440
441 q = kmem_cache_alloc_node(blk_requestq_cachep, GFP_KERNEL | __GFP_ZERO,
442 node_id);
443 if (!q)
444 return ERR_PTR(-ENOMEM);
445
446 q->last_merge = NULL;
447
448 q->id = ida_alloc(&blk_queue_ida, GFP_KERNEL);
449 if (q->id < 0) {
450 error = q->id;
451 goto fail_q;
452 }
453
454 q->stats = blk_alloc_queue_stats();
455 if (!q->stats) {
456 error = -ENOMEM;
457 goto fail_id;
458 }
459
460 error = blk_set_default_limits(lim);
461 if (error)
462 goto fail_stats;
463 q->limits = *lim;
464
465 q->node = node_id;
466
467 atomic_set(&q->nr_active_requests_shared_tags, 0);
468
469 timer_setup(&q->timeout, blk_rq_timed_out_timer, 0);
470 INIT_WORK(&q->timeout_work, blk_timeout_work);
471 INIT_LIST_HEAD(&q->icq_list);
472
473 refcount_set(&q->refs, 1);
474 mutex_init(&q->debugfs_mutex);
475 mutex_init(&q->elevator_lock);
476 mutex_init(&q->sysfs_lock);
477 mutex_init(&q->limits_lock);
478 mutex_init(&q->rq_qos_mutex);
479 spin_lock_init(&q->queue_lock);
480
481 init_waitqueue_head(&q->mq_freeze_wq);
482 mutex_init(&q->mq_freeze_lock);
483
484 blkg_init_queue(q);
485
486 /*
487 * Init percpu_ref in atomic mode so that it's faster to shutdown.
488 * See blk_register_queue() for details.
489 */
490 error = percpu_ref_init(&q->q_usage_counter,
491 blk_queue_usage_counter_release,
492 PERCPU_REF_INIT_ATOMIC, GFP_KERNEL);
493 if (error)
494 goto fail_stats;
495 lockdep_register_key(&q->io_lock_cls_key);
496 lockdep_register_key(&q->q_lock_cls_key);
497 lockdep_init_map(&q->io_lockdep_map, "&q->q_usage_counter(io)",
498 &q->io_lock_cls_key, 0);
499 lockdep_init_map(&q->q_lockdep_map, "&q->q_usage_counter(queue)",
500 &q->q_lock_cls_key, 0);
501
502 /* Teach lockdep about lock ordering (reclaim WRT queue freeze lock). */
503 fs_reclaim_acquire(GFP_KERNEL);
504 rwsem_acquire_read(&q->io_lockdep_map, 0, 0, _RET_IP_);
505 rwsem_release(&q->io_lockdep_map, _RET_IP_);
506 fs_reclaim_release(GFP_KERNEL);
507
508 q->nr_requests = BLKDEV_DEFAULT_RQ;
509 q->async_depth = BLKDEV_DEFAULT_RQ;
510
511 return q;
512
513fail_stats:
514 blk_free_queue_stats(q->stats);
515fail_id:
516 ida_free(&blk_queue_ida, q->id);
517fail_q:
518 kmem_cache_free(blk_requestq_cachep, q);
519 return ERR_PTR(error);
520}
521
522/**
523 * blk_get_queue - increment the request_queue refcount
524 * @q: the request_queue structure to increment the refcount for
525 *
526 * Increment the refcount of the request_queue kobject.
527 *
528 * Context: Any context.
529 */
530bool blk_get_queue(struct request_queue *q)
531{
532 if (unlikely(blk_queue_dying(q)))
533 return false;
534 refcount_inc(&q->refs);
535 return true;
536}
537EXPORT_SYMBOL(blk_get_queue);
538
539#ifdef CONFIG_FAIL_MAKE_REQUEST
540
541static DECLARE_FAULT_ATTR(fail_make_request);
542
543static int __init setup_fail_make_request(char *str)
544{
545 return setup_fault_attr(&fail_make_request, str);
546}
547__setup("fail_make_request=", setup_fail_make_request);
548
549bool should_fail_request(struct block_device *part, unsigned int bytes)
550{
551 return bdev_test_flag(part, BD_MAKE_IT_FAIL) &&
552 should_fail(&fail_make_request, bytes);
553}
554
555static int __init fail_make_request_debugfs(void)
556{
557 struct dentry *dir = fault_create_debugfs_attr("fail_make_request",
558 NULL, &fail_make_request);
559
560 return PTR_ERR_OR_ZERO(dir);
561}
562
563late_initcall(fail_make_request_debugfs);
564#endif /* CONFIG_FAIL_MAKE_REQUEST */
565
566static inline void bio_check_ro(struct bio *bio)
567{
568 if (op_is_write(bio_op(bio)) && bdev_read_only(bio->bi_bdev)) {
569 if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
570 return;
571
572 if (bdev_test_flag(bio->bi_bdev, BD_RO_WARNED))
573 return;
574
575 bdev_set_flag(bio->bi_bdev, BD_RO_WARNED);
576
577 /*
578 * Use ioctl to set underlying disk of raid/dm to read-only
579 * will trigger this.
580 */
581 pr_warn("Trying to write to read-only block-device %pg\n",
582 bio->bi_bdev);
583 }
584}
585
586int should_fail_bio(struct bio *bio)
587{
588 if (should_fail_request(bdev_whole(bio->bi_bdev), bio->bi_iter.bi_size))
589 return -EIO;
590 return 0;
591}
592ALLOW_ERROR_INJECTION(should_fail_bio, ERRNO);
593
594/*
595 * Check whether this bio extends beyond the end of the device or partition.
596 * This may well happen - the kernel calls bread() without checking the size of
597 * the device, e.g., when mounting a file system.
598 */
599static inline int bio_check_eod(struct bio *bio)
600{
601 sector_t maxsector = bdev_nr_sectors(bio->bi_bdev);
602 unsigned int nr_sectors = bio_sectors(bio);
603
604 if (nr_sectors &&
605 (nr_sectors > maxsector ||
606 bio->bi_iter.bi_sector > maxsector - nr_sectors)) {
607 if (!maxsector)
608 return -EIO;
609 pr_info_ratelimited("%s: attempt to access beyond end of device\n"
610 "%pg: rw=%d, sector=%llu, nr_sectors = %u limit=%llu\n",
611 current->comm, bio->bi_bdev, bio->bi_opf,
612 bio->bi_iter.bi_sector, nr_sectors, maxsector);
613 return -EIO;
614 }
615 return 0;
616}
617
618/*
619 * Remap block n of partition p to block n+start(p) of the disk.
620 */
621static int blk_partition_remap(struct bio *bio)
622{
623 struct block_device *p = bio->bi_bdev;
624
625 if (unlikely(should_fail_request(p, bio->bi_iter.bi_size)))
626 return -EIO;
627 if (bio_sectors(bio)) {
628 bio->bi_iter.bi_sector += p->bd_start_sect;
629 trace_block_bio_remap(bio, p->bd_dev,
630 bio->bi_iter.bi_sector -
631 p->bd_start_sect);
632 }
633 bio_set_flag(bio, BIO_REMAPPED);
634 return 0;
635}
636
637/*
638 * Check write append to a zoned block device.
639 */
640static inline blk_status_t blk_check_zone_append(struct request_queue *q,
641 struct bio *bio)
642{
643 int nr_sectors = bio_sectors(bio);
644
645 /* Only applicable to zoned block devices */
646 if (!bdev_is_zoned(bio->bi_bdev))
647 return BLK_STS_NOTSUPP;
648
649 /* The bio sector must point to the start of a sequential zone */
650 if (!bdev_is_zone_start(bio->bi_bdev, bio->bi_iter.bi_sector))
651 return BLK_STS_IOERR;
652
653 /*
654 * Not allowed to cross zone boundaries. Otherwise, the BIO will be
655 * split and could result in non-contiguous sectors being written in
656 * different zones.
657 */
658 if (nr_sectors > q->limits.chunk_sectors)
659 return BLK_STS_IOERR;
660
661 /* Make sure the BIO is small enough and will not get split */
662 if (nr_sectors > q->limits.max_zone_append_sectors)
663 return BLK_STS_IOERR;
664
665 bio->bi_opf |= REQ_NOMERGE;
666
667 return BLK_STS_OK;
668}
669
670static void __submit_bio(struct bio *bio)
671{
672 if (!bdev_test_flag(bio->bi_bdev, BD_HAS_SUBMIT_BIO)) {
673 blk_mq_submit_bio(bio);
674 } else if (likely(bio_queue_enter(bio) == 0)) {
675 struct gendisk *disk = bio->bi_bdev->bd_disk;
676
677 if ((bio->bi_opf & REQ_POLLED) &&
678 !(disk->queue->limits.features & BLK_FEAT_POLL))
679 bio_endio_status(bio, BLK_STS_NOTSUPP);
680 else
681 disk->fops->submit_bio(bio);
682 blk_queue_exit(disk->queue);
683 }
684}
685
686/*
687 * The loop in this function may be a bit non-obvious, and so deserves some
688 * explanation:
689 *
690 * - Before entering the loop, bio->bi_next is NULL (as all callers ensure
691 * that), so we have a list with a single bio.
692 * - We pretend that we have just taken it off a longer list, so we assign
693 * bio_list to a pointer to the bio_list_on_stack, thus initialising the
694 * bio_list of new bios to be added. ->submit_bio() may indeed add some more
695 * bios through a recursive call to submit_bio_noacct. If it did, we find a
696 * non-NULL value in bio_list and re-enter the loop from the top.
697 * - In this case we really did just take the bio off the top of the list (no
698 * pretending) and so remove it from bio_list, and call into ->submit_bio()
699 * again.
700 *
701 * bio_list_on_stack[0] contains bios submitted by the current ->submit_bio.
702 * bio_list_on_stack[1] contains bios that were submitted before the current
703 * ->submit_bio(), but that haven't been processed yet.
704 */
705static void __submit_bio_noacct(struct bio *bio)
706{
707 struct bio_list bio_list_on_stack[2];
708
709 BUG_ON(bio->bi_next);
710
711 bio_list_init(&bio_list_on_stack[0]);
712 current->bio_list = bio_list_on_stack;
713
714 do {
715 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
716 struct bio_list lower, same;
717
718 /*
719 * Create a fresh bio_list for all subordinate requests.
720 */
721 bio_list_on_stack[1] = bio_list_on_stack[0];
722 bio_list_init(&bio_list_on_stack[0]);
723
724 __submit_bio(bio);
725
726 /*
727 * Sort new bios into those for a lower level and those for the
728 * same level.
729 */
730 bio_list_init(&lower);
731 bio_list_init(&same);
732 while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL)
733 if (q == bdev_get_queue(bio->bi_bdev))
734 bio_list_add(&same, bio);
735 else
736 bio_list_add(&lower, bio);
737
738 /*
739 * Now assemble so we handle the lowest level first.
740 */
741 bio_list_merge(&bio_list_on_stack[0], &lower);
742 bio_list_merge(&bio_list_on_stack[0], &same);
743 bio_list_merge(&bio_list_on_stack[0], &bio_list_on_stack[1]);
744 } while ((bio = bio_list_pop(&bio_list_on_stack[0])));
745
746 current->bio_list = NULL;
747}
748
749static void __submit_bio_noacct_mq(struct bio *bio)
750{
751 struct bio_list bio_list[2] = { };
752
753 current->bio_list = bio_list;
754
755 do {
756 __submit_bio(bio);
757 } while ((bio = bio_list_pop(&bio_list[0])));
758
759 current->bio_list = NULL;
760}
761
762void submit_bio_noacct_nocheck(struct bio *bio, bool split)
763{
764 if (unlikely(blk_error_inject(bio)))
765 return;
766
767 blk_cgroup_bio_start(bio);
768
769 if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) {
770 trace_block_bio_queue(bio);
771 /*
772 * Now that enqueuing has been traced, we need to trace
773 * completion as well.
774 */
775 bio_set_flag(bio, BIO_TRACE_COMPLETION);
776 }
777
778 /*
779 * We only want one ->submit_bio to be active at a time, else stack
780 * usage with stacked devices could be a problem. Use current->bio_list
781 * to collect a list of requests submitted by a ->submit_bio method
782 * while it is active, and then process them after it returned.
783 */
784 if (current->bio_list) {
785 if (split)
786 bio_list_add_head(&current->bio_list[0], bio);
787 else
788 bio_list_add(&current->bio_list[0], bio);
789 } else if (!bdev_test_flag(bio->bi_bdev, BD_HAS_SUBMIT_BIO)) {
790 __submit_bio_noacct_mq(bio);
791 } else {
792 __submit_bio_noacct(bio);
793 }
794}
795
796static blk_status_t blk_validate_atomic_write_op_size(struct request_queue *q,
797 struct bio *bio)
798{
799 if (bio->bi_iter.bi_size > queue_atomic_write_unit_max_bytes(q))
800 return BLK_STS_INVAL;
801
802 if (bio->bi_iter.bi_size % queue_atomic_write_unit_min_bytes(q))
803 return BLK_STS_INVAL;
804
805 return BLK_STS_OK;
806}
807
808/**
809 * submit_bio_noacct - re-submit a bio to the block device layer for I/O
810 * @bio: The bio describing the location in memory and on the device.
811 *
812 * This is a version of submit_bio() that shall only be used for I/O that is
813 * resubmitted to lower level drivers by stacking block drivers. All file
814 * systems and other upper level users of the block layer should use
815 * submit_bio() instead.
816 */
817void submit_bio_noacct(struct bio *bio)
818{
819 struct block_device *bdev = bio->bi_bdev;
820 struct request_queue *q = bdev_get_queue(bdev);
821 blk_status_t status = BLK_STS_IOERR;
822
823 might_sleep();
824
825 /*
826 * For a REQ_NOWAIT based request, return -EOPNOTSUPP
827 * if queue does not support NOWAIT.
828 */
829 if ((bio->bi_opf & REQ_NOWAIT) && !bdev_nowait(bdev))
830 goto not_supported;
831
832 if (bio_has_crypt_ctx(bio)) {
833 if (WARN_ON_ONCE(!bio_has_data(bio)))
834 goto end_io;
835 if (!blk_crypto_supported(bio))
836 goto not_supported;
837 }
838
839 if (should_fail_bio(bio))
840 goto end_io;
841 bio_check_ro(bio);
842 if (!bio_flagged(bio, BIO_REMAPPED)) {
843 if (unlikely(bio_check_eod(bio)))
844 goto end_io;
845 if (bdev_is_partition(bdev) &&
846 unlikely(blk_partition_remap(bio)))
847 goto end_io;
848 }
849
850 /*
851 * Filter flush bio's early so that bio based drivers without flush
852 * support don't have to worry about them.
853 */
854 if (op_is_flush(bio->bi_opf)) {
855 if (WARN_ON_ONCE(bio_op(bio) != REQ_OP_WRITE &&
856 bio_op(bio) != REQ_OP_ZONE_APPEND))
857 goto end_io;
858 if (!bdev_write_cache(bdev)) {
859 bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA);
860 if (!bio_sectors(bio)) {
861 status = BLK_STS_OK;
862 goto end_io;
863 }
864 }
865 }
866
867 switch (bio_op(bio)) {
868 case REQ_OP_READ:
869 break;
870 case REQ_OP_WRITE:
871 if (bio->bi_opf & REQ_ATOMIC) {
872 status = blk_validate_atomic_write_op_size(q, bio);
873 if (status != BLK_STS_OK)
874 goto end_io;
875 }
876 break;
877 case REQ_OP_FLUSH:
878 /*
879 * REQ_OP_FLUSH can't be submitted through bios, it is only
880 * synthetized in struct request by the flush state machine.
881 */
882 goto not_supported;
883 case REQ_OP_DISCARD:
884 if (!bdev_max_discard_sectors(bdev))
885 goto not_supported;
886 break;
887 case REQ_OP_SECURE_ERASE:
888 if (!bdev_max_secure_erase_sectors(bdev))
889 goto not_supported;
890 break;
891 case REQ_OP_ZONE_APPEND:
892 status = blk_check_zone_append(q, bio);
893 if (status != BLK_STS_OK)
894 goto end_io;
895 break;
896 case REQ_OP_WRITE_ZEROES:
897 if (!q->limits.max_write_zeroes_sectors)
898 goto not_supported;
899 break;
900 case REQ_OP_ZONE_OPEN:
901 case REQ_OP_ZONE_CLOSE:
902 case REQ_OP_ZONE_RESET:
903 case REQ_OP_ZONE_FINISH:
904 /* Zone management operations require sequential zones. */
905 if (!bdev_zone_is_seq(bio->bi_bdev, bio->bi_iter.bi_sector))
906 goto end_io;
907 break;
908 case REQ_OP_ZONE_RESET_ALL:
909 if (!bdev_is_zoned(bio->bi_bdev))
910 goto not_supported;
911 break;
912 case REQ_OP_DRV_IN:
913 case REQ_OP_DRV_OUT:
914 /*
915 * Driver private operations are only used with passthrough
916 * requests.
917 */
918 fallthrough;
919 default:
920 goto not_supported;
921 }
922
923 if (blk_throtl_bio(bio))
924 return;
925 submit_bio_noacct_nocheck(bio, false);
926 return;
927
928not_supported:
929 status = BLK_STS_NOTSUPP;
930end_io:
931 bio_endio_status(bio, status);
932}
933EXPORT_SYMBOL(submit_bio_noacct);
934
935static void bio_set_ioprio(struct bio *bio)
936{
937 /* Nobody set ioprio so far? Initialize it based on task's nice value */
938 if (IOPRIO_PRIO_CLASS(bio->bi_ioprio) == IOPRIO_CLASS_NONE)
939 bio->bi_ioprio = get_current_ioprio();
940 blkcg_set_ioprio(bio);
941}
942
943/**
944 * submit_bio - submit a bio to the block device layer for I/O
945 * @bio: The &struct bio which describes the I/O
946 *
947 * submit_bio() is used to submit I/O requests to block devices. It is passed a
948 * fully set up &struct bio that describes the I/O that needs to be done. The
949 * bio will be sent to the device described by the bi_bdev field.
950 *
951 * The success/failure status of the request, along with notification of
952 * completion, is delivered asynchronously through the ->bi_end_io() callback
953 * in @bio. The bio must NOT be touched by the caller until ->bi_end_io() has
954 * been called.
955 */
956void submit_bio(struct bio *bio)
957{
958 if (bio_op(bio) == REQ_OP_READ) {
959 task_io_account_read(bio->bi_iter.bi_size);
960 count_vm_events(PGPGIN, bio_sectors(bio));
961 } else if (bio_op(bio) == REQ_OP_WRITE) {
962 count_vm_events(PGPGOUT, bio_sectors(bio));
963 }
964
965 bio_set_ioprio(bio);
966 submit_bio_noacct(bio);
967}
968EXPORT_SYMBOL(submit_bio);
969
970/**
971 * bio_poll - poll for BIO completions
972 * @bio: bio to poll for
973 * @iob: batches of IO
974 * @flags: BLK_POLL_* flags that control the behavior
975 *
976 * Poll for completions on queue associated with the bio. Returns number of
977 * completed entries found.
978 *
979 * Note: the caller must either be the context that submitted @bio, or
980 * be in a RCU critical section to prevent freeing of @bio.
981 */
982int bio_poll(struct bio *bio, struct io_comp_batch *iob, unsigned int flags)
983{
984 blk_qc_t cookie = READ_ONCE(bio->bi_cookie);
985 struct block_device *bdev;
986 struct request_queue *q;
987 int ret = 0;
988
989 bdev = READ_ONCE(bio->bi_bdev);
990 if (!bdev)
991 return 0;
992
993 q = bdev_get_queue(bdev);
994 if (cookie == BLK_QC_T_NONE)
995 return 0;
996
997 blk_flush_plug(current->plug, false);
998
999 /*
1000 * We need to be able to enter a frozen queue, similar to how
1001 * timeouts also need to do that. If that is blocked, then we can
1002 * have pending IO when a queue freeze is started, and then the
1003 * wait for the freeze to finish will wait for polled requests to
1004 * timeout as the poller is preventer from entering the queue and
1005 * completing them. As long as we prevent new IO from being queued,
1006 * that should be all that matters.
1007 */
1008 if (!percpu_ref_tryget(&q->q_usage_counter))
1009 return 0;
1010 if (queue_is_mq(q)) {
1011 ret = blk_mq_poll(q, cookie, iob, flags);
1012 } else {
1013 struct gendisk *disk = q->disk;
1014
1015 if ((q->limits.features & BLK_FEAT_POLL) && disk &&
1016 disk->fops->poll_bio)
1017 ret = disk->fops->poll_bio(bio, iob, flags);
1018 }
1019 blk_queue_exit(q);
1020 return ret;
1021}
1022EXPORT_SYMBOL_GPL(bio_poll);
1023
1024/*
1025 * Helper to implement file_operations.iopoll. Requires the bio to be stored
1026 * in iocb->private, and cleared before freeing the bio.
1027 */
1028int iocb_bio_iopoll(struct kiocb *kiocb, struct io_comp_batch *iob,
1029 unsigned int flags)
1030{
1031 struct bio *bio;
1032 int ret = 0;
1033
1034 /*
1035 * Note: the bio cache only uses SLAB_TYPESAFE_BY_RCU, so bio can
1036 * point to a freshly allocated bio at this point. If that happens
1037 * we have a few cases to consider:
1038 *
1039 * 1) the bio is being initialized and bi_bdev is NULL. We can just
1040 * simply nothing in this case
1041 * 2) the bio points to a not poll enabled device. bio_poll will catch
1042 * this and return 0
1043 * 3) the bio points to a poll capable device, including but not
1044 * limited to the one that the original bio pointed to. In this
1045 * case we will call into the actual poll method and poll for I/O,
1046 * even if we don't need to, but it won't cause harm either.
1047 *
1048 * For cases 2) and 3) above the RCU grace period ensures that bi_bdev
1049 * is still allocated. Because partitions hold a reference to the whole
1050 * device bdev and thus disk, the disk is also still valid. Grabbing
1051 * a reference to the queue in bio_poll() ensures the hctxs and requests
1052 * are still valid as well.
1053 */
1054 rcu_read_lock();
1055 bio = READ_ONCE(kiocb->private);
1056 if (bio)
1057 ret = bio_poll(bio, iob, flags);
1058 rcu_read_unlock();
1059
1060 return ret;
1061}
1062EXPORT_SYMBOL_GPL(iocb_bio_iopoll);
1063
1064void update_io_ticks(struct block_device *part, unsigned long now, bool end)
1065{
1066 unsigned long stamp;
1067again:
1068 stamp = READ_ONCE(part->bd_stamp);
1069 if (unlikely(time_after(now, stamp)) &&
1070 likely(try_cmpxchg(&part->bd_stamp, &stamp, now)) &&
1071 (end || bdev_count_inflight(part)))
1072 __part_stat_add(part, io_ticks, now - stamp);
1073
1074 if (bdev_is_partition(part)) {
1075 part = bdev_whole(part);
1076 goto again;
1077 }
1078}
1079
1080unsigned long bdev_start_io_acct(struct block_device *bdev, enum req_op op,
1081 unsigned long start_time)
1082{
1083 part_stat_lock();
1084 update_io_ticks(bdev, start_time, false);
1085 bdev_inc_in_flight(bdev, op);
1086 part_stat_unlock();
1087
1088 return start_time;
1089}
1090EXPORT_SYMBOL(bdev_start_io_acct);
1091
1092/**
1093 * bio_start_io_acct - start I/O accounting for bio based drivers
1094 * @bio: bio to start account for
1095 *
1096 * Returns the start time that should be passed back to bio_end_io_acct().
1097 */
1098unsigned long bio_start_io_acct(struct bio *bio)
1099{
1100 return bdev_start_io_acct(bio->bi_bdev, bio_op(bio), jiffies);
1101}
1102EXPORT_SYMBOL_GPL(bio_start_io_acct);
1103
1104void bdev_end_io_acct(struct block_device *bdev, enum req_op op,
1105 unsigned int sectors, unsigned long start_time)
1106{
1107 const int sgrp = op_stat_group(op);
1108 unsigned long now = READ_ONCE(jiffies);
1109 unsigned long duration = now - start_time;
1110
1111 part_stat_lock();
1112 update_io_ticks(bdev, now, true);
1113 part_stat_inc(bdev, ios[sgrp]);
1114 part_stat_add(bdev, sectors[sgrp], sectors);
1115 part_stat_add(bdev, nsecs[sgrp], jiffies_to_nsecs(duration));
1116 bdev_dec_in_flight(bdev, op);
1117 part_stat_unlock();
1118}
1119EXPORT_SYMBOL(bdev_end_io_acct);
1120
1121void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time,
1122 struct block_device *orig_bdev)
1123{
1124 bdev_end_io_acct(orig_bdev, bio_op(bio), bio_sectors(bio), start_time);
1125}
1126EXPORT_SYMBOL_GPL(bio_end_io_acct_remapped);
1127
1128/**
1129 * blk_lld_busy - Check if underlying low-level drivers of a device are busy
1130 * @q : the queue of the device being checked
1131 *
1132 * Description:
1133 * Check if underlying low-level drivers of a device are busy.
1134 * If the drivers want to export their busy state, they must set own
1135 * exporting function using blk_queue_lld_busy() first.
1136 *
1137 * Basically, this function is used only by request stacking drivers
1138 * to stop dispatching requests to underlying devices when underlying
1139 * devices are busy. This behavior helps more I/O merging on the queue
1140 * of the request stacking driver and prevents I/O throughput regression
1141 * on burst I/O load.
1142 *
1143 * Return:
1144 * 0 - Not busy (The request stacking driver should dispatch request)
1145 * 1 - Busy (The request stacking driver should stop dispatching request)
1146 */
1147int blk_lld_busy(struct request_queue *q)
1148{
1149 if (queue_is_mq(q) && q->mq_ops->busy)
1150 return q->mq_ops->busy(q);
1151
1152 return 0;
1153}
1154EXPORT_SYMBOL_GPL(blk_lld_busy);
1155
1156int kblockd_schedule_work(struct work_struct *work)
1157{
1158 return queue_work(kblockd_workqueue, work);
1159}
1160EXPORT_SYMBOL(kblockd_schedule_work);
1161
1162int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork,
1163 unsigned long delay)
1164{
1165 return mod_delayed_work_on(cpu, kblockd_workqueue, dwork, delay);
1166}
1167EXPORT_SYMBOL(kblockd_mod_delayed_work_on);
1168
1169void blk_start_plug_nr_ios(struct blk_plug *plug, unsigned short nr_ios)
1170{
1171 struct task_struct *tsk = current;
1172
1173 /*
1174 * If this is a nested plug, don't actually assign it.
1175 */
1176 if (tsk->plug)
1177 return;
1178
1179 plug->cur_ktime = 0;
1180 rq_list_init(&plug->mq_list);
1181 rq_list_init(&plug->cached_rqs);
1182 plug->nr_ios = min_t(unsigned short, nr_ios, BLK_MAX_REQUEST_COUNT);
1183 plug->rq_count = 0;
1184 plug->multiple_queues = false;
1185 plug->has_elevator = false;
1186 INIT_LIST_HEAD(&plug->cb_list);
1187
1188 /*
1189 * Store ordering should not be needed here, since a potential
1190 * preempt will imply a full memory barrier
1191 */
1192 tsk->plug = plug;
1193}
1194
1195/**
1196 * blk_start_plug - initialize blk_plug and track it inside the task_struct
1197 * @plug: The &struct blk_plug that needs to be initialized
1198 *
1199 * Description:
1200 * blk_start_plug() indicates to the block layer an intent by the caller
1201 * to submit multiple I/O requests in a batch. The block layer may use
1202 * this hint to defer submitting I/Os from the caller until blk_finish_plug()
1203 * is called. However, the block layer may choose to submit requests
1204 * before a call to blk_finish_plug() if the number of queued I/Os
1205 * exceeds %BLK_MAX_REQUEST_COUNT, or if the size of the I/O is larger than
1206 * %BLK_PLUG_FLUSH_SIZE. The queued I/Os may also be submitted early if
1207 * the task schedules (see below).
1208 *
1209 * Tracking blk_plug inside the task_struct will help with auto-flushing the
1210 * pending I/O should the task end up blocking between blk_start_plug() and
1211 * blk_finish_plug(). This is important from a performance perspective, but
1212 * also ensures that we don't deadlock. For instance, if the task is blocking
1213 * for a memory allocation, memory reclaim could end up wanting to free a
1214 * page belonging to that request that is currently residing in our private
1215 * plug. By flushing the pending I/O when the process goes to sleep, we avoid
1216 * this kind of deadlock.
1217 */
1218void blk_start_plug(struct blk_plug *plug)
1219{
1220 blk_start_plug_nr_ios(plug, 1);
1221}
1222EXPORT_SYMBOL(blk_start_plug);
1223
1224static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
1225{
1226 LIST_HEAD(callbacks);
1227
1228 while (!list_empty(&plug->cb_list)) {
1229 list_splice_init(&plug->cb_list, &callbacks);
1230
1231 while (!list_empty(&callbacks)) {
1232 struct blk_plug_cb *cb = list_first_entry(&callbacks,
1233 struct blk_plug_cb,
1234 list);
1235 list_del(&cb->list);
1236 cb->callback(cb, from_schedule);
1237 }
1238 }
1239}
1240
1241struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
1242 int size)
1243{
1244 struct blk_plug *plug = current->plug;
1245 struct blk_plug_cb *cb;
1246
1247 if (!plug)
1248 return NULL;
1249
1250 list_for_each_entry(cb, &plug->cb_list, list)
1251 if (cb->callback == unplug && cb->data == data)
1252 return cb;
1253
1254 /* Not currently on the callback list */
1255 BUG_ON(size < sizeof(*cb));
1256 cb = kzalloc(size, GFP_ATOMIC);
1257 if (cb) {
1258 cb->data = data;
1259 cb->callback = unplug;
1260 list_add(&cb->list, &plug->cb_list);
1261 }
1262 return cb;
1263}
1264EXPORT_SYMBOL(blk_check_plugged);
1265
1266void __blk_flush_plug(struct blk_plug *plug, bool from_schedule)
1267{
1268 if (!list_empty(&plug->cb_list))
1269 flush_plug_callbacks(plug, from_schedule);
1270 blk_mq_flush_plug_list(plug, from_schedule);
1271 /*
1272 * Unconditionally flush out cached requests, even if the unplug
1273 * event came from schedule. Since we know hold references to the
1274 * queue for cached requests, we don't want a blocked task holding
1275 * up a queue freeze/quiesce event.
1276 */
1277 if (unlikely(!rq_list_empty(&plug->cached_rqs)))
1278 blk_mq_free_plug_rqs(plug);
1279
1280 plug->cur_ktime = 0;
1281 current->flags &= ~PF_BLOCK_TS;
1282}
1283
1284/**
1285 * blk_finish_plug - mark the end of a batch of submitted I/O
1286 * @plug: The &struct blk_plug passed to blk_start_plug()
1287 *
1288 * Description:
1289 * Indicate that a batch of I/O submissions is complete. This function
1290 * must be paired with an initial call to blk_start_plug(). The intent
1291 * is to allow the block layer to optimize I/O submission. See the
1292 * documentation for blk_start_plug() for more information.
1293 */
1294void blk_finish_plug(struct blk_plug *plug)
1295{
1296 if (plug == current->plug) {
1297 __blk_flush_plug(plug, false);
1298 current->plug = NULL;
1299 }
1300}
1301EXPORT_SYMBOL(blk_finish_plug);
1302
1303void blk_io_schedule(void)
1304{
1305 /* Prevent hang_check timer from firing at us during very long I/O */
1306 unsigned long timeout = sysctl_hung_task_timeout_secs * HZ / 2;
1307
1308 if (timeout)
1309 io_schedule_timeout(timeout);
1310 else
1311 io_schedule();
1312}
1313
1314int __init blk_dev_init(void)
1315{
1316 BUILD_BUG_ON((__force u32)REQ_OP_LAST >= (1 << REQ_OP_BITS));
1317 BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
1318 sizeof_field(struct request, cmd_flags));
1319 BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
1320 sizeof_field(struct bio, bi_opf));
1321
1322 /* used for unplugging and affects IO latency/throughput - HIGHPRI */
1323 kblockd_workqueue = alloc_workqueue("kblockd",
1324 WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU, 0);
1325 if (!kblockd_workqueue)
1326 panic("Failed to create kblockd\n");
1327
1328 blk_requestq_cachep = KMEM_CACHE(request_queue, SLAB_PANIC);
1329
1330 blk_debugfs_root = debugfs_create_dir("block", NULL);
1331
1332 return 0;
1333}