projects/linux·blob master · refs

2143 lines · 57.2 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2001 Jens Axboe <[email protected]>
4 */
5#include <linux/mm.h>
6#include <linux/swap.h>
7#include <linux/bio-integrity.h>
8#include <linux/blkdev.h>
9#include <linux/uio.h>
10#include <linux/iocontext.h>
11#include <linux/slab.h>
12#include <linux/init.h>
13#include <linux/kernel.h>
14#include <linux/export.h>
15#include <linux/mempool.h>
16#include <linux/workqueue.h>
17#include <linux/cgroup.h>
18#include <linux/highmem.h>
19#include <linux/blk-crypto.h>
20#include <linux/xarray.h>
21#include <linux/kmemleak.h>
22
23#include <trace/events/block.h>
24#include "blk.h"
25#include "blk-rq-qos.h"
26#include "blk-cgroup.h"
27
28#define ALLOC_CACHE_THRESHOLD 16
29#define ALLOC_CACHE_MAX 256
30
31struct bio_alloc_cache {
32 struct bio *free_list;
33 struct bio *free_list_irq;
34 unsigned int nr;
35 unsigned int nr_irq;
36};
37
38#define BIO_INLINE_VECS 4
39
40static struct biovec_slab {
41 int nr_vecs;
42 char *name;
43 struct kmem_cache *slab;
44} bvec_slabs[] __read_mostly = {
45 { .nr_vecs = 16, .name = "biovec-16" },
46 { .nr_vecs = 64, .name = "biovec-64" },
47 { .nr_vecs = 128, .name = "biovec-128" },
48 { .nr_vecs = BIO_MAX_VECS, .name = "biovec-max" },
49};
50
51static struct biovec_slab *biovec_slab(unsigned short nr_vecs)
52{
53 switch (nr_vecs) {
54 /* smaller bios use inline vecs */
55 case 5 ... 16:
56 return &bvec_slabs[0];
57 case 17 ... 64:
58 return &bvec_slabs[1];
59 case 65 ... 128:
60 return &bvec_slabs[2];
61 case 129 ... BIO_MAX_VECS:
62 return &bvec_slabs[3];
63 default:
64 BUG();
65 return NULL;
66 }
67}
68
69/*
70 * fs_bio_set is the bio_set containing bio and iovec memory pools used by
71 * IO code that does not need private memory pools.
72 */
73struct bio_set fs_bio_set;
74EXPORT_SYMBOL(fs_bio_set);
75
76/*
77 * Our slab pool management
78 */
79struct bio_slab {
80 struct kmem_cache *slab;
81 unsigned int slab_ref;
82 unsigned int slab_size;
83 char name[12];
84};
85static DEFINE_MUTEX(bio_slab_lock);
86static DEFINE_XARRAY(bio_slabs);
87
88static struct bio_slab *create_bio_slab(unsigned int size)
89{
90 struct bio_slab *bslab = kzalloc_obj(*bslab);
91
92 if (!bslab)
93 return NULL;
94
95 snprintf(bslab->name, sizeof(bslab->name), "bio-%d", size);
96 bslab->slab = kmem_cache_create(bslab->name, size,
97 ARCH_KMALLOC_MINALIGN,
98 SLAB_HWCACHE_ALIGN | SLAB_TYPESAFE_BY_RCU, NULL);
99 if (!bslab->slab)
100 goto fail_alloc_slab;
101
102 bslab->slab_ref = 1;
103 bslab->slab_size = size;
104
105 if (!xa_err(xa_store(&bio_slabs, size, bslab, GFP_KERNEL)))
106 return bslab;
107
108 kmem_cache_destroy(bslab->slab);
109
110fail_alloc_slab:
111 kfree(bslab);
112 return NULL;
113}
114
115static inline unsigned int bs_bio_slab_size(struct bio_set *bs)
116{
117 return bs->front_pad + sizeof(struct bio) + bs->back_pad;
118}
119
120static inline void *bio_slab_addr(struct bio *bio)
121{
122 return (void *)bio - bio->bi_pool->front_pad;
123}
124
125static struct kmem_cache *bio_find_or_create_slab(struct bio_set *bs)
126{
127 unsigned int size = bs_bio_slab_size(bs);
128 struct bio_slab *bslab;
129
130 mutex_lock(&bio_slab_lock);
131 bslab = xa_load(&bio_slabs, size);
132 if (bslab)
133 bslab->slab_ref++;
134 else
135 bslab = create_bio_slab(size);
136 mutex_unlock(&bio_slab_lock);
137
138 if (bslab)
139 return bslab->slab;
140 return NULL;
141}
142
143static void bio_put_slab(struct bio_set *bs)
144{
145 struct bio_slab *bslab = NULL;
146 unsigned int slab_size = bs_bio_slab_size(bs);
147
148 mutex_lock(&bio_slab_lock);
149
150 bslab = xa_load(&bio_slabs, slab_size);
151 if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n"))
152 goto out;
153
154 WARN_ON_ONCE(bslab->slab != bs->bio_slab);
155
156 WARN_ON(!bslab->slab_ref);
157
158 if (--bslab->slab_ref)
159 goto out;
160
161 xa_erase(&bio_slabs, slab_size);
162
163 kmem_cache_destroy(bslab->slab);
164 kfree(bslab);
165
166out:
167 mutex_unlock(&bio_slab_lock);
168}
169
170/*
171 * Make the first allocation restricted and don't dump info on allocation
172 * failures, since we'll fall back to the mempool in case of failure.
173 */
174static inline gfp_t try_alloc_gfp(gfp_t gfp)
175{
176 return (gfp & ~(__GFP_DIRECT_RECLAIM | __GFP_IO)) |
177 __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
178}
179
180void bio_uninit(struct bio *bio)
181{
182#ifdef CONFIG_BLK_CGROUP
183 if (bio->bi_blkg) {
184 blkg_put(bio->bi_blkg);
185 bio->bi_blkg = NULL;
186 }
187#endif
188 if (bio_integrity(bio))
189 bio_integrity_free(bio);
190
191 bio_crypt_free_ctx(bio);
192}
193EXPORT_SYMBOL(bio_uninit);
194
195static void bio_free(struct bio *bio)
196{
197 struct bio_set *bs = bio->bi_pool;
198 void *p = bio;
199
200 WARN_ON_ONCE(!bs);
201 WARN_ON_ONCE(bio->bi_max_vecs > BIO_MAX_VECS);
202
203 bio_uninit(bio);
204 if (bio->bi_max_vecs == BIO_MAX_VECS)
205 mempool_free(bio->bi_io_vec, &bs->bvec_pool);
206 else if (bio->bi_max_vecs > BIO_INLINE_VECS)
207 kmem_cache_free(biovec_slab(bio->bi_max_vecs)->slab,
208 bio->bi_io_vec);
209 mempool_free(p - bs->front_pad, &bs->bio_pool);
210}
211
212/*
213 * Users of this function have their own bio allocation. Subsequently,
214 * they must remember to pair any call to bio_init() with bio_uninit()
215 * when IO has completed, or when the bio is released.
216 */
217void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table,
218 unsigned short max_vecs, blk_opf_t opf)
219{
220 bio->bi_next = NULL;
221 bio->bi_bdev = bdev;
222 bio->bi_opf = opf;
223 bio->bi_flags = 0;
224 bio->bi_ioprio = 0;
225 bio->bi_write_hint = 0;
226 bio->bi_write_stream = 0;
227 bio->bi_status = 0;
228 bio->bi_bvec_gap_bit = 0;
229 bio->bi_iter.bi_sector = 0;
230 bio->bi_iter.bi_size = 0;
231 bio->bi_iter.bi_idx = 0;
232 bio->bi_iter.bi_offset = 0;
233 bio->bi_end_io = NULL;
234 bio->bi_private = NULL;
235#ifdef CONFIG_BLK_CGROUP
236 bio->bi_blkg = NULL;
237 bio->issue_time_ns = 0;
238 if (bdev)
239 bio_associate_blkg(bio);
240#ifdef CONFIG_BLK_CGROUP_IOCOST
241 bio->bi_iocost_cost = 0;
242#endif
243#endif
244#ifdef CONFIG_BLK_INLINE_ENCRYPTION
245 bio->bi_crypt_context = NULL;
246#endif
247#ifdef CONFIG_BLK_DEV_INTEGRITY
248 bio->bi_integrity = NULL;
249#endif
250 bio->bi_vcnt = 0;
251
252 atomic_set(&bio->__bi_remaining, 1);
253 atomic_set(&bio->__bi_cnt, 1);
254 bio->bi_cookie = BLK_QC_T_NONE;
255
256 bio->bi_max_vecs = max_vecs;
257 bio->bi_io_vec = table;
258 bio->bi_pool = NULL;
259}
260EXPORT_SYMBOL(bio_init);
261
262/**
263 * bio_reset - reinitialize a bio
264 * @bio: bio to reset
265 * @bdev: block device to use the bio for
266 * @opf: operation and flags for bio
267 *
268 * Description:
269 * After calling bio_reset(), @bio will be in the same state as a freshly
270 * allocated bio returned bio bio_alloc_bioset() - the only fields that are
271 * preserved are the ones that are initialized by bio_alloc_bioset(). See
272 * comment in struct bio.
273 */
274void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf)
275{
276 struct bio_vec *bv = bio->bi_io_vec;
277
278 bio_uninit(bio);
279 memset(bio, 0, BIO_RESET_BYTES);
280 atomic_set(&bio->__bi_remaining, 1);
281 bio->bi_io_vec = bv;
282 bio->bi_bdev = bdev;
283 if (bio->bi_bdev)
284 bio_associate_blkg(bio);
285 bio->bi_opf = opf;
286}
287EXPORT_SYMBOL(bio_reset);
288
289/**
290 * bio_reuse - reuse a bio with the payload left intact
291 * @bio: bio to reuse
292 * @opf: operation and flags for the next I/O
293 *
294 * Allow reusing an existing bio for another operation with all set up
295 * fields including the payload, device and end_io handler left intact.
296 *
297 * Typically used when @bio is first used to read data which is then written
298 * to another location without modification. @bio must not be in-flight and
299 * owned by the caller. Can't be used for cloned bios.
300 *
301 * Note: Can't be used when @bio has integrity or blk-crypto contexts for now.
302 * Feel free to add that support when you need it, though.
303 */
304void bio_reuse(struct bio *bio, blk_opf_t opf)
305{
306 unsigned short vcnt = bio->bi_vcnt, i;
307 bio_end_io_t *end_io = bio->bi_end_io;
308 void *private = bio->bi_private;
309
310 WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
311 WARN_ON_ONCE(bio_integrity(bio));
312 WARN_ON_ONCE(bio_has_crypt_ctx(bio));
313
314 bio_reset(bio, bio->bi_bdev, opf);
315 for (i = 0; i < vcnt; i++)
316 bio->bi_iter.bi_size += bio->bi_io_vec[i].bv_len;
317 bio->bi_vcnt = vcnt;
318 bio->bi_private = private;
319 bio->bi_end_io = end_io;
320}
321EXPORT_SYMBOL_GPL(bio_reuse);
322
323/**
324 * bio_prepare_reissue - prepare a bio for reuissing the original I/O
325 * @bio: bio to reuse
326 * @bdev: block device to use the bio for
327 *
328 * Prepare @bio to be resubmitted to retry the original operation.
329 * The caller must reset bio->bi_iter to the original state.
330 */
331void bio_prepare_reissue(struct bio *bio, struct block_device *bdev)
332{
333 bio->bi_bdev = bdev;
334 bio_associate_blkg(bio);
335 bio->bi_flags &=
336 (BIO_PAGE_PINNED | BIO_CLONED | BIO_QUIET | BIO_REFFED);
337 bio->bi_status = BLK_STS_OK;
338 bio->bi_bvec_gap_bit = 0;
339 atomic_set(&bio->__bi_remaining, 1);
340}
341EXPORT_SYMBOL_GPL(bio_prepare_reissue);
342
343static struct bio *__bio_chain_endio(struct bio *bio)
344{
345 struct bio *parent = bio->bi_private;
346
347 if (bio->bi_status && !parent->bi_status)
348 parent->bi_status = bio->bi_status;
349 bio_put(bio);
350 return parent;
351}
352
353/*
354 * This function should only be used as a flag and must never be called.
355 * If execution reaches here, it indicates a serious programming error.
356 */
357static void bio_chain_endio(struct bio *bio)
358{
359 BUG();
360}
361
362/**
363 * bio_chain - chain bio completions
364 * @bio: the target bio
365 * @parent: the parent bio of @bio
366 *
367 * The caller won't have a bi_end_io called when @bio completes - instead,
368 * @parent's bi_end_io won't be called until both @parent and @bio have
369 * completed; the chained bio will also be freed when it completes.
370 *
371 * The caller must not set bi_private or bi_end_io in @bio.
372 */
373void bio_chain(struct bio *bio, struct bio *parent)
374{
375 BUG_ON(bio->bi_private || bio->bi_end_io);
376
377 bio->bi_private = parent;
378 bio->bi_end_io = bio_chain_endio;
379 bio_inc_remaining(parent);
380}
381EXPORT_SYMBOL(bio_chain);
382
383/**
384 * bio_chain_and_submit - submit a bio after chaining it to another one
385 * @prev: bio to chain and submit
386 * @new: bio to chain to
387 *
388 * If @prev is non-NULL, chain it to @new and submit it.
389 *
390 * Return: @new.
391 */
392struct bio *bio_chain_and_submit(struct bio *prev, struct bio *new)
393{
394 if (prev) {
395 bio_chain(prev, new);
396 submit_bio(prev);
397 }
398 return new;
399}
400
401struct bio *blk_next_bio(struct bio *bio, struct block_device *bdev,
402 unsigned int nr_pages, blk_opf_t opf, gfp_t gfp)
403{
404 return bio_chain_and_submit(bio, bio_alloc(bdev, nr_pages, opf, gfp));
405}
406EXPORT_SYMBOL_GPL(blk_next_bio);
407
408static void bio_alloc_rescue(struct work_struct *work)
409{
410 struct bio_set *bs = container_of(work, struct bio_set, rescue_work);
411 struct bio *bio;
412
413 while (1) {
414 spin_lock(&bs->rescue_lock);
415 bio = bio_list_pop(&bs->rescue_list);
416 spin_unlock(&bs->rescue_lock);
417
418 if (!bio)
419 break;
420
421 submit_bio_noacct(bio);
422 }
423}
424
425/*
426 * submit_bio_noacct() converts recursion to iteration; this means if we're
427 * running beneath it, any bios we allocate and submit will not be submitted
428 * (and thus freed) until after we return.
429 *
430 * This exposes us to a potential deadlock if we allocate multiple bios from the
431 * same bio_set while running underneath submit_bio_noacct(). If we were to
432 * allocate multiple bios (say a stacking block driver that was splitting bios),
433 * we would deadlock if we exhausted the mempool's reserve.
434 *
435 * We solve this, and guarantee forward progress by punting the bios on
436 * current->bio_list to a per bio_set rescuer workqueue before blocking to wait
437 * for elements being returned to the mempool.
438 */
439static void punt_bios_to_rescuer(struct bio_set *bs)
440{
441 struct bio_list punt, nopunt;
442 struct bio *bio;
443
444 if (!current->bio_list || !bs->rescue_workqueue)
445 return;
446 if (bio_list_empty(&current->bio_list[0]) &&
447 bio_list_empty(&current->bio_list[1]))
448 return;
449
450 /*
451 * In order to guarantee forward progress we must punt only bios that
452 * were allocated from this bio_set; otherwise, if there was a bio on
453 * there for a stacking driver higher up in the stack, processing it
454 * could require allocating bios from this bio_set, and doing that from
455 * our own rescuer would be bad.
456 *
457 * Since bio lists are singly linked, pop them all instead of trying to
458 * remove from the middle of the list:
459 */
460
461 bio_list_init(&punt);
462 bio_list_init(&nopunt);
463
464 while ((bio = bio_list_pop(&current->bio_list[0])))
465 bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
466 current->bio_list[0] = nopunt;
467
468 bio_list_init(&nopunt);
469 while ((bio = bio_list_pop(&current->bio_list[1])))
470 bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
471 current->bio_list[1] = nopunt;
472
473 spin_lock(&bs->rescue_lock);
474 bio_list_merge(&bs->rescue_list, &punt);
475 spin_unlock(&bs->rescue_lock);
476
477 queue_work(bs->rescue_workqueue, &bs->rescue_work);
478}
479
480static void bio_alloc_irq_cache_splice(struct bio_alloc_cache *cache)
481{
482 unsigned long flags;
483
484 /* cache->free_list must be empty */
485 if (WARN_ON_ONCE(cache->free_list))
486 return;
487
488 local_irq_save(flags);
489 cache->free_list = cache->free_list_irq;
490 cache->free_list_irq = NULL;
491 cache->nr += cache->nr_irq;
492 cache->nr_irq = 0;
493 local_irq_restore(flags);
494}
495
496static struct bio *bio_alloc_percpu_cache(struct bio_set *bs)
497{
498 struct bio_alloc_cache *cache;
499 struct bio *bio;
500
501 cache = per_cpu_ptr(bs->cache, get_cpu());
502 if (!cache->free_list) {
503 if (READ_ONCE(cache->nr_irq) >= ALLOC_CACHE_THRESHOLD)
504 bio_alloc_irq_cache_splice(cache);
505 if (!cache->free_list) {
506 put_cpu();
507 return NULL;
508 }
509 }
510 bio = cache->free_list;
511 cache->free_list = bio->bi_next;
512 cache->nr--;
513 put_cpu();
514 bio->bi_pool = bs;
515
516 kmemleak_alloc(bio_slab_addr(bio),
517 kmem_cache_size(bs->bio_slab), 1, GFP_NOIO);
518 return bio;
519}
520
521/**
522 * bio_alloc_bioset - allocate a bio for I/O
523 * @bdev: block device to allocate the bio for (can be %NULL)
524 * @nr_vecs: number of bvecs to pre-allocate
525 * @opf: operation and flags for bio
526 * @gfp: the GFP_* mask given to the slab allocator
527 * @bs: the bio_set to allocate from.
528 *
529 * Allocate a bio from the mempools in @bs.
530 *
531 * If %__GFP_DIRECT_RECLAIM is set then bio_alloc will always be able to
532 * allocate a bio. This is due to the mempool guarantees. To make this work,
533 * callers must never allocate more than 1 bio at a time from the general pool.
534 * Callers that need to allocate more than 1 bio must always submit the
535 * previously allocated bio for IO before attempting to allocate a new one.
536 * Failure to do so can cause deadlocks under memory pressure.
537 *
538 * Note that when running under submit_bio_noacct() (i.e. any block driver),
539 * bios are not submitted until after you return - see the code in
540 * submit_bio_noacct() that converts recursion into iteration, to prevent
541 * stack overflows.
542 *
543 * This would normally mean allocating multiple bios under submit_bio_noacct()
544 * would be susceptible to deadlocks, but we have
545 * deadlock avoidance code that resubmits any blocked bios from a rescuer
546 * thread.
547 *
548 * However, we do not guarantee forward progress for allocations from other
549 * mempools. Doing multiple allocations from the same mempool under
550 * submit_bio_noacct() should be avoided - instead, use bio_set's front_pad
551 * for per bio allocations.
552 *
553 * Returns: Pointer to new bio on success, NULL on failure.
554 */
555struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
556 blk_opf_t opf, gfp_t gfp, struct bio_set *bs)
557{
558 struct bio_vec *bvecs = NULL;
559 struct bio *bio = NULL;
560 gfp_t saved_gfp = gfp;
561 void *p;
562
563 /* should not use nobvec bioset for nr_vecs > 0 */
564 if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && nr_vecs > 0))
565 return NULL;
566
567 if (saved_gfp & __GFP_DIRECT_RECLAIM)
568 gfp = try_alloc_gfp(gfp);
569 if (bs->cache && nr_vecs <= BIO_INLINE_VECS) {
570 /*
571 * Set REQ_ALLOC_CACHE even if no cached bio is available to
572 * return the allocated bio to the percpu cache when done.
573 */
574 opf |= REQ_ALLOC_CACHE;
575 bio = bio_alloc_percpu_cache(bs);
576 } else {
577 opf &= ~REQ_ALLOC_CACHE;
578 }
579
580 /*
581 * For a bioset without a percpu cache, or when the percpu cache was
582 * empty, try a slab allocation with optimistic GFP_ flags before
583 * falling back to the mempool.
584 */
585 if (!bio) {
586 p = kmem_cache_alloc(bs->bio_slab, gfp);
587 if (p)
588 bio = p + bs->front_pad;
589 }
590
591 if (bio && nr_vecs > BIO_INLINE_VECS) {
592 struct biovec_slab *bvs = biovec_slab(nr_vecs);
593
594 /*
595 * Upgrade nr_vecs to take full advantage of the allocation.
596 * We also rely on this in bio_free().
597 */
598 nr_vecs = bvs->nr_vecs;
599 bvecs = kmem_cache_alloc(bvs->slab, gfp);
600 if (unlikely(!bvecs)) {
601 kmem_cache_free(bs->bio_slab, p);
602 bio = NULL;
603 }
604 }
605
606 if (unlikely(!bio)) {
607 /*
608 * Give up if we are not allow to sleep as non-blocking mempool
609 * allocations just go back to the slab allocation.
610 */
611 if (!(saved_gfp & __GFP_DIRECT_RECLAIM))
612 return NULL;
613
614 punt_bios_to_rescuer(bs);
615
616 /*
617 * Don't rob the mempools by returning to the per-CPU cache if
618 * we're tight on memory.
619 */
620 opf &= ~REQ_ALLOC_CACHE;
621
622 p = mempool_alloc(&bs->bio_pool, saved_gfp);
623 bio = p + bs->front_pad;
624 if (nr_vecs > BIO_INLINE_VECS) {
625 nr_vecs = BIO_MAX_VECS;
626 bvecs = mempool_alloc(&bs->bvec_pool, saved_gfp);
627 }
628 }
629
630 if (nr_vecs && nr_vecs <= BIO_INLINE_VECS)
631 bio_init_inline(bio, bdev, nr_vecs, opf);
632 else
633 bio_init(bio, bdev, bvecs, nr_vecs, opf);
634 bio->bi_pool = bs;
635 return bio;
636}
637EXPORT_SYMBOL(bio_alloc_bioset);
638
639/**
640 * bio_kmalloc - kmalloc a bio
641 * @nr_vecs: number of bio_vecs to allocate
642 * @gfp_mask: the GFP_* mask given to the slab allocator
643 *
644 * Use kmalloc to allocate a bio (including bvecs). The bio must be initialized
645 * using bio_init() before use. To free a bio returned from this function use
646 * kfree() after calling bio_uninit(). A bio returned from this function can
647 * be reused by calling bio_uninit() before calling bio_init() again.
648 *
649 * Note that unlike bio_alloc() or bio_alloc_bioset() allocations from this
650 * function are not backed by a mempool can fail. Do not use this function
651 * for allocations in the file system I/O path.
652 *
653 * Returns: Pointer to new bio on success, NULL on failure.
654 */
655struct bio *bio_kmalloc(unsigned short nr_vecs, gfp_t gfp_mask)
656{
657 struct bio *bio;
658
659 if (nr_vecs > BIO_MAX_INLINE_VECS)
660 return NULL;
661 return kmalloc(sizeof(*bio) + nr_vecs * sizeof(struct bio_vec),
662 gfp_mask);
663}
664EXPORT_SYMBOL(bio_kmalloc);
665
666void zero_fill_bio(struct bio *bio)
667{
668 struct bio_vec bv;
669 struct bvec_iter iter;
670
671 bio_for_each_segment(bv, bio, iter)
672 memzero_bvec(&bv);
673}
674EXPORT_SYMBOL(zero_fill_bio);
675
676/**
677 * bio_truncate - truncate the bio to small size of @new_size
678 * @bio: the bio to be truncated
679 * @new_size: new size for truncating the bio
680 *
681 * Description:
682 * Truncate the bio to new size of @new_size. If bio_op(bio) is
683 * REQ_OP_READ, zero the truncated part. This function should only
684 * be used for handling corner cases, such as bio eod.
685 */
686static void bio_truncate(struct bio *bio, unsigned new_size)
687{
688 struct bio_vec bv;
689 struct bvec_iter iter;
690 unsigned int done = 0;
691 bool truncated = false;
692
693 if (new_size >= bio->bi_iter.bi_size)
694 return;
695
696 if (bio_op(bio) != REQ_OP_READ)
697 goto exit;
698
699 bio_for_each_segment(bv, bio, iter) {
700 if (done + bv.bv_len > new_size) {
701 size_t offset;
702
703 if (!truncated)
704 offset = new_size - done;
705 else
706 offset = 0;
707 memzero_page(bv.bv_page, bv.bv_offset + offset,
708 bv.bv_len - offset);
709 truncated = true;
710 }
711 done += bv.bv_len;
712 }
713
714 exit:
715 /*
716 * Don't touch bvec table here and make it really immutable, since
717 * fs bio user has to retrieve all pages via bio_for_each_segment_all
718 * in its .end_bio() callback.
719 *
720 * It is enough to truncate bio by updating .bi_size since we can make
721 * correct bvec with the updated .bi_size for drivers.
722 */
723 bio->bi_iter.bi_size = new_size;
724}
725
726/**
727 * guard_bio_eod - truncate a BIO to fit the block device
728 * @bio: bio to truncate
729 *
730 * This allows us to do IO even on the odd last sectors of a device, even if the
731 * block size is some multiple of the physical sector size.
732 *
733 * We'll just truncate the bio to the size of the device, and clear the end of
734 * the buffer head manually. Truly out-of-range accesses will turn into actual
735 * I/O errors, this only handles the "we need to be able to do I/O at the final
736 * sector" case.
737 */
738void guard_bio_eod(struct bio *bio)
739{
740 sector_t maxsector = bdev_nr_sectors(bio->bi_bdev);
741
742 if (!maxsector)
743 return;
744
745 /*
746 * If the *whole* IO is past the end of the device,
747 * let it through, and the IO layer will turn it into
748 * an EIO.
749 */
750 if (unlikely(bio->bi_iter.bi_sector >= maxsector))
751 return;
752
753 maxsector -= bio->bi_iter.bi_sector;
754 if (likely((bio->bi_iter.bi_size >> 9) <= maxsector))
755 return;
756
757 bio_truncate(bio, maxsector << 9);
758}
759
760static int __bio_alloc_cache_prune(struct bio_alloc_cache *cache,
761 unsigned int nr)
762{
763 unsigned int i = 0;
764 struct bio *bio;
765
766 while ((bio = cache->free_list) != NULL) {
767 cache->free_list = bio->bi_next;
768 cache->nr--;
769 kmemleak_alloc(bio_slab_addr(bio),
770 kmem_cache_size(bio->bi_pool->bio_slab),
771 1, GFP_KERNEL);
772 bio_free(bio);
773 if (++i == nr)
774 break;
775 }
776 return i;
777}
778
779static void bio_alloc_cache_prune(struct bio_alloc_cache *cache,
780 unsigned int nr)
781{
782 nr -= __bio_alloc_cache_prune(cache, nr);
783 if (!READ_ONCE(cache->free_list)) {
784 bio_alloc_irq_cache_splice(cache);
785 __bio_alloc_cache_prune(cache, nr);
786 }
787}
788
789static int bio_cpu_dead(unsigned int cpu, struct hlist_node *node)
790{
791 struct bio_set *bs;
792
793 bs = hlist_entry_safe(node, struct bio_set, cpuhp_dead);
794 if (bs->cache) {
795 struct bio_alloc_cache *cache = per_cpu_ptr(bs->cache, cpu);
796
797 bio_alloc_cache_prune(cache, -1U);
798 }
799 return 0;
800}
801
802static void bio_alloc_cache_destroy(struct bio_set *bs)
803{
804 int cpu;
805
806 if (!bs->cache)
807 return;
808
809 cpuhp_state_remove_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead);
810 for_each_possible_cpu(cpu) {
811 struct bio_alloc_cache *cache;
812
813 cache = per_cpu_ptr(bs->cache, cpu);
814 bio_alloc_cache_prune(cache, -1U);
815 }
816 free_percpu(bs->cache);
817 bs->cache = NULL;
818}
819
820static inline void bio_put_percpu_cache(struct bio *bio)
821{
822 struct bio_alloc_cache *cache;
823
824 cache = per_cpu_ptr(bio->bi_pool->cache, get_cpu());
825 if (READ_ONCE(cache->nr_irq) + cache->nr > ALLOC_CACHE_MAX)
826 goto out_free;
827
828 if (in_task()) {
829 bio_uninit(bio);
830 bio->bi_next = cache->free_list;
831 /* Not necessary but helps not to iopoll already freed bios */
832 bio->bi_bdev = NULL;
833 cache->free_list = bio;
834 cache->nr++;
835 kmemleak_free(bio_slab_addr(bio));
836 } else if (in_hardirq()) {
837 lockdep_assert_irqs_disabled();
838
839 bio_uninit(bio);
840 bio->bi_next = cache->free_list_irq;
841 cache->free_list_irq = bio;
842 cache->nr_irq++;
843 kmemleak_free(bio_slab_addr(bio));
844 } else {
845 goto out_free;
846 }
847 put_cpu();
848 return;
849out_free:
850 put_cpu();
851 bio_free(bio);
852}
853
854/**
855 * bio_put - release a reference to a bio
856 * @bio: bio to release reference to
857 *
858 * Description:
859 * Put a reference to a &struct bio, either one you have gotten with
860 * bio_alloc, bio_get or bio_clone_*. The last put of a bio will free it.
861 **/
862void bio_put(struct bio *bio)
863{
864 if (unlikely(bio_flagged(bio, BIO_REFFED))) {
865 BUG_ON(!atomic_read(&bio->__bi_cnt));
866 if (!atomic_dec_and_test(&bio->__bi_cnt))
867 return;
868 }
869 if (bio->bi_opf & REQ_ALLOC_CACHE)
870 bio_put_percpu_cache(bio);
871 else
872 bio_free(bio);
873}
874EXPORT_SYMBOL(bio_put);
875
876static int __bio_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp)
877{
878 bio_set_flag(bio, BIO_CLONED);
879 bio->bi_ioprio = bio_src->bi_ioprio;
880 bio->bi_write_hint = bio_src->bi_write_hint;
881 bio->bi_write_stream = bio_src->bi_write_stream;
882 bio->bi_bvec_gap_bit = bio_src->bi_bvec_gap_bit;
883 bio->bi_iter = bio_src->bi_iter;
884 bio->bi_io_vec = bio_src->bi_io_vec;
885
886 if (bio->bi_bdev) {
887 if (bio->bi_bdev == bio_src->bi_bdev &&
888 bio_flagged(bio_src, BIO_REMAPPED))
889 bio_set_flag(bio, BIO_REMAPPED);
890 bio_clone_blkg_association(bio, bio_src);
891 }
892
893 if (bio_crypt_clone(bio, bio_src, gfp) < 0)
894 return -ENOMEM;
895 if (bio_integrity(bio_src) &&
896 bio_integrity_clone(bio, bio_src, gfp) < 0)
897 return -ENOMEM;
898 return 0;
899}
900
901/**
902 * bio_alloc_clone - clone a bio that shares the original bio's biovec
903 * @bdev: block_device to clone onto
904 * @bio_src: bio to clone from
905 * @gfp: allocation priority
906 * @bs: bio_set to allocate from
907 *
908 * Allocate a new bio that is a clone of @bio_src. This reuses the bio_vecs
909 * pointed to by @bio_src->bi_io_vec, and clones the iterator pointing to
910 * the current position in it. The caller owns the returned bio, but not
911 * the bio_vecs, and must ensure the bio is freed before the memory
912 * pointed to by @bio_Src->bi_io_vecs.
913 */
914struct bio *bio_alloc_clone(struct block_device *bdev, struct bio *bio_src,
915 gfp_t gfp, struct bio_set *bs)
916{
917 struct bio *bio;
918
919 bio = bio_alloc_bioset(bdev, 0, bio_src->bi_opf, gfp, bs);
920 if (!bio)
921 return NULL;
922
923 if (__bio_clone(bio, bio_src, gfp) < 0) {
924 bio_put(bio);
925 return NULL;
926 }
927 return bio;
928}
929EXPORT_SYMBOL(bio_alloc_clone);
930
931/**
932 * bio_init_clone - clone a bio that shares the original bio's biovec
933 * @bdev: block_device to clone onto
934 * @bio: bio to clone into
935 * @bio_src: bio to clone from
936 * @gfp: allocation priority
937 *
938 * Initialize a new bio in caller provided memory that is a clone of @bio_src.
939 * The same bio_vecs reuse and bio lifetime rules as bio_alloc_clone() apply.
940 */
941int bio_init_clone(struct block_device *bdev, struct bio *bio,
942 struct bio *bio_src, gfp_t gfp)
943{
944 int ret;
945
946 bio_init(bio, bdev, NULL, 0, bio_src->bi_opf);
947 ret = __bio_clone(bio, bio_src, gfp);
948 if (ret)
949 bio_uninit(bio);
950 return ret;
951}
952EXPORT_SYMBOL(bio_init_clone);
953
954/**
955 * bio_full - check if the bio is full
956 * @bio: bio to check
957 * @len: length of one segment to be added
958 *
959 * Return true if @bio is full and one segment with @len bytes can't be
960 * added to the bio, otherwise return false
961 */
962static inline bool bio_full(struct bio *bio, unsigned len)
963{
964 if (bio->bi_vcnt >= bio->bi_max_vecs)
965 return true;
966 if (bio->bi_iter.bi_size > BIO_MAX_SIZE - len)
967 return true;
968 return false;
969}
970
971static bool bvec_try_merge_page(struct bio_vec *bv, struct page *page,
972 unsigned int len, unsigned int off)
973{
974 size_t bv_end = bv->bv_offset + bv->bv_len;
975 phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) + bv_end - 1;
976 phys_addr_t page_addr = page_to_phys(page);
977
978 if (vec_end_addr + 1 != page_addr + off)
979 return false;
980 if (xen_domain() && !xen_biovec_phys_mergeable(bv, page))
981 return false;
982
983 if ((vec_end_addr & PAGE_MASK) != ((page_addr + off) & PAGE_MASK)) {
984 if (IS_ENABLED(CONFIG_KMSAN))
985 return false;
986 if (bv->bv_page + bv_end / PAGE_SIZE != page + off / PAGE_SIZE)
987 return false;
988 }
989
990 bv->bv_len += len;
991 return true;
992}
993
994/*
995 * Try to merge a page into a segment, while obeying the hardware segment
996 * size limit.
997 *
998 * This is kept around for the integrity metadata, which is still tries
999 * to build the initial bio to the hardware limit and doesn't have proper
1000 * helpers to split. Hopefully this will go away soon.
1001 */
1002bool bvec_try_merge_hw_page(struct request_queue *q, struct bio_vec *bv,
1003 struct page *page, unsigned len, unsigned offset)
1004{
1005 unsigned long mask = queue_segment_boundary(q);
1006 phys_addr_t addr1 = bvec_phys(bv);
1007 phys_addr_t addr2 = page_to_phys(page) + offset + len - 1;
1008
1009 if ((addr1 | mask) != (addr2 | mask))
1010 return false;
1011 if (len > queue_max_segment_size(q) - bv->bv_len)
1012 return false;
1013 return bvec_try_merge_page(bv, page, len, offset);
1014}
1015
1016/**
1017 * __bio_add_page - add page(s) to a bio in a new segment
1018 * @bio: destination bio
1019 * @page: start page to add
1020 * @len: length of the data to add, may cross pages
1021 * @off: offset of the data relative to @page, may cross pages
1022 *
1023 * Add the data at @page + @off to @bio as a new bvec. The caller must ensure
1024 * that @bio has space for another bvec.
1025 */
1026void __bio_add_page(struct bio *bio, struct page *page,
1027 unsigned int len, unsigned int off)
1028{
1029 WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
1030 WARN_ON_ONCE(bio_full(bio, len));
1031
1032 if (is_pci_p2pdma_page(page))
1033 bio->bi_opf |= REQ_NOMERGE;
1034
1035 bvec_set_page(&bio->bi_io_vec[bio->bi_vcnt], page, len, off);
1036 bio->bi_iter.bi_size += len;
1037 bio->bi_vcnt++;
1038}
1039EXPORT_SYMBOL_GPL(__bio_add_page);
1040
1041/**
1042 * bio_add_virt_nofail - add data in the direct kernel mapping to a bio
1043 * @bio: destination bio
1044 * @vaddr: data to add
1045 * @len: length of the data to add, may cross pages
1046 *
1047 * Add the data at @vaddr to @bio. The caller must have ensure a segment
1048 * is available for the added data. No merging into an existing segment
1049 * will be performed.
1050 */
1051void bio_add_virt_nofail(struct bio *bio, void *vaddr, unsigned len)
1052{
1053 __bio_add_page(bio, virt_to_page(vaddr), len, offset_in_page(vaddr));
1054}
1055EXPORT_SYMBOL_GPL(bio_add_virt_nofail);
1056
1057/**
1058 * bio_add_page - attempt to add page(s) to bio
1059 * @bio: destination bio
1060 * @page: start page to add
1061 * @len: vec entry length, may cross pages
1062 * @offset: vec entry offset relative to @page, may cross pages
1063 *
1064 * Attempt to add page(s) to the bio_vec maplist. This will only fail
1065 * if either bio->bi_vcnt == bio->bi_max_vecs or it's a cloned bio.
1066 */
1067int bio_add_page(struct bio *bio, struct page *page,
1068 unsigned int len, unsigned int offset)
1069{
1070 if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
1071 return 0;
1072 if (WARN_ON_ONCE(len == 0))
1073 return 0;
1074 if (bio->bi_iter.bi_size > BIO_MAX_SIZE - len)
1075 return 0;
1076
1077 if (bio->bi_vcnt > 0) {
1078 struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
1079
1080 if (!zone_device_pages_compatible(bv->bv_page, page))
1081 return 0;
1082 if (zone_device_pages_have_same_pgmap(bv->bv_page, page) &&
1083 bvec_try_merge_page(bv, page, len, offset)) {
1084 bio->bi_iter.bi_size += len;
1085 return len;
1086 }
1087 }
1088
1089 if (bio->bi_vcnt >= bio->bi_max_vecs)
1090 return 0;
1091 __bio_add_page(bio, page, len, offset);
1092 return len;
1093}
1094EXPORT_SYMBOL(bio_add_page);
1095
1096void bio_add_folio_nofail(struct bio *bio, struct folio *folio, size_t len,
1097 size_t off)
1098{
1099 unsigned long nr = off / PAGE_SIZE;
1100
1101 WARN_ON_ONCE(len > BIO_MAX_SIZE);
1102 __bio_add_page(bio, folio_page(folio, nr), len, off % PAGE_SIZE);
1103}
1104EXPORT_SYMBOL_GPL(bio_add_folio_nofail);
1105
1106/**
1107 * bio_add_folio - Attempt to add part of a folio to a bio.
1108 * @bio: BIO to add to.
1109 * @folio: Folio to add.
1110 * @len: How many bytes from the folio to add.
1111 * @off: First byte in this folio to add.
1112 *
1113 * Filesystems that use folios can call this function instead of calling
1114 * bio_add_page() for each page in the folio. If @off is bigger than
1115 * PAGE_SIZE, this function can create a bio_vec that starts in a page
1116 * after the bv_page. BIOs do not support folios that are 4GiB or larger.
1117 *
1118 * Return: Whether the addition was successful.
1119 */
1120bool bio_add_folio(struct bio *bio, struct folio *folio, size_t len,
1121 size_t off)
1122{
1123 unsigned long nr = off / PAGE_SIZE;
1124
1125 if (len > BIO_MAX_SIZE)
1126 return false;
1127 return bio_add_page(bio, folio_page(folio, nr), len, off % PAGE_SIZE) > 0;
1128}
1129EXPORT_SYMBOL(bio_add_folio);
1130
1131/**
1132 * bio_add_vmalloc_chunk - add a vmalloc chunk to a bio
1133 * @bio: destination bio
1134 * @vaddr: vmalloc address to add
1135 * @len: total length in bytes of the data to add
1136 *
1137 * Add data starting at @vaddr to @bio and return how many bytes were added.
1138 * This may be less than the amount originally asked. Returns 0 if no data
1139 * could be added to @bio.
1140 *
1141 * This helper calls flush_kernel_vmap_range() for the range added. For reads
1142 * the caller still needs to manually call invalidate_kernel_vmap_range() in
1143 * the completion handler.
1144 */
1145unsigned int bio_add_vmalloc_chunk(struct bio *bio, void *vaddr, unsigned len)
1146{
1147 unsigned int offset = offset_in_page(vaddr);
1148
1149 len = min(len, PAGE_SIZE - offset);
1150 if (bio_add_page(bio, vmalloc_to_page(vaddr), len, offset) < len)
1151 return 0;
1152 if (op_is_write(bio_op(bio)))
1153 flush_kernel_vmap_range(vaddr, len);
1154 return len;
1155}
1156EXPORT_SYMBOL_GPL(bio_add_vmalloc_chunk);
1157
1158/**
1159 * bio_add_vmalloc - add a vmalloc region to a bio
1160 * @bio: destination bio
1161 * @vaddr: vmalloc address to add
1162 * @len: total length in bytes of the data to add
1163 *
1164 * Add data starting at @vaddr to @bio. Return %true on success or %false if
1165 * @bio does not have enough space for the payload.
1166 *
1167 * This helper calls flush_kernel_vmap_range() for the range added. For reads
1168 * the caller still needs to manually call invalidate_kernel_vmap_range() in
1169 * the completion handler.
1170 */
1171bool bio_add_vmalloc(struct bio *bio, void *vaddr, unsigned int len)
1172{
1173 do {
1174 unsigned int added = bio_add_vmalloc_chunk(bio, vaddr, len);
1175
1176 if (!added)
1177 return false;
1178 vaddr += added;
1179 len -= added;
1180 } while (len);
1181
1182 return true;
1183}
1184EXPORT_SYMBOL_GPL(bio_add_vmalloc);
1185
1186void __bio_release_pages(struct bio *bio, bool mark_dirty)
1187{
1188 struct folio_iter fi;
1189
1190 bio_for_each_folio_all(fi, bio) {
1191 size_t nr_pages;
1192
1193 if (mark_dirty) {
1194 folio_lock(fi.folio);
1195 folio_mark_dirty(fi.folio);
1196 folio_unlock(fi.folio);
1197 }
1198 nr_pages = (fi.offset + fi.length - 1) / PAGE_SIZE -
1199 fi.offset / PAGE_SIZE + 1;
1200 unpin_user_folio(fi.folio, nr_pages);
1201 }
1202}
1203EXPORT_SYMBOL_GPL(__bio_release_pages);
1204
1205bool bio_iov_iter_set(struct bio *bio, const struct iov_iter *iter)
1206{
1207 if (!iov_iter_is_bvec(iter))
1208 return false;
1209
1210 WARN_ON_ONCE(bio->bi_max_vecs);
1211
1212 bio->bi_io_vec = (struct bio_vec *)iter->bvec;
1213 bio->bi_iter.bi_idx = 0;
1214 bio->bi_iter.bi_offset = iter->iov_offset;
1215 bio->bi_iter.bi_size = iov_iter_count(iter);
1216 bio_set_flag(bio, BIO_CLONED);
1217 return true;
1218}
1219
1220/*
1221 * Aligns the bio size to the len_align_mask, releasing excessive bio vecs that
1222 * __bio_iov_iter_get_pages may have inserted, and reverts the trimmed length
1223 * for the next iteration.
1224 */
1225static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter,
1226 unsigned len_align_mask)
1227{
1228 struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
1229 size_t nbytes = bio->bi_iter.bi_size & len_align_mask;
1230
1231 if (!nbytes)
1232 return 0;
1233
1234 iov_iter_revert(iter, nbytes);
1235 bio->bi_iter.bi_size -= nbytes;
1236 while (nbytes >= bv->bv_len) {
1237 if (bio_flagged(bio, BIO_PAGE_PINNED))
1238 unpin_user_page(bv->bv_page);
1239
1240 if (!--bio->bi_vcnt)
1241 return -EFAULT;
1242 nbytes -= bv->bv_len;
1243 bv--;
1244 }
1245 bv->bv_len -= nbytes;
1246 return 0;
1247}
1248
1249#ifdef CONFIG_DEBUG_KERNEL
1250static inline bool bio_iov_bvec_aligned(const struct bio *bio,
1251 unsigned mem_align_mask)
1252{
1253 struct bvec_iter iter;
1254 struct bio_vec bv;
1255
1256 /*
1257 * Correct callers never break the alignment requirements, so this
1258 * exhaustive check is only paid for in debug builds.
1259 */
1260 for_each_mp_bvec(bv, bio->bi_io_vec, iter, bio->bi_iter)
1261 if ((bv.bv_offset | bv.bv_len) & mem_align_mask)
1262 return false;
1263 return true;
1264}
1265#else
1266static inline bool bio_iov_bvec_aligned(const struct bio *bio,
1267 unsigned mem_align_mask)
1268{
1269 /*
1270 * We forward the bio_vec as-is, so ITER_BVEC callers must provide
1271 * segments already aligned to the device's DMA alignment. The only
1272 * unchecked user-controllable offset that reaches here is an io_uring
1273 * registered buffer where just the first segment can be unaligned
1274 * (the rest is virtually contiguous), so checking only that one is
1275 * sufficient to know if the entire vector is valid.
1276 */
1277 return !(mp_bvec_iter_offset(bio->bi_io_vec, bio->bi_iter) &
1278 mem_align_mask);
1279}
1280#endif
1281
1282/**
1283 * bio_iov_iter_get_pages - add user or kernel pages to a bio
1284 * @bio: bio to add pages to
1285 * @iter: iov iterator describing the region to be added
1286 * @maxlen: maximum size to consume from @iter
1287 * @mem_align_mask: the mask the source address and length must be aligned to,
1288 * 0 for no requirement
1289 * @len_align_mask: the mask to align the total size to, 0 for any length
1290 *
1291 * This takes either an iterator pointing to user memory, or one pointing to
1292 * kernel pages (BVEC iterator). If we're adding user pages, we pin them and
1293 * map them into the kernel. On IO completion, the caller should put those
1294 * pages. For bvec based iterators bio_iov_iter_get_pages() uses the provided
1295 * bvecs rather than copying them. Hence anyone issuing kiocb based IO needs
1296 * to ensure the bvecs and pages stay referenced until the submitted I/O is
1297 * completed by a call to ->ki_complete() or returns with an error other than
1298 * -EIOCBQUEUED. The caller needs to check if the bio is flagged BIO_NO_PAGE_REF
1299 * on IO completion. If it isn't, then pages should be released.
1300 *
1301 * The function tries, but does not guarantee, to pin as many pages as
1302 * fit into the bio, or are requested in @iter, whatever is smaller. If
1303 * MM encounters an error pinning the requested pages, it stops. Error
1304 * is returned only if 0 pages could be pinned.
1305 */
1306int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
1307 unsigned maxlen, unsigned mem_align_mask,
1308 unsigned len_align_mask)
1309{
1310 iov_iter_extraction_t flags = 0;
1311
1312 if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
1313 return -EIO;
1314
1315 if (bio_iov_iter_set(bio, iter)) {
1316 if (iov_iter_is_bvec(iter) &&
1317 !bio_iov_bvec_aligned(bio, mem_align_mask))
1318 return -EINVAL;
1319
1320 /* Truncate to the maximum size that the caller can handle */
1321 bio->bi_iter.bi_size = min(bio->bi_iter.bi_size, maxlen);
1322 iov_iter_advance(iter, bio->bi_iter.bi_size);
1323 return 0;
1324 }
1325
1326 if (iov_iter_extract_will_pin(iter))
1327 bio_set_flag(bio, BIO_PAGE_PINNED);
1328 if (bio->bi_bdev && blk_queue_pci_p2pdma(bio->bi_bdev->bd_disk->queue))
1329 flags |= ITER_ALLOW_P2PDMA;
1330
1331 do {
1332 ssize_t ret;
1333
1334 ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec,
1335 maxlen - bio->bi_iter.bi_size,
1336 &bio->bi_vcnt, bio->bi_max_vecs,
1337 mem_align_mask, flags);
1338 if (ret <= 0) {
1339 /*
1340 * A misaligned vector fails the whole I/O. Release any
1341 * pages pinned by earlier iterations before returning
1342 * since this bio won't be submitted to release them.
1343 */
1344 if (ret == -EINVAL) {
1345 bio_release_pages(bio, false);
1346 bio_clear_flag(bio, BIO_PAGE_PINNED);
1347 bio->bi_vcnt = 0;
1348 }
1349 if (!bio->bi_vcnt)
1350 return ret;
1351 break;
1352 }
1353 bio->bi_iter.bi_size += ret;
1354 } while (iov_iter_count(iter) && !bio_full(bio, 0));
1355
1356 if (is_pci_p2pdma_page(bio->bi_io_vec->bv_page))
1357 bio->bi_opf |= REQ_NOMERGE;
1358 return bio_iov_iter_align_down(bio, iter, len_align_mask);
1359}
1360
1361static struct folio *folio_alloc_greedy(gfp_t gfp, size_t *size,
1362 size_t minsize)
1363{
1364 struct folio *folio;
1365
1366 while (*size > minsize) {
1367 folio = folio_alloc(gfp | __GFP_NORETRY | __GFP_NOWARN,
1368 get_order(*size));
1369 if (folio)
1370 return folio;
1371 *size = rounddown_pow_of_two(*size - 1);
1372 }
1373
1374 return folio_alloc(gfp, get_order(*size));
1375}
1376
1377void bio_free_folios(struct bio *bio)
1378{
1379 struct bio_vec *bv;
1380 int i;
1381
1382 bio_for_each_bvec_all(bv, bio, i) {
1383 struct folio *folio = bvec_folio(bv);
1384
1385 if (!is_zero_folio(folio) && !is_huge_zero_folio(folio))
1386 folio_put(folio);
1387 }
1388}
1389
1390int bio_alloc_bounce_folios(struct bio *bio, size_t total_len, size_t minsize)
1391{
1392 if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
1393 return -EINVAL;
1394 if (WARN_ON_ONCE(bio->bi_iter.bi_size))
1395 return -EINVAL;
1396 if (WARN_ON_ONCE(bio->bi_vcnt >= bio->bi_max_vecs))
1397 return -EINVAL;
1398
1399 do {
1400 size_t this_len = min(total_len, SZ_1M);
1401 struct folio *folio;
1402
1403 if (this_len > minsize * 2)
1404 this_len = rounddown_pow_of_two(this_len);
1405
1406 if (bio->bi_iter.bi_size > BIO_MAX_SIZE - this_len)
1407 break;
1408
1409 folio = folio_alloc_greedy(GFP_KERNEL, &this_len, minsize);
1410 if (!folio)
1411 break;
1412
1413 /*
1414 * Align down the size to the minimum alignment. In practice
1415 * this should not happen as minsize is expected to be a power
1416 * of two, as is the allocation size, but it offers us a cheap
1417 * extra safety belt.
1418 */
1419 this_len &= ~(minsize - 1);
1420 bio_add_folio_nofail(bio, folio, this_len, 0);
1421 total_len -= this_len;
1422 } while (total_len && bio->bi_vcnt < bio->bi_max_vecs);
1423
1424 if (!bio->bi_iter.bi_size)
1425 return -ENOMEM;
1426 return 0;
1427}
1428
1429/**
1430 * bio_iov_iter_bounce_write - bounce buffer data from an iter into a bio
1431 * @bio: bio to send
1432 * @iter: iter to read from
1433 * @maxlen: maximum size to bounce
1434 * @minsize: minimum folio allocation size
1435 *
1436 * Helper for direct I/O write implementations that need to bounce buffer
1437 * because they need need to checksum the data or perform other operations that
1438 * require consistency. Allocates folios to back the bounce buffer, and copies
1439 * the data into it. Needs to be paired with bio_free_folios() called on
1440 * completion.
1441 */
1442int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter,
1443 size_t maxlen, size_t minsize)
1444{
1445 size_t total_len = min(maxlen, iov_iter_count(iter));
1446 size_t total_copied = 0;
1447 struct bio_vec *bv;
1448 int i, error;
1449
1450 error = bio_alloc_bounce_folios(bio, total_len, minsize);
1451 if (error)
1452 return error;
1453
1454 bio_for_each_bvec_all(bv, bio, i) {
1455 struct folio *folio = page_folio(bv->bv_page);
1456 size_t copied;
1457
1458 if (iter->nofault)
1459 copied = copy_folio_from_iter_atomic(folio, 0,
1460 bv->bv_len, iter);
1461 else
1462 copied = copy_folio_from_iter(folio, 0, bv->bv_len,
1463 iter);
1464 total_copied += copied;
1465 if (copied < bv->bv_len) {
1466 iov_iter_revert(iter, total_copied);
1467 bio_free_folios(bio);
1468 return -EFAULT;
1469 }
1470 }
1471
1472 return 0;
1473}
1474EXPORT_SYMBOL_GPL(bio_iov_iter_bounce_write);
1475
1476static void bio_wait_end_io(struct bio *bio)
1477{
1478 complete(bio->bi_private);
1479}
1480
1481/**
1482 * bio_await - call a function on a bio, and wait until it completes
1483 * @bio: the bio which describes the I/O
1484 * @submit: function called to submit the bio
1485 * @priv: private data passed to @submit
1486 *
1487 * Wait for the bio as well as any bio chained off it after executing the
1488 * passed in callback @submit. The wait for the bio is set up before calling
1489 * @submit to ensure that the completion is captured. If @submit is %NULL,
1490 * submit_bio() is used instead to submit the bio.
1491 *
1492 * Note: this overrides the bi_private and bi_end_io fields in the bio.
1493 */
1494void bio_await(struct bio *bio, void *priv,
1495 void (*submit)(struct bio *bio, void *priv))
1496{
1497 DECLARE_COMPLETION_ONSTACK_MAP(done,
1498 bio->bi_bdev->bd_disk->lockdep_map);
1499
1500 bio->bi_private = &done;
1501 bio->bi_end_io = bio_wait_end_io;
1502 bio->bi_opf |= REQ_SYNC;
1503 if (submit)
1504 submit(bio, priv);
1505 else
1506 submit_bio(bio);
1507 blk_wait_io(&done);
1508}
1509EXPORT_SYMBOL_GPL(bio_await);
1510
1511/**
1512 * submit_bio_wait - submit a bio, and wait until it completes
1513 * @bio: The &struct bio which describes the I/O
1514 *
1515 * Simple wrapper around submit_bio(). Returns 0 on success, or the error from
1516 * bio_endio() on failure.
1517 *
1518 * WARNING: Unlike to how submit_bio() is usually used, this function does not
1519 * result in bio reference to be consumed. The caller must drop the reference
1520 * on his own.
1521 */
1522int submit_bio_wait(struct bio *bio)
1523{
1524 bio_await(bio, NULL, NULL);
1525 return blk_status_to_errno(bio->bi_status);
1526}
1527EXPORT_SYMBOL(submit_bio_wait);
1528
1529static void bio_endio_cb(struct bio *bio, void *priv)
1530{
1531 bio_endio(bio);
1532}
1533
1534/*
1535 * Submit @bio synchronously, or call bio_endio on it if the current process
1536 * is being killed.
1537 */
1538int bio_submit_or_kill(struct bio *bio, unsigned int flags)
1539{
1540 if ((flags & BLKDEV_ZERO_KILLABLE) && fatal_signal_pending(current)) {
1541 bio_await(bio, NULL, bio_endio_cb);
1542 return -EINTR;
1543 }
1544
1545 return submit_bio_wait(bio);
1546}
1547
1548/**
1549 * bdev_rw_virt - synchronously read into / write from kernel mapping
1550 * @bdev: block device to access
1551 * @sector: sector to access
1552 * @data: data to read/write
1553 * @len: length in byte to read/write
1554 * @op: operation (e.g. REQ_OP_READ/REQ_OP_WRITE)
1555 *
1556 * Performs synchronous I/O to @bdev for @data/@len. @data must be in
1557 * the kernel direct mapping and not a vmalloc address.
1558 */
1559int bdev_rw_virt(struct block_device *bdev, sector_t sector, void *data,
1560 size_t len, enum req_op op)
1561{
1562 struct bio_vec bv;
1563 struct bio bio;
1564 int error;
1565
1566 if (WARN_ON_ONCE(is_vmalloc_addr(data)))
1567 return -EIO;
1568
1569 bio_init(&bio, bdev, &bv, 1, op);
1570 bio.bi_iter.bi_sector = sector;
1571 bio_add_virt_nofail(&bio, data, len);
1572 error = submit_bio_wait(&bio);
1573 bio_uninit(&bio);
1574 return error;
1575}
1576EXPORT_SYMBOL_GPL(bdev_rw_virt);
1577
1578void __bio_advance(struct bio *bio, unsigned bytes)
1579{
1580 if (bio_integrity(bio))
1581 bio_integrity_advance(bio, bytes);
1582
1583 bio_crypt_advance(bio, bytes);
1584 bio_advance_iter(bio, &bio->bi_iter, bytes);
1585}
1586EXPORT_SYMBOL(__bio_advance);
1587
1588
1589/**
1590 * bio_copy_data - copy contents of data buffers from one bio to another
1591 * @src: source bio
1592 * @dst: destination bio
1593 *
1594 * Stops when it reaches the end of either @src or @dst - that is, copies
1595 * min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of bios).
1596 */
1597void bio_copy_data(struct bio *dst, struct bio *src)
1598{
1599 struct bvec_iter src_iter = src->bi_iter;
1600 struct bvec_iter dst_iter = dst->bi_iter;
1601
1602 while (src_iter.bi_size && dst_iter.bi_size) {
1603 struct bio_vec src_bv = bio_iter_iovec(src, src_iter);
1604 struct bio_vec dst_bv = bio_iter_iovec(dst, dst_iter);
1605 unsigned int bytes = min(src_bv.bv_len, dst_bv.bv_len);
1606 void *src_buf = bvec_kmap_local(&src_bv);
1607 void *dst_buf = bvec_kmap_local(&dst_bv);
1608
1609 memcpy(dst_buf, src_buf, bytes);
1610
1611 kunmap_local(dst_buf);
1612 kunmap_local(src_buf);
1613
1614 bio_advance_iter_single(src, &src_iter, bytes);
1615 bio_advance_iter_single(dst, &dst_iter, bytes);
1616 }
1617}
1618EXPORT_SYMBOL(bio_copy_data);
1619
1620void bio_free_pages(struct bio *bio)
1621{
1622 struct bio_vec *bvec;
1623 struct bvec_iter_all iter_all;
1624
1625 bio_for_each_segment_all(bvec, bio, iter_all)
1626 __free_page(bvec->bv_page);
1627}
1628EXPORT_SYMBOL(bio_free_pages);
1629
1630/*
1631 * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions
1632 * for performing direct-IO in BIOs.
1633 *
1634 * The problem is that we cannot run folio_mark_dirty() from interrupt context
1635 * because the required locks are not interrupt-safe. So what we can do is to
1636 * mark the pages dirty _before_ performing IO. And in interrupt context,
1637 * check that the pages are still dirty. If so, fine. If not, redirty them
1638 * in process context.
1639 *
1640 * Note that this code is very hard to test under normal circumstances because
1641 * direct-io pins the pages with get_user_pages(). This makes
1642 * is_page_cache_freeable return false, and the VM will not clean the pages.
1643 * But other code (eg, flusher threads) could clean the pages if they are mapped
1644 * pagecache.
1645 *
1646 * Simply disabling the call to bio_set_pages_dirty() is a good way to test the
1647 * deferred bio dirtying paths.
1648 */
1649
1650/*
1651 * bio_set_pages_dirty() will mark all the bio's pages as dirty.
1652 */
1653void bio_set_pages_dirty(struct bio *bio)
1654{
1655 struct folio_iter fi;
1656
1657 bio_for_each_folio_all(fi, bio) {
1658 folio_lock(fi.folio);
1659 folio_mark_dirty(fi.folio);
1660 folio_unlock(fi.folio);
1661 }
1662}
1663
1664/*
1665 * bio_check_pages_dirty() will check that all the BIO's pages are still dirty.
1666 * If they are, then fine. If, however, some pages are clean then they must
1667 * have been written out during the direct-IO read. So we take another ref on
1668 * the BIO and re-dirty the pages in process context.
1669 *
1670 * It is expected that bio_check_pages_dirty() will wholly own the BIO from
1671 * here on. It will unpin each page and will run one bio_put() against the
1672 * BIO.
1673 */
1674
1675static void bio_dirty_fn(struct work_struct *work);
1676
1677static DECLARE_WORK(bio_dirty_work, bio_dirty_fn);
1678static DEFINE_SPINLOCK(bio_dirty_lock);
1679static struct bio *bio_dirty_list;
1680
1681/*
1682 * This runs in process context
1683 */
1684static void bio_dirty_fn(struct work_struct *work)
1685{
1686 struct bio *bio, *next;
1687
1688 spin_lock_irq(&bio_dirty_lock);
1689 next = bio_dirty_list;
1690 bio_dirty_list = NULL;
1691 spin_unlock_irq(&bio_dirty_lock);
1692
1693 while ((bio = next) != NULL) {
1694 next = bio->bi_private;
1695
1696 bio_release_pages(bio, true);
1697 bio_put(bio);
1698 }
1699}
1700
1701void bio_check_pages_dirty(struct bio *bio)
1702{
1703 struct folio_iter fi;
1704 unsigned long flags;
1705
1706 bio_for_each_folio_all(fi, bio) {
1707 if (!folio_test_dirty(fi.folio))
1708 goto defer;
1709 }
1710
1711 bio_release_pages(bio, false);
1712 bio_put(bio);
1713 return;
1714defer:
1715 spin_lock_irqsave(&bio_dirty_lock, flags);
1716 bio->bi_private = bio_dirty_list;
1717 bio_dirty_list = bio;
1718 spin_unlock_irqrestore(&bio_dirty_lock, flags);
1719 schedule_work(&bio_dirty_work);
1720}
1721
1722/*
1723 * Infrastructure for deferring bio completions to task-context via a per-CPU
1724 * workqueue. Triggered either by the BIO_COMPLETE_IN_TASK bio flag (static
1725 * decision at submit time) or by calling bio_complete_in_task() from
1726 * bi_end_io() (dynamic decision at completion time).
1727 */
1728
1729struct bio_complete_batch {
1730 struct bio_list list;
1731 struct work_struct work;
1732 int cpu;
1733};
1734
1735static DEFINE_PER_CPU(struct bio_complete_batch, bio_complete_batch);
1736static struct workqueue_struct *bio_complete_wq;
1737
1738static void bio_complete_work_fn(struct work_struct *w)
1739{
1740 struct bio_complete_batch *batch =
1741 container_of(w, struct bio_complete_batch, work);
1742
1743 while (1) {
1744 struct bio_list list;
1745 struct bio *bio;
1746
1747 local_irq_disable();
1748 list = batch->list;
1749 bio_list_init(&batch->list);
1750 local_irq_enable();
1751
1752 if (bio_list_empty(&list))
1753 break;
1754
1755 while ((bio = bio_list_pop(&list)))
1756 bio->bi_end_io(bio);
1757 }
1758}
1759
1760void __bio_complete_in_task(struct bio *bio)
1761{
1762 struct bio_complete_batch *batch;
1763 unsigned long flags;
1764 bool was_empty;
1765
1766 local_irq_save(flags);
1767 batch = this_cpu_ptr(&bio_complete_batch);
1768 was_empty = bio_list_empty(&batch->list);
1769 bio_list_add(&batch->list, bio);
1770 local_irq_restore(flags);
1771
1772 if (was_empty)
1773 queue_work_on(batch->cpu, bio_complete_wq, &batch->work);
1774}
1775EXPORT_SYMBOL_GPL(__bio_complete_in_task);
1776
1777static inline bool bio_remaining_done(struct bio *bio)
1778{
1779 /*
1780 * If we're not chaining, then ->__bi_remaining is always 1 and
1781 * we always end io on the first invocation.
1782 */
1783 if (!bio_flagged(bio, BIO_CHAIN))
1784 return true;
1785
1786 BUG_ON(atomic_read(&bio->__bi_remaining) <= 0);
1787
1788 if (atomic_dec_and_test(&bio->__bi_remaining)) {
1789 bio_clear_flag(bio, BIO_CHAIN);
1790 return true;
1791 }
1792
1793 return false;
1794}
1795
1796/**
1797 * bio_endio - end I/O on a bio
1798 * @bio: bio
1799 *
1800 * Description:
1801 * bio_endio() will end I/O on the whole bio. bio_endio() is the preferred
1802 * way to end I/O on a bio. No one should call bi_end_io() directly on a
1803 * bio unless they own it and thus know that it has an end_io function.
1804 *
1805 * bio_endio() can be called several times on a bio that has been chained
1806 * using bio_chain(). The ->bi_end_io() function will only be called the
1807 * last time.
1808 **/
1809void bio_endio(struct bio *bio)
1810{
1811again:
1812 if (!bio_remaining_done(bio))
1813 return;
1814 if (!bio_integrity_endio(bio))
1815 return;
1816
1817 blk_zone_bio_endio(bio);
1818
1819 rq_qos_done_bio(bio);
1820
1821 if (bio->bi_bdev && bio_flagged(bio, BIO_TRACE_COMPLETION)) {
1822 trace_block_bio_complete(bdev_get_queue(bio->bi_bdev), bio);
1823 bio_clear_flag(bio, BIO_TRACE_COMPLETION);
1824 }
1825
1826 /*
1827 * Need to have a real endio function for chained bios, otherwise
1828 * various corner cases will break (like stacking block devices that
1829 * save/restore bi_end_io) - however, we want to avoid unbounded
1830 * recursion and blowing the stack. Tail call optimization would
1831 * handle this, but compiling with frame pointers also disables
1832 * gcc's sibling call optimization.
1833 */
1834 if (bio->bi_end_io == bio_chain_endio) {
1835 bio = __bio_chain_endio(bio);
1836 goto again;
1837 }
1838
1839#ifdef CONFIG_BLK_CGROUP
1840 /*
1841 * Release cgroup info. We shouldn't have to do this here, but quite
1842 * a few callers of bio_init fail to call bio_uninit, so we cover up
1843 * for that here at least for now.
1844 */
1845 if (bio->bi_blkg) {
1846 blkg_put(bio->bi_blkg);
1847 bio->bi_blkg = NULL;
1848 }
1849#endif
1850
1851 if (bio_flagged(bio, BIO_COMPLETE_IN_TASK) && bio_in_atomic())
1852 __bio_complete_in_task(bio);
1853 else if (bio->bi_end_io)
1854 bio->bi_end_io(bio);
1855}
1856EXPORT_SYMBOL(bio_endio);
1857
1858/**
1859 * bio_split - split a bio
1860 * @bio: bio to split
1861 * @sectors: number of sectors to split from the front of @bio
1862 * @gfp: gfp mask
1863 * @bs: bio set to allocate from
1864 *
1865 * Allocates and returns a new bio which represents @sectors from the start of
1866 * @bio, and updates @bio to represent the remaining sectors.
1867 *
1868 * Unless this is a discard request the newly allocated bio will point
1869 * to @bio's bi_io_vec. It is the caller's responsibility to ensure that
1870 * neither @bio nor @bs are freed before the split bio.
1871 */
1872struct bio *bio_split(struct bio *bio, int sectors,
1873 gfp_t gfp, struct bio_set *bs)
1874{
1875 struct bio *split;
1876
1877 if (WARN_ON_ONCE(sectors <= 0))
1878 return ERR_PTR(-EINVAL);
1879 if (WARN_ON_ONCE(sectors >= bio_sectors(bio)))
1880 return ERR_PTR(-EINVAL);
1881
1882 /* Zone append commands cannot be split */
1883 if (WARN_ON_ONCE(bio_op(bio) == REQ_OP_ZONE_APPEND))
1884 return ERR_PTR(-EINVAL);
1885
1886 /* atomic writes cannot be split */
1887 if (bio->bi_opf & REQ_ATOMIC)
1888 return ERR_PTR(-EINVAL);
1889
1890 split = bio_alloc_clone(bio->bi_bdev, bio, gfp, bs);
1891 if (!split)
1892 return ERR_PTR(-ENOMEM);
1893
1894 split->bi_iter.bi_size = sectors << 9;
1895
1896 if (bio_integrity(split))
1897 bio_integrity_trim(split);
1898
1899 bio_advance(bio, split->bi_iter.bi_size);
1900
1901 /*
1902 * The gap bit is set when splitting to limits and only applies to the
1903 * front bio that was split off. The remaining bio will calcualte its
1904 * gap value when it is subsequently split to limits, so it is safe to
1905 * re-initialize the value back to 0.
1906 */
1907 bio->bi_bvec_gap_bit = 0;
1908
1909 if (bio_flagged(bio, BIO_TRACE_COMPLETION))
1910 bio_set_flag(split, BIO_TRACE_COMPLETION);
1911
1912 return split;
1913}
1914EXPORT_SYMBOL(bio_split);
1915
1916/**
1917 * bio_trim - trim a bio
1918 * @bio: bio to trim
1919 * @offset: number of sectors to trim from the front of @bio
1920 * @size: size we want to trim @bio to, in sectors
1921 *
1922 * This function is typically used for bios that are cloned and submitted
1923 * to the underlying device in parts.
1924 */
1925void bio_trim(struct bio *bio, sector_t offset, sector_t size)
1926{
1927 /* We should never trim an atomic write */
1928 if (WARN_ON_ONCE(bio->bi_opf & REQ_ATOMIC && size))
1929 return;
1930
1931 if (WARN_ON_ONCE(offset > BIO_MAX_SECTORS || size > BIO_MAX_SECTORS ||
1932 offset + size > bio_sectors(bio)))
1933 return;
1934
1935 size <<= 9;
1936 if (offset == 0 && size == bio->bi_iter.bi_size)
1937 return;
1938
1939 bio_advance(bio, offset << 9);
1940 bio->bi_iter.bi_size = size;
1941
1942 if (bio_integrity(bio))
1943 bio_integrity_trim(bio);
1944}
1945EXPORT_SYMBOL_GPL(bio_trim);
1946
1947/*
1948 * create memory pools for biovec's in a bio_set.
1949 * use the global biovec slabs created for general use.
1950 */
1951static int biovec_init_pool(mempool_t *pool, int pool_entries)
1952{
1953 struct biovec_slab *bp = bvec_slabs + ARRAY_SIZE(bvec_slabs) - 1;
1954
1955 return mempool_init_slab_pool(pool, pool_entries, bp->slab);
1956}
1957
1958/*
1959 * bioset_exit - exit a bioset initialized with bioset_init()
1960 *
1961 * May be called on a zeroed but uninitialized bioset (i.e. allocated with
1962 * kzalloc()).
1963 */
1964void bioset_exit(struct bio_set *bs)
1965{
1966 bio_alloc_cache_destroy(bs);
1967 if (bs->rescue_workqueue)
1968 destroy_workqueue(bs->rescue_workqueue);
1969 bs->rescue_workqueue = NULL;
1970
1971 mempool_exit(&bs->bio_pool);
1972 mempool_exit(&bs->bvec_pool);
1973
1974 if (bs->bio_slab)
1975 bio_put_slab(bs);
1976 bs->bio_slab = NULL;
1977}
1978EXPORT_SYMBOL(bioset_exit);
1979
1980/**
1981 * bioset_init - Initialize a bio_set
1982 * @bs: pool to initialize
1983 * @pool_size: Number of bio and bio_vecs to cache in the mempool
1984 * @front_pad: Number of bytes to allocate in front of the returned bio
1985 * @flags: Flags to modify behavior, currently %BIOSET_NEED_BVECS
1986 * and %BIOSET_NEED_RESCUER
1987 *
1988 * Description:
1989 * Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller
1990 * to ask for a number of bytes to be allocated in front of the bio.
1991 * Front pad allocation is useful for embedding the bio inside
1992 * another structure, to avoid allocating extra data to go with the bio.
1993 * Note that the bio must be embedded at the END of that structure always,
1994 * or things will break badly.
1995 * If %BIOSET_NEED_BVECS is set in @flags, a separate pool will be allocated
1996 * for allocating iovecs. This pool is not needed e.g. for bio_init_clone().
1997 * If %BIOSET_NEED_RESCUER is set, a workqueue is created which can be used
1998 * to dispatch queued requests when the mempool runs out of space.
1999 *
2000 */
2001int bioset_init(struct bio_set *bs,
2002 unsigned int pool_size,
2003 unsigned int front_pad,
2004 int flags)
2005{
2006 bs->front_pad = front_pad;
2007 if (flags & BIOSET_NEED_BVECS)
2008 bs->back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
2009 else
2010 bs->back_pad = 0;
2011
2012 spin_lock_init(&bs->rescue_lock);
2013 bio_list_init(&bs->rescue_list);
2014 INIT_WORK(&bs->rescue_work, bio_alloc_rescue);
2015
2016 bs->bio_slab = bio_find_or_create_slab(bs);
2017 if (!bs->bio_slab)
2018 return -ENOMEM;
2019
2020 if (mempool_init_slab_pool(&bs->bio_pool, pool_size, bs->bio_slab))
2021 goto bad;
2022
2023 if ((flags & BIOSET_NEED_BVECS) &&
2024 biovec_init_pool(&bs->bvec_pool, pool_size))
2025 goto bad;
2026
2027 if (flags & BIOSET_NEED_RESCUER) {
2028 bs->rescue_workqueue = alloc_workqueue("bioset",
2029 WQ_MEM_RECLAIM | WQ_PERCPU, 0);
2030 if (!bs->rescue_workqueue)
2031 goto bad;
2032 }
2033 if (flags & BIOSET_PERCPU_CACHE) {
2034 bs->cache = alloc_percpu(struct bio_alloc_cache);
2035 if (!bs->cache)
2036 goto bad;
2037 cpuhp_state_add_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead);
2038 }
2039
2040 return 0;
2041bad:
2042 bioset_exit(bs);
2043 return -ENOMEM;
2044}
2045EXPORT_SYMBOL(bioset_init);
2046
2047static int bio_complete_batch_cpu_online(unsigned int cpu)
2048{
2049 struct bio_complete_batch *batch = &per_cpu(bio_complete_batch, cpu);
2050
2051 enable_work(&batch->work);
2052 if (!bio_list_empty(&batch->list))
2053 queue_work_on(cpu, bio_complete_wq, &batch->work);
2054 return 0;
2055}
2056
2057/*
2058 * Disable this CPU's work item so that it cannot run on an unbound worker
2059 * after the CPU is offlined.
2060 */
2061static int bio_complete_batch_cpu_down_prep(unsigned int cpu)
2062{
2063 disable_work_sync(&per_cpu(bio_complete_batch, cpu).work);
2064 return 0;
2065}
2066
2067/*
2068 * Drain a dead CPU's deferred bio completions. The CPU is dead and the worker
2069 * is canceled so no locking is needed.
2070 */
2071static int bio_complete_batch_cpu_dead(unsigned int cpu)
2072{
2073 struct bio_complete_batch *batch =
2074 per_cpu_ptr(&bio_complete_batch, cpu);
2075 struct bio *bio;
2076
2077 while ((bio = bio_list_pop(&batch->list)))
2078 bio->bi_end_io(bio);
2079
2080 return 0;
2081}
2082
2083static void __init bio_complete_batch_init(int cpu)
2084{
2085 struct bio_complete_batch *batch =
2086 per_cpu_ptr(&bio_complete_batch, cpu);
2087
2088 bio_list_init(&batch->list);
2089 INIT_WORK(&batch->work, bio_complete_work_fn);
2090 batch->cpu = cpu;
2091
2092 if (!cpu_online(cpu))
2093 disable_work_sync(&batch->work);
2094}
2095
2096static int __init init_bio(void)
2097{
2098 int i;
2099
2100 BUILD_BUG_ON(BIO_FLAG_LAST > 8 * sizeof_field(struct bio, bi_flags));
2101
2102 for (i = 0; i < ARRAY_SIZE(bvec_slabs); i++) {
2103 struct biovec_slab *bvs = bvec_slabs + i;
2104
2105 bvs->slab = kmem_cache_create(bvs->name,
2106 bvs->nr_vecs * sizeof(struct bio_vec), 0,
2107 SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
2108 }
2109
2110 for_each_possible_cpu(i)
2111 bio_complete_batch_init(i);
2112
2113 bio_complete_wq = alloc_workqueue("bio_complete",
2114 WQ_MEM_RECLAIM | WQ_PERCPU, 0);
2115 if (!bio_complete_wq)
2116 panic("bio: can't allocate bio_complete workqueue\n");
2117
2118 /*
2119 * bio task-context completion draining on hot-unplugged CPUs:
2120 *
2121 * 1. Stop the per-CPU work item while the CPU is still online, so
2122 * that it cannot run on an unbound worker later.
2123 * 2. Drain leftover bios added between worker disabling and CPU
2124 * offlining.
2125 */
2126 cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
2127 "block/bio:complete:online",
2128 bio_complete_batch_cpu_online,
2129 bio_complete_batch_cpu_down_prep);
2130 cpuhp_setup_state_nocalls(CPUHP_BP_PREPARE_DYN,
2131 "block/bio:complete:dead",
2132 NULL, bio_complete_batch_cpu_dead);
2133
2134 cpuhp_setup_state_multi(CPUHP_BIO_DEAD, "block/bio:dead", NULL,
2135 bio_cpu_dead);
2136
2137 if (bioset_init(&fs_bio_set, BIO_POOL_SIZE, 0,
2138 BIOSET_NEED_BVECS | BIOSET_PERCPU_CACHE))
2139 panic("bio: can't allocate bios\n");
2140
2141 return 0;
2142}
2143subsys_initcall(init_bio);