projects/linux·blob master · refs

820 lines · 24.5 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef BLK_INTERNAL_H
3#define BLK_INTERNAL_H
4
5#include <linux/bio-integrity.h>
6#include <linux/blk-crypto.h>
7#include <linux/part_stat.h>
8#include <linux/lockdep.h>
9#include <linux/memblock.h> /* for max_pfn/max_low_pfn */
10#include <linux/sched/sysctl.h>
11#include <linux/timekeeping.h>
12#include <xen/xen.h>
13#include "blk-crypto-internal.h"
14
15struct elv_change_ctx;
16
17/*
18 * Default upper limit for the software max_sectors limit used for regular I/Os.
19 * This can be increased through sysfs.
20 *
21 * This should not be confused with the max_hw_sector limit that is entirely
22 * controlled by the block device driver, usually based on hardware limits.
23 */
24#define BLK_DEF_MAX_SECTORS_CAP (SZ_4M >> SECTOR_SHIFT)
25
26#define BLK_DEV_MAX_SECTORS (LLONG_MAX >> 9)
27#define BLK_MIN_SEGMENT_SIZE 4096
28
29/* Max future timer expiry for timeouts */
30#define BLK_MAX_TIMEOUT (5 * HZ)
31
32extern const struct kobj_type blk_queue_ktype;
33extern struct dentry *blk_debugfs_root;
34
35struct blk_flush_queue {
36 spinlock_t mq_flush_lock;
37 unsigned int flush_pending_idx:1;
38 unsigned int flush_running_idx:1;
39 blk_status_t rq_status;
40 unsigned long flush_pending_since;
41 struct list_head flush_queue[2];
42 unsigned long flush_data_in_flight;
43 struct request *flush_rq;
44 struct rcu_head rcu_head;
45};
46
47bool is_flush_rq(struct request *req);
48
49struct blk_flush_queue *blk_alloc_flush_queue(int node, int cmd_size,
50 gfp_t flags);
51void blk_free_flush_queue(struct blk_flush_queue *q);
52
53const char *blk_status_to_str(blk_status_t status);
54const char *blk_status_to_tag(blk_status_t status);
55blk_status_t tag_to_blk_status(const char *tag);
56enum req_op str_to_blk_op(const char *op);
57
58bool __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic);
59bool blk_queue_start_drain(struct request_queue *q);
60bool __blk_freeze_queue_start(struct request_queue *q,
61 struct task_struct *owner);
62int __bio_queue_enter(struct request_queue *q, struct bio *bio);
63void submit_bio_noacct_nocheck(struct bio *bio, bool split);
64int bio_submit_or_kill(struct bio *bio, unsigned int flags);
65
66static inline bool blk_try_enter_queue(struct request_queue *q, bool pm)
67{
68 rcu_read_lock();
69 if (!percpu_ref_tryget_live_rcu(&q->q_usage_counter))
70 goto fail;
71
72 /*
73 * The code that increments the pm_only counter must ensure that the
74 * counter is globally visible before the queue is unfrozen.
75 */
76 if (blk_queue_pm_only(q) &&
77 (!pm || queue_rpm_status(q) == RPM_SUSPENDED))
78 goto fail_put;
79
80 rcu_read_unlock();
81 return true;
82
83fail_put:
84 blk_queue_exit(q);
85fail:
86 rcu_read_unlock();
87 return false;
88}
89
90static inline int bio_queue_enter(struct bio *bio)
91{
92 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
93
94 if (blk_try_enter_queue(q, false)) {
95 rwsem_acquire_read(&q->io_lockdep_map, 0, 0, _RET_IP_);
96 rwsem_release(&q->io_lockdep_map, _RET_IP_);
97 return 0;
98 }
99 return __bio_queue_enter(q, bio);
100}
101
102static inline void blk_wait_io(struct completion *done)
103{
104 /* Prevent hang_check timer from firing at us during very long I/O */
105 unsigned long timeout = sysctl_hung_task_timeout_secs * HZ / 2;
106
107 if (timeout)
108 while (!wait_for_completion_io_timeout(done, timeout))
109 ;
110 else
111 wait_for_completion_io(done);
112}
113
114struct block_device *blkdev_get_no_open(dev_t dev, bool autoload);
115void blkdev_put_no_open(struct block_device *bdev);
116
117bool bvec_try_merge_hw_page(struct request_queue *q, struct bio_vec *bv,
118 struct page *page, unsigned len, unsigned offset);
119
120static inline bool biovec_phys_mergeable(struct request_queue *q,
121 struct bio_vec *vec1, struct bio_vec *vec2)
122{
123 unsigned long mask = queue_segment_boundary(q);
124 phys_addr_t addr1 = bvec_phys(vec1);
125 phys_addr_t addr2 = bvec_phys(vec2);
126
127 /*
128 * Merging adjacent physical pages may not work correctly under KMSAN
129 * if their metadata pages aren't adjacent. Just disable merging.
130 */
131 if (IS_ENABLED(CONFIG_KMSAN))
132 return false;
133
134 if (addr1 + vec1->bv_len != addr2)
135 return false;
136 if (!zone_device_pages_have_same_pgmap(vec1->bv_page, vec2->bv_page))
137 return false;
138 if (xen_domain() && !xen_biovec_phys_mergeable(vec1, vec2->bv_page))
139 return false;
140 if ((addr1 | mask) != ((addr2 + vec2->bv_len - 1) | mask))
141 return false;
142 return true;
143}
144
145/*
146 * Check if two pages from potentially different zone device pgmaps can
147 * coexist as separate bvec entries in the same bio.
148 *
149 * The block DMA iterator (blk_dma_map_iter_start) caches the P2PDMA mapping
150 * state from the first segment and applies it to all subsequent segments, so
151 * P2PDMA pages from different pgmaps must not be mixed in the same bio.
152 *
153 * Other zone device types (FS_DAX, GENERIC) use the same dma_map_phys() path
154 * as normal RAM. PRIVATE and COHERENT pages never appear in bios.
155 */
156static inline bool zone_device_pages_compatible(const struct page *a,
157 const struct page *b)
158{
159 if (is_pci_p2pdma_page(a) || is_pci_p2pdma_page(b))
160 return zone_device_pages_have_same_pgmap(a, b);
161 return true;
162}
163
164static inline bool __bvec_gap_to_prev(const struct queue_limits *lim,
165 struct bio_vec *bprv, unsigned int offset)
166{
167 return (offset & lim->virt_boundary_mask) ||
168 ((bprv->bv_offset + bprv->bv_len) & lim->virt_boundary_mask);
169}
170
171/*
172 * Check if adding a bio_vec after bprv with offset would create a gap in
173 * the SG list. Most drivers don't care about this, but some do.
174 */
175static inline bool bvec_gap_to_prev(const struct queue_limits *lim,
176 struct bio_vec *bprv, unsigned int offset)
177{
178 if (!lim->virt_boundary_mask)
179 return false;
180 return __bvec_gap_to_prev(lim, bprv, offset);
181}
182
183static inline bool rq_mergeable(struct request *rq)
184{
185 if (blk_rq_is_passthrough(rq))
186 return false;
187
188 if (req_op(rq) == REQ_OP_FLUSH)
189 return false;
190
191 if (req_op(rq) == REQ_OP_WRITE_ZEROES)
192 return false;
193
194 if (req_op(rq) == REQ_OP_ZONE_APPEND)
195 return false;
196
197 if (rq->cmd_flags & REQ_NOMERGE_FLAGS)
198 return false;
199 if (rq->rq_flags & RQF_NOMERGE_FLAGS)
200 return false;
201
202 return true;
203}
204
205/*
206 * There are two different ways to handle DISCARD merges:
207 * 1) If max_discard_segments > 1, the driver treats every bio as a range and
208 * send the bios to controller together. The ranges don't need to be
209 * contiguous.
210 * 2) Otherwise, the request will be normal read/write requests. The ranges
211 * need to be contiguous.
212 */
213static inline bool blk_discard_mergable(struct request *req)
214{
215 if (req_op(req) == REQ_OP_DISCARD &&
216 queue_max_discard_segments(req->q) > 1)
217 return true;
218 return false;
219}
220
221static inline unsigned int blk_rq_get_max_segments(struct request *rq)
222{
223 if (req_op(rq) == REQ_OP_DISCARD)
224 return queue_max_discard_segments(rq->q);
225 return queue_max_segments(rq->q);
226}
227
228static inline unsigned int blk_queue_get_max_sectors(struct request *rq)
229{
230 struct request_queue *q = rq->q;
231 enum req_op op = req_op(rq);
232
233 if (unlikely(op == REQ_OP_DISCARD))
234 return min(q->limits.max_discard_sectors,
235 UINT_MAX >> SECTOR_SHIFT);
236
237 if (unlikely(op == REQ_OP_SECURE_ERASE))
238 return min(q->limits.max_secure_erase_sectors,
239 UINT_MAX >> SECTOR_SHIFT);
240
241 if (unlikely(op == REQ_OP_WRITE_ZEROES))
242 return q->limits.max_write_zeroes_sectors;
243
244 if (rq->cmd_flags & REQ_ATOMIC)
245 return q->limits.atomic_write_max_sectors;
246
247 return q->limits.max_sectors;
248}
249
250#ifdef CONFIG_BLK_DEV_INTEGRITY
251void blk_flush_integrity(void);
252void bio_integrity_free(struct bio *bio);
253
254/*
255 * Integrity payloads can either be owned by the submitter, in which case
256 * bio_uninit will free them, or owned and generated by the block layer,
257 * in which case we'll verify them here (for reads) and free them before
258 * the bio is handed back to the submitted.
259 */
260bool __bio_integrity_endio(struct bio *bio);
261static inline bool bio_integrity_endio(struct bio *bio)
262{
263 struct bio_integrity_payload *bip = bio_integrity(bio);
264
265 if (bip && (bip->bip_flags & BIP_BLOCK_INTEGRITY))
266 return __bio_integrity_endio(bio);
267 return true;
268}
269
270bool blk_integrity_merge_rq(struct request_queue *, struct request *,
271 struct request *);
272bool blk_integrity_merge_bio(struct request_queue *, struct request *,
273 struct bio *);
274
275static inline bool integrity_req_gap_back_merge(struct request *req,
276 struct bio *next)
277{
278 struct bio_integrity_payload *bip = bio_integrity(req->bio);
279 struct bio_integrity_payload *bip_next = bio_integrity(next);
280
281 return bvec_gap_to_prev(&req->q->limits,
282 &bip->bip_vec[bip->bip_vcnt - 1],
283 bip_next->bip_vec[0].bv_offset);
284}
285
286static inline bool integrity_req_gap_front_merge(struct request *req,
287 struct bio *bio)
288{
289 struct bio_integrity_payload *bip = bio_integrity(bio);
290 struct bio_integrity_payload *bip_next = bio_integrity(req->bio);
291
292 return bvec_gap_to_prev(&req->q->limits,
293 &bip->bip_vec[bip->bip_vcnt - 1],
294 bip_next->bip_vec[0].bv_offset);
295}
296
297extern const struct attribute_group blk_integrity_attr_group;
298#else /* CONFIG_BLK_DEV_INTEGRITY */
299static inline bool blk_integrity_merge_rq(struct request_queue *rq,
300 struct request *r1, struct request *r2)
301{
302 return true;
303}
304static inline bool blk_integrity_merge_bio(struct request_queue *rq,
305 struct request *r, struct bio *b)
306{
307 return true;
308}
309static inline bool integrity_req_gap_back_merge(struct request *req,
310 struct bio *next)
311{
312 return false;
313}
314static inline bool integrity_req_gap_front_merge(struct request *req,
315 struct bio *bio)
316{
317 return false;
318}
319
320static inline void blk_flush_integrity(void)
321{
322}
323static inline bool bio_integrity_endio(struct bio *bio)
324{
325 return true;
326}
327static inline void bio_integrity_free(struct bio *bio)
328{
329}
330#endif /* CONFIG_BLK_DEV_INTEGRITY */
331
332unsigned long blk_rq_timeout(unsigned long timeout);
333void blk_add_timer(struct request *req);
334
335enum bio_merge_status {
336 BIO_MERGE_OK,
337 BIO_MERGE_NONE,
338 BIO_MERGE_FAILED,
339};
340
341enum bio_merge_status bio_attempt_back_merge(struct request *req,
342 struct bio *bio, unsigned int nr_segs);
343bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
344 unsigned int nr_segs);
345bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
346 struct bio *bio, unsigned int nr_segs);
347
348/*
349 * Plug flush limits
350 */
351#define BLK_MAX_REQUEST_COUNT 32
352#define BLK_PLUG_FLUSH_SIZE (128 * 1024)
353
354/*
355 * Internal elevator interface
356 */
357#define ELV_ON_HASH(rq) ((rq)->rq_flags & RQF_HASHED)
358
359bool blk_insert_flush(struct request *rq);
360
361void elv_update_nr_hw_queues(struct request_queue *q,
362 struct elv_change_ctx *ctx);
363void elevator_set_default(struct request_queue *q);
364void elevator_set_none(struct request_queue *q);
365
366ssize_t part_size_show(struct device *dev, struct device_attribute *attr,
367 char *buf);
368ssize_t part_stat_show(struct device *dev, struct device_attribute *attr,
369 char *buf);
370ssize_t part_inflight_show(struct device *dev, struct device_attribute *attr,
371 char *buf);
372ssize_t part_fail_show(struct device *dev, struct device_attribute *attr,
373 char *buf);
374ssize_t part_fail_store(struct device *dev, struct device_attribute *attr,
375 const char *buf, size_t count);
376ssize_t part_timeout_show(struct device *, struct device_attribute *, char *);
377ssize_t part_timeout_store(struct device *, struct device_attribute *,
378 const char *, size_t);
379
380struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim,
381 unsigned *nsegs);
382struct bio *bio_split_write_zeroes(struct bio *bio,
383 const struct queue_limits *lim, unsigned *nsegs);
384struct bio *bio_split_rw(struct bio *bio, const struct queue_limits *lim,
385 unsigned *nr_segs);
386struct bio *bio_split_zone_append(struct bio *bio,
387 const struct queue_limits *lim, unsigned *nr_segs);
388
389/*
390 * All drivers must accept single-segments bios that are smaller than PAGE_SIZE.
391 *
392 * This is a quick and dirty check that relies on the fact that bi_io_vec[0] is
393 * always valid if a bio has data. The check might lead to occasional false
394 * positives when bios are cloned, but compared to the performance impact of
395 * cloned bios themselves the loop below doesn't matter anyway.
396 */
397static inline bool bio_may_need_split(struct bio *bio,
398 const struct queue_limits *lim)
399{
400 const struct bio_vec *bv;
401
402 if (lim->chunk_sectors)
403 return true;
404
405 if (!bio->bi_io_vec)
406 return true;
407
408 bv = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
409 if (bio->bi_iter.bi_size > bv->bv_len - bio->bi_iter.bi_offset)
410 return true;
411 if ((bv->bv_offset | bv->bv_len) & lim->dma_alignment)
412 return true;
413 return bv->bv_len + bv->bv_offset > lim->max_fast_segment_size;
414}
415
416/**
417 * __bio_split_to_limits - split a bio to fit the queue limits
418 * @bio: bio to be split
419 * @lim: queue limits to split based on
420 * @nr_segs: returns the number of segments in the returned bio
421 *
422 * Check if @bio needs splitting based on the queue limits, and if so split off
423 * a bio fitting the limits from the beginning of @bio and return it. @bio is
424 * shortened to the remainder and re-submitted.
425 *
426 * The split bio is allocated from @q->bio_split, which is provided by the
427 * block layer.
428 */
429static inline struct bio *__bio_split_to_limits(struct bio *bio,
430 const struct queue_limits *lim, unsigned int *nr_segs)
431{
432 switch (bio_op(bio)) {
433 case REQ_OP_READ:
434 case REQ_OP_WRITE:
435 if (bio_may_need_split(bio, lim))
436 return bio_split_rw(bio, lim, nr_segs);
437 *nr_segs = 1;
438 return bio;
439 case REQ_OP_ZONE_APPEND:
440 return bio_split_zone_append(bio, lim, nr_segs);
441 case REQ_OP_DISCARD:
442 case REQ_OP_SECURE_ERASE:
443 return bio_split_discard(bio, lim, nr_segs);
444 case REQ_OP_WRITE_ZEROES:
445 return bio_split_write_zeroes(bio, lim, nr_segs);
446 default:
447 /* other operations can't be split */
448 *nr_segs = 0;
449 return bio;
450 }
451}
452
453/**
454 * get_max_segment_size() - maximum number of bytes to add as a single segment
455 * @lim: Request queue limits.
456 * @paddr: address of the range to add
457 * @len: maximum length available to add at @paddr
458 *
459 * Returns the maximum number of bytes of the range starting at @paddr that can
460 * be added to a single segment.
461 */
462static inline unsigned get_max_segment_size(const struct queue_limits *lim,
463 phys_addr_t paddr, unsigned int len)
464{
465 /*
466 * Prevent an overflow if mask = ULONG_MAX and offset = 0 by adding 1
467 * after having calculated the minimum.
468 */
469 return min_t(unsigned long, len,
470 min(lim->seg_boundary_mask - (lim->seg_boundary_mask & paddr),
471 (unsigned long)lim->max_segment_size - 1) + 1);
472}
473
474int ll_back_merge_fn(struct request *req, struct bio *bio,
475 unsigned int nr_segs);
476bool blk_attempt_req_merge(struct request_queue *q, struct request *rq,
477 struct request *next);
478unsigned int blk_recalc_rq_segments(struct request *rq);
479bool blk_rq_merge_ok(struct request *rq, struct bio *bio);
480enum elv_merge blk_try_merge(struct request *rq, struct bio *bio);
481
482int blk_set_default_limits(struct queue_limits *lim);
483void blk_apply_bdi_limits(struct backing_dev_info *bdi,
484 struct queue_limits *lim);
485int blk_dev_init(void);
486
487void update_io_ticks(struct block_device *part, unsigned long now, bool end);
488
489static inline void req_set_nomerge(struct request_queue *q, struct request *req)
490{
491 req->cmd_flags |= REQ_NOMERGE;
492 if (req == q->last_merge)
493 q->last_merge = NULL;
494}
495
496static inline void bdev_inc_in_flight(struct block_device *bdev,
497 enum req_op op)
498{
499 bool rw = op_is_write(op);
500
501 part_stat_local_inc(bdev, in_flight[rw]);
502 if (bdev_is_partition(bdev))
503 part_stat_local_inc(bdev_whole(bdev), in_flight[rw]);
504}
505
506static inline void bdev_dec_in_flight(struct block_device *bdev,
507 enum req_op op)
508{
509 bool rw = op_is_write(op);
510
511 part_stat_local_dec(bdev, in_flight[rw]);
512 if (bdev_is_partition(bdev))
513 part_stat_local_dec(bdev_whole(bdev), in_flight[rw]);
514}
515
516/*
517 * Internal io_context interface
518 */
519struct io_cq *ioc_find_get_icq(struct request_queue *q);
520struct io_cq *ioc_lookup_icq(struct request_queue *q);
521#ifdef CONFIG_BLK_ICQ
522void ioc_clear_queue(struct request_queue *q);
523#else
524static inline void ioc_clear_queue(struct request_queue *q)
525{
526}
527#endif /* CONFIG_BLK_ICQ */
528
529#ifdef CONFIG_BLK_DEV_ZONED
530void disk_init_zone_resources(struct gendisk *disk);
531void disk_release_zone_resources(struct gendisk *disk);
532static inline bool bio_zone_write_plugging(struct bio *bio)
533{
534 return bio_flagged(bio, BIO_ZONE_WRITE_PLUGGING);
535}
536static inline bool blk_req_bio_is_zone_append(struct request *rq,
537 struct bio *bio)
538{
539 return req_op(rq) == REQ_OP_ZONE_APPEND ||
540 bio_flagged(bio, BIO_EMULATES_ZONE_APPEND);
541}
542void blk_zone_write_plug_bio_merged(struct bio *bio);
543void blk_zone_write_plug_init_request(struct request *rq);
544void blk_zone_append_update_request_bio(struct request *rq, struct bio *bio);
545void blk_zone_mgmt_bio_endio(struct bio *bio);
546void blk_zone_write_plug_bio_endio(struct bio *bio);
547static inline void blk_zone_bio_endio(struct bio *bio)
548{
549 /*
550 * Zone management BIOs may impact zone write plugs (e.g. a zone reset
551 * changes a zone write plug zone write pointer offset), but these
552 * operation do not go through zone write plugging as they may operate
553 * on zones that do not have a zone write
554 * plug. blk_zone_mgmt_bio_endio() handles the potential changes to zone
555 * write plugs that are present.
556 */
557 if (op_is_zone_mgmt(bio_op(bio))) {
558 blk_zone_mgmt_bio_endio(bio);
559 return;
560 }
561
562 /*
563 * For write BIOs to zoned devices, signal the completion of the BIO so
564 * that the next write BIO can be submitted by zone write plugging.
565 */
566 if (bio_zone_write_plugging(bio))
567 blk_zone_write_plug_bio_endio(bio);
568}
569
570void blk_zone_write_plug_finish_request(struct request *rq);
571static inline void blk_zone_finish_request(struct request *rq)
572{
573 if (rq->rq_flags & RQF_ZONE_WRITE_PLUGGING)
574 blk_zone_write_plug_finish_request(rq);
575}
576int blkdev_report_zones_ioctl(struct block_device *bdev, unsigned int cmd,
577 unsigned long arg);
578int blkdev_zone_mgmt_ioctl(struct block_device *bdev, blk_mode_t mode,
579 unsigned int cmd, unsigned long arg);
580#else /* CONFIG_BLK_DEV_ZONED */
581static inline void disk_init_zone_resources(struct gendisk *disk)
582{
583}
584static inline void disk_release_zone_resources(struct gendisk *disk)
585{
586}
587static inline bool bio_zone_write_plugging(struct bio *bio)
588{
589 return false;
590}
591static inline bool blk_req_bio_is_zone_append(struct request *req,
592 struct bio *bio)
593{
594 return false;
595}
596static inline void blk_zone_write_plug_bio_merged(struct bio *bio)
597{
598}
599static inline void blk_zone_write_plug_init_request(struct request *rq)
600{
601}
602static inline void blk_zone_append_update_request_bio(struct request *rq,
603 struct bio *bio)
604{
605}
606static inline void blk_zone_bio_endio(struct bio *bio)
607{
608}
609static inline void blk_zone_finish_request(struct request *rq)
610{
611}
612static inline int blkdev_report_zones_ioctl(struct block_device *bdev,
613 unsigned int cmd, unsigned long arg)
614{
615 return -ENOTTY;
616}
617static inline int blkdev_zone_mgmt_ioctl(struct block_device *bdev,
618 blk_mode_t mode, unsigned int cmd, unsigned long arg)
619{
620 return -ENOTTY;
621}
622#endif /* CONFIG_BLK_DEV_ZONED */
623
624struct block_device *bdev_alloc(struct gendisk *disk, u8 partno);
625void bdev_add(struct block_device *bdev, dev_t dev);
626void bdev_unhash(struct block_device *bdev);
627void bdev_drop(struct block_device *bdev);
628
629int blk_alloc_ext_minor(void);
630void blk_free_ext_minor(unsigned int minor);
631#define ADDPART_FLAG_NONE 0
632#define ADDPART_FLAG_RAID 1
633#define ADDPART_FLAG_WHOLEDISK 2
634#define ADDPART_FLAG_READONLY 4
635int bdev_add_partition(struct gendisk *disk, int partno, sector_t start,
636 sector_t length);
637int bdev_del_partition(struct gendisk *disk, int partno);
638int bdev_resize_partition(struct gendisk *disk, int partno, sector_t start,
639 sector_t length);
640void drop_partition(struct block_device *part);
641
642void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors);
643
644struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id,
645 struct lock_class_key *lkclass);
646struct request_queue *blk_alloc_queue(struct queue_limits *lim, int node_id);
647
648int disk_scan_partitions(struct gendisk *disk, blk_mode_t mode);
649
650int disk_alloc_events(struct gendisk *disk);
651void disk_add_events(struct gendisk *disk);
652void disk_del_events(struct gendisk *disk);
653void disk_release_events(struct gendisk *disk);
654void disk_block_events(struct gendisk *disk);
655void disk_unblock_events(struct gendisk *disk);
656void disk_flush_events(struct gendisk *disk, unsigned int mask);
657extern struct device_attribute dev_attr_events;
658extern struct device_attribute dev_attr_events_async;
659extern struct device_attribute dev_attr_events_poll_msecs;
660
661extern struct attribute_group blk_trace_attr_group;
662
663blk_mode_t file_to_blk_mode(struct file *file);
664int truncate_bdev_range(struct block_device *bdev, blk_mode_t mode,
665 loff_t lstart, loff_t lend);
666long blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg);
667int blkdev_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags);
668long compat_blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg);
669
670extern const struct address_space_operations def_blk_aops;
671
672int disk_register_independent_access_ranges(struct gendisk *disk);
673void disk_unregister_independent_access_ranges(struct gendisk *disk);
674
675int should_fail_bio(struct bio *bio);
676#ifdef CONFIG_FAIL_MAKE_REQUEST
677bool should_fail_request(struct block_device *part, unsigned int bytes);
678#else /* CONFIG_FAIL_MAKE_REQUEST */
679static inline bool should_fail_request(struct block_device *part,
680 unsigned int bytes)
681{
682 return false;
683}
684#endif /* CONFIG_FAIL_MAKE_REQUEST */
685
686/*
687 * Optimized request reference counting. Ideally we'd make timeouts be more
688 * clever, as that's the only reason we need references at all... But until
689 * this happens, this is faster than using refcount_t. Also see:
690 *
691 * abc54d634334 ("io_uring: switch to atomic_t for io_kiocb reference count")
692 */
693#define req_ref_zero_or_close_to_overflow(req) \
694 ((unsigned int) atomic_read(&(req->ref)) + 127u <= 127u)
695
696static inline bool req_ref_inc_not_zero(struct request *req)
697{
698 return atomic_inc_not_zero(&req->ref);
699}
700
701static inline bool req_ref_put_and_test(struct request *req)
702{
703 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
704 return atomic_dec_and_test(&req->ref);
705}
706
707static inline void req_ref_set(struct request *req, int value)
708{
709 atomic_set(&req->ref, value);
710}
711
712static inline int req_ref_read(struct request *req)
713{
714 return atomic_read(&req->ref);
715}
716
717static inline u64 blk_time_get_ns(void)
718{
719 struct blk_plug *plug = current->plug;
720 u64 now;
721
722 if (!plug || !in_task())
723 return ktime_get_ns();
724
725 /*
726 * 0 could very well be a valid time, but rather than flag "this is
727 * a valid timestamp" separately, just accept that we'll do an extra
728 * ktime_get_ns() if we just happen to get 0 as the current time.
729 *
730 * cur_ktime can be zeroed by pre-emption the moment PF_BLOCK_TS is set.
731 */
732 now = READ_ONCE(plug->cur_ktime);
733 if (!now) {
734 now = ktime_get_ns();
735 WRITE_ONCE(plug->cur_ktime, now);
736 /* Ensure PF_BLOCK_TS is set after cur_ktime. */
737 barrier();
738 current->flags |= PF_BLOCK_TS;
739 }
740 return now;
741}
742
743static inline ktime_t blk_time_get(void)
744{
745 return ns_to_ktime(blk_time_get_ns());
746}
747
748void bdev_release(struct file *bdev_file);
749int bdev_open(struct block_device *bdev, blk_mode_t mode, void *holder,
750 const struct blk_holder_ops *hops, struct file *bdev_file);
751int bdev_permission(dev_t dev, blk_mode_t mode, void *holder);
752
753void bio_integrity_generate(struct bio *bio);
754blk_status_t bio_integrity_verify(struct bio *bio,
755 struct bvec_iter *saved_iter);
756
757void blk_integrity_prepare(struct request *rq);
758void blk_integrity_complete(struct request *rq, unsigned int nr_bytes);
759
760#ifdef CONFIG_LOCKDEP
761static inline void blk_freeze_acquire_lock(struct request_queue *q)
762{
763 if (!q->mq_freeze_disk_dead)
764 rwsem_acquire(&q->io_lockdep_map, 0, 1, _RET_IP_);
765 if (!q->mq_freeze_queue_dying)
766 rwsem_acquire(&q->q_lockdep_map, 0, 1, _RET_IP_);
767}
768
769static inline void blk_unfreeze_release_lock(struct request_queue *q)
770{
771 if (!q->mq_freeze_queue_dying)
772 rwsem_release(&q->q_lockdep_map, _RET_IP_);
773 if (!q->mq_freeze_disk_dead)
774 rwsem_release(&q->io_lockdep_map, _RET_IP_);
775}
776#else
777static inline void blk_freeze_acquire_lock(struct request_queue *q)
778{
779}
780static inline void blk_unfreeze_release_lock(struct request_queue *q)
781{
782}
783#endif
784
785/*
786 * debugfs directory and file creation can trigger fs reclaim, which can enter
787 * back into the block layer request_queue. This can cause deadlock if the
788 * queue is frozen. Use NOIO context together with debugfs_mutex to prevent fs
789 * reclaim from triggering block I/O.
790 */
791static inline void blk_debugfs_lock_nomemsave(struct request_queue *q)
792 __acquires(&q->debugfs_mutex)
793{
794 mutex_lock(&q->debugfs_mutex);
795}
796
797static inline void blk_debugfs_unlock_nomemrestore(struct request_queue *q)
798 __releases(&q->debugfs_mutex)
799{
800 mutex_unlock(&q->debugfs_mutex);
801}
802
803static inline unsigned int __must_check blk_debugfs_lock(struct request_queue *q)
804 __acquires(&q->debugfs_mutex)
805{
806 unsigned int memflags = memalloc_noio_save();
807
808 blk_debugfs_lock_nomemsave(q);
809 return memflags;
810}
811
812static inline void blk_debugfs_unlock(struct request_queue *q,
813 unsigned int memflags)
814 __releases(&q->debugfs_mutex)
815{
816 blk_debugfs_unlock_nomemrestore(q);
817 memalloc_noio_restore(memflags);
818}
819
820#endif /* BLK_INTERNAL_H */