linux - fs/buffer.c

3005 lines · 78.1 KiB

1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* linux/fs/buffer.c
4	*
5	* Copyright (C) 1991, 1992, 2002 Linus Torvalds
6	*/
7
8	/*
9	* Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
10	*
11	* Removed a lot of unnecessary code and simplified things now that
12	* the buffer cache isn't our primary cache - Andrew Tridgell 12/96
13	*
14	* Speed up hash, lru, and free list operations. Use gfp() for allocating
15	* hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
16	*
17	* Added 32k buffer block sizes - these are required older ARM systems. - RMK
18	*
19	* async buffer flushing, 1999 Andrea Arcangeli <[email protected]>
20	*/
21
22	#include <linux/kernel.h>
23	#include <linux/sched/signal.h>
24	#include <linux/syscalls.h>
25	#include <linux/fs.h>
26	#include <linux/iomap.h>
27	#include <linux/mm.h>
28	#include <linux/percpu.h>
29	#include <linux/slab.h>
30	#include <linux/capability.h>
31	#include <linux/blkdev.h>
32	#include <linux/blk-crypto.h>
33	#include <linux/file.h>
34	#include <linux/quotaops.h>
35	#include <linux/highmem.h>
36	#include <linux/export.h>
37	#include <linux/backing-dev.h>
38	#include <linux/writeback.h>
39	#include <linux/hash.h>
40	#include <linux/suspend.h>
41	#include <linux/buffer_head.h>
42	#include <linux/task_io_accounting_ops.h>
43	#include <linux/bio.h>
44	#include <linux/cpu.h>
45	#include <linux/bitops.h>
46	#include <linux/mpage.h>
47	#include <linux/bit_spinlock.h>
48	#include <linux/folio_batch.h>
49	#include <linux/sched/mm.h>
50	#include <trace/events/block.h>
51	#include <linux/fscrypt.h>
52	#include <linux/fsverity.h>
53	#include <linux/sched/isolation.h>
54
55	#include "internal.h"
56
57	#define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
58
59	inline void touch_buffer(struct buffer_head *bh)
60	{
61	trace_block_touch_buffer(bh);
62	folio_mark_accessed(bh->b_folio);
63	}
64	EXPORT_SYMBOL(touch_buffer);
65
66	void __lock_buffer(struct buffer_head *bh)
67	{
68	wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
69	}
70	EXPORT_SYMBOL(__lock_buffer);
71
72	void unlock_buffer(struct buffer_head *bh)
73	{
74	clear_and_wake_up_bit(BH_Lock, &bh->b_state);
75	}
76	EXPORT_SYMBOL(unlock_buffer);
77
78	/*
79	* Returns if the folio has dirty or writeback buffers. If all the buffers
80	* are unlocked and clean then the folio_test_dirty information is stale. If
81	* any of the buffers are locked, it is assumed they are locked for IO.
82	*/
83	void buffer_check_dirty_writeback(struct folio *folio,
84	bool dirty, bool writeback)
85	{
86	struct buffer_head head, bh;
87	*dirty = false;
88	*writeback = false;
89
90	BUG_ON(!folio_test_locked(folio));
91
92	head = folio_buffers(folio);
93	if (!head)
94	return;
95
96	if (folio_test_writeback(folio))
97	*writeback = true;
98
99	bh = head;
100	do {
101	if (buffer_locked(bh))
102	*writeback = true;
103
104	if (buffer_dirty(bh))
105	*dirty = true;
106
107	bh = bh->b_this_page;
108	} while (bh != head);
109	}
110
111	/*
112	* Block until a buffer comes unlocked. This doesn't stop it
113	* from becoming locked again - you have to lock it yourself
114	* if you want to preserve its state.
115	*/
116	void __wait_on_buffer(struct buffer_head * bh)
117	{
118	wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
119	}
120	EXPORT_SYMBOL(__wait_on_buffer);
121
122	static void buffer_io_error(struct buffer_head bh, char msg)
123	{
124	if (!test_bit(BH_Quiet, &bh->b_state))
125	printk_ratelimited(KERN_ERR
126	"Buffer I/O error on dev %pg, logical block %llu%s\n",
127	bh->b_bdev, (unsigned long long)bh->b_blocknr, msg);
128	}
129
130	/**
131	* bio_endio_bh - Discard the bio used to submit a buffer.
132	* @bio: The bio.
133	* @bhp: Where to return the buffer_head.
134	*
135	* Call this in your bio_end_io handler to retrieve the buffer_head
136	* submitted in bh_submit(). If you did not call bh_submit(), do not
137	* call this function; it will return garbage.
138	*
139	* This function consumes the bio refcount which will probably free the
140	* bio.
141	*
142	* Return: True if the I/O succeeded.
143	*/
144	bool bio_endio_bh(struct bio bio, struct buffer_head *bhp)
145	{
146	bool success = bio->bi_status == BLK_STS_OK;
147	struct buffer_head *bh = bio->bi_private;
148
149	if (unlikely(bio_flagged(bio, BIO_QUIET)))
150	set_bit(BH_Quiet, &bh->b_state);
151	bio_put(bio);
152
153	*bhp = bh;
154	return success;
155	}
156	EXPORT_SYMBOL(bio_endio_bh);
157
158	/**
159	* end_buffer_read_sync - Handle buffer reads finishing
160	* @bh: The buffer.
161	* @uptodate: True if the read was successful.
162	*
163	* If a buffer is read through a mechanism that isn't bh_submit(), you
164	* can call this function to finish the read.
165	*/
166	void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
167	{
168	if (uptodate) {
169	set_buffer_uptodate(bh);
170	} else {
171	/* This happens, due to failed read-ahead attempts. */
172	clear_buffer_uptodate(bh);
173	}
174	unlock_buffer(bh);
175	}
176	EXPORT_SYMBOL(end_buffer_read_sync);
177
178	/**
179	* bh_end_read - I/O end handler for reads
180	* @bio: The bio being completed.
181	*
182	* Pass this function to bh_submit() if you're reading into the buffer,
183	* unless you need your own special I/O end handler.
184	*/
185	void bh_end_read(struct bio *bio)
186	{
187	struct buffer_head *bh;
188	bool uptodate = bio_endio_bh(bio, &bh);
189	end_buffer_read_sync(bh, uptodate);
190	}
191	EXPORT_SYMBOL(bh_end_read);
192
193	/**
194	* bh_end_write - I/O end handler for writes
195	* @bio: The bio being completed.
196	*
197	* Pass this function to bh_submit() if you're writing from the buffer,
198	* unless you need your own special I/O end handler.
199	*/
200	void bh_end_write(struct bio *bio)
201	{
202	struct buffer_head *bh;
203	bool success = bio_endio_bh(bio, &bh);
204
205	if (success) {
206	set_buffer_uptodate(bh);
207	} else {
208	buffer_io_error(bh, ", lost sync page write");
209	mark_buffer_write_io_error(bh);
210	clear_buffer_uptodate(bh);
211	}
212	unlock_buffer(bh);
213	}
214	EXPORT_SYMBOL(bh_end_write);
215
216	static struct buffer_head *
217	__find_get_block_slow(struct block_device *bdev, sector_t block, bool atomic)
218	{
219	struct address_space *bd_mapping = bdev->bd_mapping;
220	const int blkbits = bd_mapping->host->i_blkbits;
221	struct buffer_head *ret = NULL;
222	pgoff_t index;
223	struct buffer_head *bh;
224	struct buffer_head *head;
225	struct folio *folio;
226	int all_mapped = 1;
227	static DEFINE_RATELIMIT_STATE(last_warned, HZ, 1);
228
229	index = ((loff_t)block << blkbits) / PAGE_SIZE;
230	folio = __filemap_get_folio(bd_mapping, index, FGP_ACCESSED, 0);
231	if (IS_ERR(folio))
232	goto out;
233
234	/*
235	* Folio lock protects the buffers. Callers that cannot block
236	* will fallback to serializing vs try_to_free_buffers() via
237	* the i_private_lock.
238	*/
239	if (atomic)
240	spin_lock(&bd_mapping->i_private_lock);
241	else
242	folio_lock(folio);
243
244	head = folio_buffers(folio);
245	if (!head)
246	goto out_unlock;
247	/*
248	* Upon a noref migration, the folio lock serializes here;
249	* otherwise bail.
250	*/
251	if (test_bit_acquire(BH_Migrate, &head->b_state)) {
252	WARN_ON(!atomic);
253	goto out_unlock;
254	}
255
256	bh = head;
257	do {
258	if (!buffer_mapped(bh))
259	all_mapped = 0;
260	else if (bh->b_blocknr == block) {
261	ret = bh;
262	get_bh(bh);
263	goto out_unlock;
264	}
265	bh = bh->b_this_page;
266	} while (bh != head);
267
268	/* we might be here because some of the buffers on this page are
269	* not mapped. This is due to various races between
270	* file io on the block device and getblk. It gets dealt with
271	* elsewhere, don't buffer_error if we had some unmapped buffers
272	*/
273	ratelimit_set_flags(&last_warned, RATELIMIT_MSG_ON_RELEASE);
274	if (all_mapped && __ratelimit(&last_warned)) {
275	printk("__find_get_block_slow() failed. block=%llu, "
276	"b_blocknr=%llu, b_state=0x%08lx, b_size=%zu, "
277	"device %pg blocksize: %d\n",
278	(unsigned long long)block,
279	(unsigned long long)bh->b_blocknr,
280	bh->b_state, bh->b_size, bdev,
281	1 << blkbits);
282	}
283	out_unlock:
284	if (atomic)
285	spin_unlock(&bd_mapping->i_private_lock);
286	else
287	folio_unlock(folio);
288	folio_put(folio);
289	out:
290	return ret;
291	}
292
293	static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
294	{
295	unsigned long flags;
296	struct buffer_head *first;
297	struct buffer_head *tmp;
298	struct folio *folio;
299	int folio_uptodate = 1;
300
301	BUG_ON(!buffer_async_read(bh));
302
303	folio = bh->b_folio;
304	if (uptodate) {
305	set_buffer_uptodate(bh);
306	} else {
307	clear_buffer_uptodate(bh);
308	buffer_io_error(bh, ", async page read");
309	}
310
311	/*
312	* Be _very_ careful from here on. Bad things can happen if
313	* two buffer heads end IO at almost the same time and both
314	* decide that the page is now completely done.
315	*/
316	first = folio_buffers(folio);
317	spin_lock_irqsave(&first->b_uptodate_lock, flags);
318	clear_buffer_async_read(bh);
319	unlock_buffer(bh);
320	tmp = bh;
321	do {
322	if (!buffer_uptodate(tmp))
323	folio_uptodate = 0;
324	if (buffer_async_read(tmp)) {
325	BUG_ON(!buffer_locked(tmp));
326	goto still_busy;
327	}
328	tmp = tmp->b_this_page;
329	} while (tmp != bh);
330	spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
331
332	folio_end_read(folio, folio_uptodate);
333	return;
334
335	still_busy:
336	spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
337	}
338
339	struct verify_bh_ctx {
340	struct work_struct work;
341	struct buffer_head *bh;
342	struct fsverity_info *vi;
343	};
344
345	static void verify_bh(struct work_struct *work)
346	{
347	struct verify_bh_ctx *ctx =
348	container_of(work, struct verify_bh_ctx, work);
349	struct buffer_head *bh = ctx->bh;
350	bool valid;
351
352	valid = fsverity_verify_blocks(ctx->vi, bh->b_folio, bh->b_size,
353	bh_offset(bh));
354	end_buffer_async_read(bh, valid);
355	kfree(ctx);
356	}
357
358	/*
359	* I/O completion handler for block_read_full_folio() - folios
360	* which come unlocked at the end of I/O.
361	*/
362	static void bh_end_async_read(struct bio *bio)
363	{
364	struct buffer_head *bh;
365	bool uptodate = bio_endio_bh(bio, &bh);
366	struct inode *inode = bh->b_folio->mapping->host;
367	struct fsverity_info *vi = NULL;
368
369	/* needed by ext4 */
370	if (bh->b_folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE))
371	vi = fsverity_get_info(inode);
372
373	/* Verify (with fsverity) if needed. */
374	if (vi && uptodate) {
375	struct verify_bh_ctx ctx = kmalloc_obj(ctx, GFP_ATOMIC);
376
377	if (ctx) {
378	ctx->bh = bh;
379	ctx->vi = vi;
380	INIT_WORK(&ctx->work, verify_bh);
381	fsverity_enqueue_verify_work(&ctx->work);
382	return;
383	}
384	uptodate = false;
385	}
386	end_buffer_async_read(bh, uptodate);
387	}
388
389	/**
390	* bh_end_async_write - I/O end handler for async folio writes
391	* @bio: The bio being completed.
392	*
393	* Pass this function to bh_submit() if you're doing the equivalent of
394	* block_write_full_folio(). That is, the folio is unlocked, and will
395	* have its writeback flag cleared once all async write buffers have
396	* completed.
397	*/
398	void bh_end_async_write(struct bio *bio)
399	{
400	struct buffer_head *bh;
401	bool success = bio_endio_bh(bio, &bh);
402	unsigned long flags;
403	struct buffer_head *first;
404	struct buffer_head *tmp;
405	struct folio *folio;
406
407	BUG_ON(!buffer_async_write(bh));
408
409	folio = bh->b_folio;
410	if (success) {
411	set_buffer_uptodate(bh);
412	} else {
413	buffer_io_error(bh, ", lost async page write");
414	mark_buffer_write_io_error(bh);
415	clear_buffer_uptodate(bh);
416	}
417
418	first = folio_buffers(folio);
419	spin_lock_irqsave(&first->b_uptodate_lock, flags);
420
421	clear_buffer_async_write(bh);
422	unlock_buffer(bh);
423	tmp = bh->b_this_page;
424	while (tmp != bh) {
425	if (buffer_async_write(tmp)) {
426	BUG_ON(!buffer_locked(tmp));
427	goto still_busy;
428	}
429	tmp = tmp->b_this_page;
430	}
431	spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
432	folio_end_writeback(folio);
433	return;
434
435	still_busy:
436	spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
437	}
438	EXPORT_SYMBOL(bh_end_async_write);
439
440
441	/*
442	* fs/buffer.c contains helper functions for buffer-backed address space's
443	* fsync functions. A common requirement for buffer-based filesystems is
444	* that certain data from the backing blockdev needs to be written out for
445	* a successful fsync(). For example, ext2 indirect blocks need to be
446	* written back and waited upon before fsync() returns.
447	*
448	* The functions mmb_mark_buffer_dirty(), mmb_sync(), mmb_has_buffers()
449	* and mmb_invalidate() are provided for the management of a list of dependent
450	* buffers in mapping_metadata_bhs struct.
451	*
452	* The locking is a little subtle: The list of buffer heads is protected by
453	* the lock in mapping_metadata_bhs so functions coming from bdev mapping
454	* (such as try_to_free_buffers()) need to safely get to mapping_metadata_bhs
455	* using RCU, grab the lock, verify we didn't race with somebody detaching the
456	* bh / moving it to different inode and only then proceeding.
457	*/
458
459	void mmb_init(struct mapping_metadata_bhs mmb, struct address_space mapping)
460	{
461	spin_lock_init(&mmb->lock);
462	INIT_LIST_HEAD(&mmb->list);
463	mmb->mapping = mapping;
464	}
465	EXPORT_SYMBOL(mmb_init);
466
467	static void __remove_assoc_queue(struct mapping_metadata_bhs *mmb,
468	struct buffer_head *bh)
469	{
470	lockdep_assert_held(&mmb->lock);
471	list_del_init(&bh->b_assoc_buffers);
472	WARN_ON(!bh->b_mmb);
473	bh->b_mmb = NULL;
474	}
475
476	static void remove_assoc_queue(struct buffer_head *bh)
477	{
478	struct mapping_metadata_bhs *mmb;
479
480	/*
481	* The locking dance is ugly here. We need to acquire the lock
482	* protecting the metadata bh list while possibly racing with bh
483	* being removed from the list or moved to a different one. We
484	* use RCU to pin mapping_metadata_bhs in memory to
485	* opportunistically acquire the lock and then recheck the bh
486	* didn't move under us.
487	*/
488	while (bh->b_mmb) {
489	rcu_read_lock();
490	mmb = READ_ONCE(bh->b_mmb);
491	if (mmb) {
492	spin_lock(&mmb->lock);
493	if (bh->b_mmb == mmb)
494	__remove_assoc_queue(mmb, bh);
495	spin_unlock(&mmb->lock);
496	}
497	rcu_read_unlock();
498	}
499	}
500
501	bool mmb_has_buffers(struct mapping_metadata_bhs *mmb)
502	{
503	return !list_empty(&mmb->list);
504	}
505	EXPORT_SYMBOL_GPL(mmb_has_buffers);
506
507	/**
508	* mmb_sync - write out & wait upon all buffers in a list
509	* @mmb: the list of buffers to write
510	*
511	* Starts I/O against the buffers in the given list and waits upon
512	* that I/O. Basically, this is a convenience function for fsync(). @mmb is
513	* for a file or directory which needs those buffers to be written for a
514	* successful fsync().
515	*
516	* We have conflicting pressures: we want to make sure that all
517	* initially dirty buffers get waited on, but that any subsequently
518	* dirtied buffers don't. After all, we don't want fsync to last
519	* forever if somebody is actively writing to the file.
520	*
521	* Do this in two main stages: first we copy dirty buffers to a
522	* temporary inode list, queueing the writes as we go. Then we clean
523	* up, waiting for those writes to complete. mark_buffer_dirty_inode()
524	* doesn't touch b_assoc_buffers list if b_mmb is not NULL so we are sure the
525	* buffer stays on our list until IO completes (at which point it can be
526	* reaped).
527	*/
528	int mmb_sync(struct mapping_metadata_bhs *mmb)
529	{
530	struct buffer_head *bh;
531	int err = 0;
532	struct blk_plug plug;
533	LIST_HEAD(tmp);
534
535	if (!mmb_has_buffers(mmb))
536	return 0;
537
538	blk_start_plug(&plug);
539
540	spin_lock(&mmb->lock);
541	while (!list_empty(&mmb->list)) {
542	bh = BH_ENTRY(mmb->list.next);
543	WARN_ON_ONCE(bh->b_mmb != mmb);
544	__remove_assoc_queue(mmb, bh);
545	/* Avoid race with mark_buffer_dirty_inode() which does
546	* a lockless check and we rely on seeing the dirty bit */
547	smp_mb();
548	if (buffer_dirty(bh) \|\| buffer_locked(bh)) {
549	list_add(&bh->b_assoc_buffers, &tmp);
550	bh->b_mmb = mmb;
551	if (buffer_dirty(bh)) {
552	get_bh(bh);
553	spin_unlock(&mmb->lock);
554	/*
555	* Ensure any pending I/O completes so that
556	* write_dirty_buffer() actually writes the
557	* current contents - it is a noop if I/O is
558	* still in flight on potentially older
559	* contents.
560	*/
561	write_dirty_buffer(bh, REQ_SYNC);
562
563	/*
564	* Kick off IO for the previous mapping. Note
565	* that we will not run the very last mapping,
566	* wait_on_buffer() will do that for us
567	* through sync_buffer().
568	*/
569	brelse(bh);
570	spin_lock(&mmb->lock);
571	}
572	}
573	}
574
575	spin_unlock(&mmb->lock);
576	blk_finish_plug(&plug);
577	spin_lock(&mmb->lock);
578
579	while (!list_empty(&tmp)) {
580	bh = BH_ENTRY(tmp.prev);
581	get_bh(bh);
582	__remove_assoc_queue(mmb, bh);
583	/* Avoid race with mark_buffer_dirty_inode() which does
584	* a lockless check and we rely on seeing the dirty bit */
585	smp_mb();
586	if (buffer_dirty(bh)) {
587	list_add(&bh->b_assoc_buffers, &mmb->list);
588	bh->b_mmb = mmb;
589	}
590	spin_unlock(&mmb->lock);
591	wait_on_buffer(bh);
592	if (!buffer_uptodate(bh))
593	err = -EIO;
594	brelse(bh);
595	spin_lock(&mmb->lock);
596	}
597	spin_unlock(&mmb->lock);
598	return err;
599	}
600	EXPORT_SYMBOL(mmb_sync);
601
602	/*
603	* Called when we've recently written block `bblock', and it is known that
604	* `bblock' was for a buffer_boundary() buffer. This means that the block at
605	* `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
606	* dirty, schedule it for IO. So that indirects merge nicely with their data.
607	*/
608	void write_boundary_block(struct block_device *bdev,
609	sector_t bblock, unsigned blocksize)
610	{
611	struct buffer_head *bh;
612
613	bh = __find_get_block_nonatomic(bdev, bblock + 1, blocksize);
614	if (bh) {
615	if (buffer_dirty(bh))
616	write_dirty_buffer(bh, 0);
617	put_bh(bh);
618	}
619	}
620
621	void mmb_mark_buffer_dirty(struct buffer_head *bh,
622	struct mapping_metadata_bhs *mmb)
623	{
624	mark_buffer_dirty(bh);
625	if (!bh->b_mmb) {
626	spin_lock(&mmb->lock);
627	/*
628	* For a corrupted filesystem with multiply claimed blocks this
629	* can fail. Avoid corrupting the linked list in that case.
630	*/
631	if (cmpxchg(&bh->b_mmb, NULL, mmb) != NULL) {
632	spin_unlock(&mmb->lock);
633	return;
634	}
635	list_move_tail(&bh->b_assoc_buffers, &mmb->list);
636	spin_unlock(&mmb->lock);
637	}
638	}
639	EXPORT_SYMBOL(mmb_mark_buffer_dirty);
640
641	/**
642	* block_dirty_folio - Mark a folio as dirty.
643	* @mapping: The address space containing this folio.
644	* @folio: The folio to mark dirty.
645	*
646	* Filesystems which use buffer_heads can use this function as their
647	* ->dirty_folio implementation. Some filesystems need to do a little
648	* work before calling this function. Filesystems which do not use
649	* buffer_heads should call filemap_dirty_folio() instead.
650	*
651	* If the folio has buffers, the uptodate buffers are set dirty, to
652	* preserve dirty-state coherency between the folio and the buffers.
653	* Buffers added to a dirty folio are created dirty.
654	*
655	* The buffers are dirtied before the folio is dirtied. There's a small
656	* race window in which writeback may see the folio cleanness but not the
657	* buffer dirtiness. That's fine. If this code were to set the folio
658	* dirty before the buffers, writeback could clear the folio dirty flag,
659	* see a bunch of clean buffers and we'd end up with dirty buffers/clean
660	* folio on the dirty folio list.
661	*
662	* We use i_private_lock to lock against try_to_free_buffers() while
663	* using the folio's buffer list. This also prevents clean buffers
664	* being added to the folio after it was set dirty.
665	*
666	* Context: May only be called from process context. Does not sleep.
667	* Caller must ensure that @folio cannot be truncated during this call,
668	* typically by holding the folio lock or having a page in the folio
669	* mapped and holding the page table lock.
670	*
671	* Return: True if the folio was dirtied; false if it was already dirtied.
672	*/
673	bool block_dirty_folio(struct address_space mapping, struct folio folio)
674	{
675	struct buffer_head *head;
676	bool newly_dirty;
677
678	spin_lock(&mapping->i_private_lock);
679	head = folio_buffers(folio);
680	if (head) {
681	struct buffer_head *bh = head;
682
683	do {
684	set_buffer_dirty(bh);
685	bh = bh->b_this_page;
686	} while (bh != head);
687	}
688	/*
689	* Lock out page's memcg migration to keep PageDirty
690	* synchronized with per-memcg dirty page counters.
691	*/
692	newly_dirty = !folio_test_set_dirty(folio);
693	spin_unlock(&mapping->i_private_lock);
694
695	if (newly_dirty)
696	__folio_mark_dirty(folio, mapping, 1);
697
698	if (newly_dirty)
699	__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
700
701	return newly_dirty;
702	}
703	EXPORT_SYMBOL(block_dirty_folio);
704
705	/*
706	* Invalidate any and all dirty buffers on a given buffers list. We are
707	* probably unmounting the fs, but that doesn't mean we have already
708	* done a sync(). Just drop the buffers from the inode list.
709	*/
710	void mmb_invalidate(struct mapping_metadata_bhs *mmb)
711	{
712	if (mmb_has_buffers(mmb)) {
713	spin_lock(&mmb->lock);
714	while (!list_empty(&mmb->list))
715	__remove_assoc_queue(mmb, BH_ENTRY(mmb->list.next));
716	spin_unlock(&mmb->lock);
717	}
718	}
719	EXPORT_SYMBOL(mmb_invalidate);
720
721	/*
722	* Create the appropriate buffers when given a folio for data area and
723	* the size of each buffer.. Use the bh->b_this_page linked list to
724	* follow the buffers created. Return NULL if unable to create more
725	* buffers.
726	*
727	* The retry flag is used to differentiate async IO (paging, swapping)
728	* which may not fail from ordinary buffer allocations.
729	*/
730	struct buffer_head folio_alloc_buffers(struct folio folio, unsigned long size,
731	gfp_t gfp)
732	{
733	struct buffer_head bh, head;
734	long offset;
735	struct mem_cgroup memcg, old_memcg;
736
737	memcg = get_mem_cgroup_from_folio(folio);
738	old_memcg = set_active_memcg(memcg);
739
740	head = NULL;
741	offset = folio_size(folio);
742	while ((offset -= size) >= 0) {
743	bh = alloc_buffer_head(gfp);
744	if (!bh)
745	goto no_grow;
746
747	bh->b_this_page = head;
748	bh->b_blocknr = -1;
749	head = bh;
750
751	bh->b_size = size;
752
753	/* Link the buffer to its folio */
754	folio_set_bh(bh, folio, offset);
755	}
756	out:
757	set_active_memcg(old_memcg);
758	mem_cgroup_put(memcg);
759	return head;
760	/*
761	* In case anything failed, we just free everything we got.
762	*/
763	no_grow:
764	if (head) {
765	do {
766	bh = head;
767	head = head->b_this_page;
768	free_buffer_head(bh);
769	} while (head);
770	}
771
772	goto out;
773	}
774	EXPORT_SYMBOL_GPL(folio_alloc_buffers);
775
776	struct buffer_head alloc_page_buffers(struct page page, unsigned long size)
777	{
778	gfp_t gfp = GFP_NOFS \| __GFP_ACCOUNT;
779
780	return folio_alloc_buffers(page_folio(page), size, gfp);
781	}
782	EXPORT_SYMBOL_GPL(alloc_page_buffers);
783
784	static inline void link_dev_buffers(struct folio *folio,
785	struct buffer_head *head)
786	{
787	struct buffer_head bh, tail;
788
789	bh = head;
790	do {
791	tail = bh;
792	bh = bh->b_this_page;
793	} while (bh);
794	tail->b_this_page = head;
795	folio_attach_private(folio, head);
796	}
797
798	static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
799	{
800	sector_t retval = ~((sector_t)0);
801	loff_t sz = bdev_nr_bytes(bdev);
802
803	if (sz) {
804	unsigned int sizebits = blksize_bits(size);
805	retval = (sz >> sizebits);
806	}
807	return retval;
808	}
809
810	/*
811	* Initialise the state of a blockdev folio's buffers.
812	*/
813	static sector_t folio_init_buffers(struct folio *folio,
814	struct block_device *bdev, unsigned size)
815	{
816	struct buffer_head *head = folio_buffers(folio);
817	struct buffer_head *bh = head;
818	bool uptodate = folio_test_uptodate(folio);
819	sector_t block = div_u64(folio_pos(folio), size);
820	sector_t end_block = blkdev_max_block(bdev, size);
821
822	do {
823	if (!buffer_mapped(bh)) {
824	bh->b_private = NULL;
825	bh->b_bdev = bdev;
826	bh->b_blocknr = block;
827	if (uptodate)
828	set_buffer_uptodate(bh);
829	if (block < end_block)
830	set_buffer_mapped(bh);
831	}
832	block++;
833	bh = bh->b_this_page;
834	} while (bh != head);
835
836	/*
837	* Caller needs to validate requested block against end of device.
838	*/
839	return end_block;
840	}
841
842	/*
843	* Create the page-cache folio that contains the requested block.
844	*
845	* This is used purely for blockdev mappings.
846	*
847	* Returns false if we have a failure which cannot be cured by retrying
848	* without sleeping. Returns true if we succeeded, or the caller should retry.
849	*/
850	static bool grow_dev_folio(struct block_device *bdev, sector_t block,
851	pgoff_t index, unsigned size, gfp_t gfp)
852	{
853	struct address_space *mapping = bdev->bd_mapping;
854	struct folio *folio;
855	struct buffer_head *bh;
856	sector_t end_block = 0;
857
858	folio = __filemap_get_folio(mapping, index,
859	FGP_LOCK \| FGP_ACCESSED \| FGP_CREAT, gfp);
860	if (IS_ERR(folio))
861	return false;
862
863	bh = folio_buffers(folio);
864	if (bh) {
865	if (bh->b_size == size) {
866	end_block = folio_init_buffers(folio, bdev, size);
867	goto unlock;
868	}
869
870	/*
871	* Retrying may succeed; for example the folio may finish
872	* writeback, or buffers may be cleaned. This should not
873	* happen very often; maybe we have old buffers attached to
874	* this blockdev's page cache and we're trying to change
875	* the block size?
876	*/
877	if (!try_to_free_buffers(folio)) {
878	end_block = ~0ULL;
879	goto unlock;
880	}
881	}
882
883	bh = folio_alloc_buffers(folio, size, gfp \| __GFP_ACCOUNT);
884	if (!bh)
885	goto unlock;
886
887	/*
888	* Link the folio to the buffers and initialise them. Take the
889	* lock to be atomic wrt __find_get_block(), which does not
890	* run under the folio lock.
891	*/
892	spin_lock(&mapping->i_private_lock);
893	link_dev_buffers(folio, bh);
894	end_block = folio_init_buffers(folio, bdev, size);
895	spin_unlock(&mapping->i_private_lock);
896	unlock:
897	folio_unlock(folio);
898	folio_put(folio);
899	return block < end_block;
900	}
901
902	/*
903	* Create buffers for the specified block device block's folio. If
904	* that folio was dirty, the buffers are set dirty also. Returns false
905	* if we've hit a permanent error.
906	*/
907	static bool grow_buffers(struct block_device *bdev, sector_t block,
908	unsigned size, gfp_t gfp)
909	{
910	loff_t pos;
911
912	/*
913	* Check for a block which lies outside our maximum possible
914	* pagecache index.
915	*/
916	if (check_mul_overflow(block, (sector_t)size, &pos) \|\| pos > MAX_LFS_FILESIZE) {
917	printk(KERN_ERR "%s: requested out-of-range block %llu for device %pg\n",
918	__func__, (unsigned long long)block,
919	bdev);
920	return false;
921	}
922
923	/* Create a folio with the proper size buffers */
924	return grow_dev_folio(bdev, block, pos / PAGE_SIZE, size, gfp);
925	}
926
927	static struct buffer_head *
928	__getblk_slow(struct block_device *bdev, sector_t block,
929	unsigned size, gfp_t gfp)
930	{
931	bool blocking = gfpflags_allow_blocking(gfp);
932
933	if (WARN_ON_ONCE(!IS_ALIGNED(size, bdev_logical_block_size(bdev)))) {
934	printk(KERN_ERR "getblk(): block size %d not aligned to logical block size %d\n",
935	size, bdev_logical_block_size(bdev));
936	return NULL;
937	}
938
939	for (;;) {
940	struct buffer_head *bh;
941
942	if (!grow_buffers(bdev, block, size, gfp))
943	return NULL;
944
945	if (blocking)
946	bh = __find_get_block_nonatomic(bdev, block, size);
947	else
948	bh = __find_get_block(bdev, block, size);
949	if (bh)
950	return bh;
951	}
952	}
953
954	/*
955	* The relationship between dirty buffers and dirty pages:
956	*
957	* Whenever a page has any dirty buffers, the page's dirty bit is set, and
958	* the page is tagged dirty in the page cache.
959	*
960	* At all times, the dirtiness of the buffers represents the dirtiness of
961	* subsections of the page. If the page has buffers, the page dirty bit is
962	* merely a hint about the true dirty state.
963	*
964	* When a page is set dirty in its entirety, all its buffers are marked dirty
965	* (if the page has buffers).
966	*
967	* When a buffer is marked dirty, its page is dirtied, but the page's other
968	* buffers are not.
969	*
970	* Also. When blockdev buffers are explicitly read with bread(), they
971	* individually become uptodate. But their backing page remains not
972	* uptodate - even if all of its buffers are uptodate. A subsequent
973	* block_read_full_folio() against that folio will discover all the uptodate
974	* buffers, will set the folio uptodate and will perform no I/O.
975	*/
976
977	/**
978	* mark_buffer_dirty - mark a buffer_head as needing writeout
979	* @bh: the buffer_head to mark dirty
980	*
981	* mark_buffer_dirty() will set the dirty bit against the buffer, then set
982	* its backing page dirty, then tag the page as dirty in the page cache
983	* and then attach the address_space's inode to its superblock's dirty
984	* inode list.
985	*
986	* mark_buffer_dirty() is atomic. It takes bh->b_folio->mapping->i_private_lock,
987	* i_pages lock and mapping->host->i_lock.
988	*/
989	void mark_buffer_dirty(struct buffer_head *bh)
990	{
991	WARN_ON_ONCE(!buffer_uptodate(bh));
992
993	trace_block_dirty_buffer(bh);
994
995	/*
996	* Very carefully optimize the it-is-already-dirty case.
997	*
998	* Don't let the final "is it dirty" escape to before we
999	* perhaps modified the buffer.
1000	*/
1001	if (buffer_dirty(bh)) {
1002	smp_mb();
1003	if (buffer_dirty(bh))
1004	return;
1005	}
1006
1007	if (!test_set_buffer_dirty(bh)) {
1008	struct folio *folio = bh->b_folio;
1009	struct address_space *mapping = NULL;
1010
1011	if (!folio_test_set_dirty(folio)) {
1012	mapping = folio->mapping;
1013	if (mapping)
1014	__folio_mark_dirty(folio, mapping, 0);
1015	}
1016	if (mapping)
1017	__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
1018	}
1019	}
1020	EXPORT_SYMBOL(mark_buffer_dirty);
1021
1022	void mark_buffer_write_io_error(struct buffer_head *bh)
1023	{
1024	struct mapping_metadata_bhs *mmb;
1025
1026	set_buffer_write_io_error(bh);
1027	/* FIXME: do we need to set this in both places? */
1028	if (bh->b_folio && bh->b_folio->mapping)
1029	mapping_set_error(bh->b_folio->mapping, -EIO);
1030	/* Protect us from mmb & inode getting freed while we work on it */
1031	rcu_read_lock();
1032	mmb = READ_ONCE(bh->b_mmb);
1033	if (mmb)
1034	mapping_set_error(mmb->mapping, -EIO);
1035	rcu_read_unlock();
1036	}
1037	EXPORT_SYMBOL(mark_buffer_write_io_error);
1038
1039	/**
1040	* __brelse - Release a buffer.
1041	* @bh: The buffer to release.
1042	*
1043	* This variant of brelse() can be called if @bh is guaranteed to not be NULL.
1044	*/
1045	void __brelse(struct buffer_head *bh)
1046	{
1047	if (atomic_read(&bh->b_count)) {
1048	put_bh(bh);
1049	return;
1050	}
1051	WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1052	}
1053	EXPORT_SYMBOL(__brelse);
1054
1055	/**
1056	* __bforget - Discard any dirty data in a buffer.
1057	* @bh: The buffer to forget.
1058	*
1059	* This variant of bforget() can be called if @bh is guaranteed to not
1060	* be NULL.
1061	*/
1062	void __bforget(struct buffer_head *bh)
1063	{
1064	clear_buffer_dirty(bh);
1065	remove_assoc_queue(bh);
1066	__brelse(bh);
1067	}
1068	EXPORT_SYMBOL(__bforget);
1069
1070	static void buffer_set_crypto_ctx(struct bio bio, const struct buffer_head bh,
1071	gfp_t gfp_mask)
1072	{
1073	const struct address_space *mapping = folio_mapping(bh->b_folio);
1074
1075	/*
1076	* The ext4 journal (jbd2) can submit a buffer_head it directly created
1077	* for a non-pagecache page. fscrypt doesn't care about these.
1078	*/
1079	if (!mapping)
1080	return;
1081	fscrypt_set_bio_crypt_ctx(bio, mapping->host,
1082	folio_pos(bh->b_folio) + bh_offset(bh), gfp_mask);
1083	}
1084
1085	static void __bh_submit(struct buffer_head *bh, blk_opf_t opf,
1086	enum rw_hint write_hint, struct writeback_control *wbc,
1087	bio_end_io_t end_bio)
1088	{
1089	const enum req_op op = opf & REQ_OP_MASK;
1090	struct bio *bio;
1091
1092	BUG_ON(!buffer_locked(bh));
1093	BUG_ON(!buffer_mapped(bh));
1094	BUG_ON(buffer_delay(bh));
1095	BUG_ON(buffer_unwritten(bh));
1096
1097	/*
1098	* Only clear out a write error when rewriting
1099	*/
1100	if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE))
1101	clear_buffer_write_io_error(bh);
1102
1103	if (buffer_meta(bh))
1104	opf \|= REQ_META;
1105	if (buffer_prio(bh))
1106	opf \|= REQ_PRIO;
1107
1108	bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO);
1109
1110	if (folio_test_dropbehind(bh->b_folio) && op_is_write(opf))
1111	bio_set_flag(bio, BIO_COMPLETE_IN_TASK);
1112
1113	if (IS_ENABLED(CONFIG_FS_ENCRYPTION))
1114	buffer_set_crypto_ctx(bio, bh, GFP_NOIO);
1115
1116	bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
1117	bio->bi_write_hint = write_hint;
1118
1119	bio_add_folio_nofail(bio, bh->b_folio, bh->b_size, bh_offset(bh));
1120
1121	bio->bi_end_io = end_bio;
1122	bio->bi_private = bh;
1123
1124	/* Take care of bh's that straddle the end of the device */
1125	guard_bio_eod(bio);
1126
1127	if (wbc) {
1128	wbc_init_bio(wbc, bio);
1129	wbc_account_cgroup_owner(wbc, bh->b_folio, bh->b_size);
1130	}
1131
1132	blk_crypto_submit_bio(bio);
1133	}
1134
1135	/**
1136	* bh_submit - Start I/O against a buffer head
1137	* @bh: The buffer head to perform I/O on.
1138	* @opf: Operation and flags for bio.
1139	* @end_io: The routine to call when I/O has completed.
1140	*
1141	* If you need to do I/O on an individual bh (instead of allowing the
1142	* page cache to do I/O on the folio that it is in), call this function.
1143	*/
1144	void bh_submit(struct buffer_head *bh, blk_opf_t opf, bio_end_io_t end_io)
1145	{
1146	__bh_submit(bh, opf, WRITE_LIFE_NOT_SET, NULL, end_io);
1147	}
1148	EXPORT_SYMBOL(bh_submit);
1149
1150	static struct buffer_head __bread_slow(struct buffer_head bh)
1151	{
1152	lock_buffer(bh);
1153	if (buffer_uptodate(bh)) {
1154	unlock_buffer(bh);
1155	return bh;
1156	} else {
1157	bh_submit(bh, REQ_OP_READ, bh_end_read);
1158	wait_on_buffer(bh);
1159	if (buffer_uptodate(bh))
1160	return bh;
1161	}
1162	brelse(bh);
1163	return NULL;
1164	}
1165
1166	/*
1167	* Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
1168	* The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
1169	* refcount elevated by one when they're in an LRU. A buffer can only appear
1170	* once in a particular CPU's LRU. A single buffer can be present in multiple
1171	* CPU's LRUs at the same time.
1172	*
1173	* This is a transparent caching front-end to sb_bread(), sb_getblk() and
1174	* sb_find_get_block().
1175	*
1176	* The LRUs themselves only need locking against invalidate_bh_lrus. We use
1177	* a local interrupt disable for that.
1178	*/
1179
1180	#define BH_LRU_SIZE 16
1181
1182	struct bh_lru {
1183	struct buffer_head *bhs[BH_LRU_SIZE];
1184	};
1185
1186	static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1187
1188	#ifdef CONFIG_SMP
1189	#define bh_lru_lock() local_irq_disable()
1190	#define bh_lru_unlock() local_irq_enable()
1191	#else
1192	#define bh_lru_lock() preempt_disable()
1193	#define bh_lru_unlock() preempt_enable()
1194	#endif
1195
1196	static inline void check_irqs_on(void)
1197	{
1198	#ifdef irqs_disabled
1199	BUG_ON(irqs_disabled());
1200	#endif
1201	}
1202
1203	/*
1204	* Install a buffer_head into this cpu's LRU. If not already in the LRU, it is
1205	* inserted at the front, and the buffer_head at the back if any is evicted.
1206	* Or, if already in the LRU it is moved to the front.
1207	*/
1208	static void bh_lru_install(struct buffer_head *bh)
1209	{
1210	struct buffer_head *evictee = bh;
1211	struct bh_lru *b;
1212	int i;
1213
1214	check_irqs_on();
1215	bh_lru_lock();
1216
1217	/*
1218	* the refcount of buffer_head in bh_lru prevents dropping the
1219	* attached page(i.e., try_to_free_buffers) so it could cause
1220	* failing page migration.
1221	* Skip putting upcoming bh into bh_lru until migration is done.
1222	*/
1223	if (lru_cache_disabled() \|\| cpu_is_isolated(smp_processor_id())) {
1224	bh_lru_unlock();
1225	return;
1226	}
1227
1228	b = this_cpu_ptr(&bh_lrus);
1229	for (i = 0; i < BH_LRU_SIZE; i++) {
1230	swap(evictee, b->bhs[i]);
1231	if (evictee == bh) {
1232	bh_lru_unlock();
1233	return;
1234	}
1235	}
1236
1237	get_bh(bh);
1238	bh_lru_unlock();
1239	brelse(evictee);
1240	}
1241
1242	/*
1243	* Look up the bh in this cpu's LRU. If it's there, move it to the head.
1244	*/
1245	static struct buffer_head *
1246	lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
1247	{
1248	struct buffer_head *ret = NULL;
1249	unsigned int i;
1250
1251	check_irqs_on();
1252	bh_lru_lock();
1253	if (cpu_is_isolated(smp_processor_id())) {
1254	bh_lru_unlock();
1255	return NULL;
1256	}
1257	for (i = 0; i < BH_LRU_SIZE; i++) {
1258	struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
1259
1260	if (bh && bh->b_blocknr == block && bh->b_bdev == bdev &&
1261	bh->b_size == size) {
1262	if (i) {
1263	while (i) {
1264	__this_cpu_write(bh_lrus.bhs[i],
1265	__this_cpu_read(bh_lrus.bhs[i - 1]));
1266	i--;
1267	}
1268	__this_cpu_write(bh_lrus.bhs[0], bh);
1269	}
1270	get_bh(bh);
1271	ret = bh;
1272	break;
1273	}
1274	}
1275	bh_lru_unlock();
1276	return ret;
1277	}
1278
1279	/*
1280	* Perform a pagecache lookup for the matching buffer. If it's there, refresh
1281	* it in the LRU and mark it as accessed. If it is not present then return
1282	* NULL. Atomic context callers may also return NULL if the buffer is being
1283	* migrated; similarly the page is not marked accessed either.
1284	*/
1285	static struct buffer_head *
1286	find_get_block_common(struct block_device *bdev, sector_t block,
1287	unsigned size, bool atomic)
1288	{
1289	struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1290
1291	if (bh == NULL) {
1292	/* __find_get_block_slow will mark the page accessed */
1293	bh = __find_get_block_slow(bdev, block, atomic);
1294	if (bh)
1295	bh_lru_install(bh);
1296	} else
1297	touch_buffer(bh);
1298
1299	return bh;
1300	}
1301
1302	struct buffer_head *
1303	__find_get_block(struct block_device *bdev, sector_t block, unsigned size)
1304	{
1305	return find_get_block_common(bdev, block, size, true);
1306	}
1307	EXPORT_SYMBOL(__find_get_block);
1308
1309	/* same as __find_get_block() but allows sleeping contexts */
1310	struct buffer_head *
1311	__find_get_block_nonatomic(struct block_device *bdev, sector_t block,
1312	unsigned size)
1313	{
1314	return find_get_block_common(bdev, block, size, false);
1315	}
1316	EXPORT_SYMBOL(__find_get_block_nonatomic);
1317
1318	/**
1319	* bdev_getblk - Get a buffer_head in a block device's buffer cache.
1320	* @bdev: The block device.
1321	* @block: The block number.
1322	* @size: The size of buffer_heads for this @bdev.
1323	* @gfp: The memory allocation flags to use.
1324	*
1325	* The returned buffer head has its reference count incremented, but is
1326	* not locked. The caller should call brelse() when it has finished
1327	* with the buffer. The buffer may not be uptodate. If needed, the
1328	* caller can bring it uptodate either by reading it or overwriting it.
1329	*
1330	* Return: The buffer head, or NULL if memory could not be allocated.
1331	*/
1332	struct buffer_head bdev_getblk(struct block_device bdev, sector_t block,
1333	unsigned size, gfp_t gfp)
1334	{
1335	struct buffer_head *bh;
1336
1337	if (gfpflags_allow_blocking(gfp))
1338	bh = __find_get_block_nonatomic(bdev, block, size);
1339	else
1340	bh = __find_get_block(bdev, block, size);
1341
1342	might_alloc(gfp);
1343	if (bh)
1344	return bh;
1345
1346	return __getblk_slow(bdev, block, size, gfp);
1347	}
1348	EXPORT_SYMBOL(bdev_getblk);
1349
1350	/*
1351	* Do async read-ahead on a buffer..
1352	*/
1353	void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
1354	{
1355	struct buffer_head *bh = bdev_getblk(bdev, block, size,
1356	GFP_NOWAIT \| __GFP_MOVABLE);
1357
1358	if (likely(bh)) {
1359	bh_readahead(bh, REQ_RAHEAD);
1360	brelse(bh);
1361	}
1362	}
1363	EXPORT_SYMBOL(__breadahead);
1364
1365	/**
1366	* __bread_gfp() - Read a block.
1367	* @bdev: The block device to read from.
1368	* @block: Block number in units of block size.
1369	* @size: The block size of this device in bytes.
1370	* @gfp: Not page allocation flags; see below.
1371	*
1372	* You are not expected to call this function. You should use one of
1373	* sb_bread(), sb_bread_unmovable() or __bread().
1374	*
1375	* Read a specified block, and return the buffer head that refers to it.
1376	* If @gfp is 0, the memory will be allocated using the block device's
1377	* default GFP flags. If @gfp is __GFP_MOVABLE, the memory may be
1378	* allocated from a movable area. Do not pass in a complete set of
1379	* GFP flags.
1380	*
1381	* The returned buffer head has its refcount increased. The caller should
1382	* call brelse() when it has finished with the buffer.
1383	*
1384	* Context: May sleep waiting for I/O.
1385	* Return: NULL if the block was unreadable.
1386	*/
1387	struct buffer_head __bread_gfp(struct block_device bdev, sector_t block,
1388	unsigned size, gfp_t gfp)
1389	{
1390	struct buffer_head *bh;
1391
1392	gfp \|= mapping_gfp_constraint(bdev->bd_mapping, ~__GFP_FS);
1393
1394	/*
1395	* Prefer looping in the allocator rather than here, at least that
1396	* code knows what it's doing.
1397	*/
1398	gfp \|= __GFP_NOFAIL;
1399
1400	bh = bdev_getblk(bdev, block, size, gfp);
1401
1402	if (likely(bh) && !buffer_uptodate(bh))
1403	bh = __bread_slow(bh);
1404	return bh;
1405	}
1406	EXPORT_SYMBOL(__bread_gfp);
1407
1408	static void __invalidate_bh_lrus(struct bh_lru *b)
1409	{
1410	int i;
1411
1412	for (i = 0; i < BH_LRU_SIZE; i++) {
1413	brelse(b->bhs[i]);
1414	b->bhs[i] = NULL;
1415	}
1416	}
1417	/*
1418	* invalidate_bh_lrus() is called rarely - but not only at unmount.
1419	* This doesn't race because it runs in each cpu either in irq
1420	* or with preempt disabled.
1421	*/
1422	static void invalidate_bh_lru(void *arg)
1423	{
1424	struct bh_lru *b = &get_cpu_var(bh_lrus);
1425
1426	__invalidate_bh_lrus(b);
1427	put_cpu_var(bh_lrus);
1428	}
1429
1430	bool has_bh_in_lru(int cpu, void *dummy)
1431	{
1432	struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
1433	int i;
1434
1435	for (i = 0; i < BH_LRU_SIZE; i++) {
1436	if (b->bhs[i])
1437	return true;
1438	}
1439
1440	return false;
1441	}
1442
1443	void invalidate_bh_lrus(void)
1444	{
1445	on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1);
1446	}
1447	EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
1448
1449	/*
1450	* It's called from workqueue context so we need a bh_lru_lock to close
1451	* the race with preemption/irq.
1452	*/
1453	void invalidate_bh_lrus_cpu(void)
1454	{
1455	struct bh_lru *b;
1456
1457	bh_lru_lock();
1458	b = this_cpu_ptr(&bh_lrus);
1459	__invalidate_bh_lrus(b);
1460	bh_lru_unlock();
1461	}
1462
1463	void folio_set_bh(struct buffer_head bh, struct folio folio,
1464	unsigned long offset)
1465	{
1466	bh->b_folio = folio;
1467	BUG_ON(offset >= folio_size(folio));
1468	if (folio_test_highmem(folio))
1469	/*
1470	* This catches illegal uses and preserves the offset:
1471	*/
1472	bh->b_data = (char *)(0 + offset);
1473	else
1474	bh->b_data = folio_address(folio) + offset;
1475	}
1476	EXPORT_SYMBOL(folio_set_bh);
1477
1478	/*
1479	* Called when truncating a buffer on a page completely.
1480	*/
1481
1482	/* Bits that are cleared during an invalidate */
1483	#define BUFFER_FLAGS_DISCARD \
1484	(1 << BH_Mapped \| 1 << BH_New \| 1 << BH_Req \| \
1485	1 << BH_Delay \| 1 << BH_Unwritten)
1486
1487	static void discard_buffer(struct buffer_head * bh)
1488	{
1489	unsigned long b_state;
1490
1491	lock_buffer(bh);
1492	clear_buffer_dirty(bh);
1493	bh->b_bdev = NULL;
1494	b_state = READ_ONCE(bh->b_state);
1495	do {
1496	} while (!try_cmpxchg_relaxed(&bh->b_state, &b_state,
1497	b_state & ~BUFFER_FLAGS_DISCARD));
1498	unlock_buffer(bh);
1499	}
1500
1501	/**
1502	* block_invalidate_folio - Invalidate part or all of a buffer-backed folio.
1503	* @folio: The folio which is affected.
1504	* @offset: start of the range to invalidate
1505	* @length: length of the range to invalidate
1506	*
1507	* block_invalidate_folio() is called when all or part of the folio has been
1508	* invalidated by a truncate operation.
1509	*
1510	* block_invalidate_folio() does not have to release all buffers, but it must
1511	* ensure that no dirty buffer is left outside @offset and that no I/O
1512	* is underway against any of the blocks which are outside the truncation
1513	* point. Because the caller is about to free (and possibly reuse) those
1514	* blocks on-disk.
1515	*/
1516	void block_invalidate_folio(struct folio *folio, size_t offset, size_t length)
1517	{
1518	struct buffer_head head, bh, *next;
1519	size_t curr_off = 0;
1520	size_t stop = length + offset;
1521
1522	BUG_ON(!folio_test_locked(folio));
1523
1524	/*
1525	* Check for overflow
1526	*/
1527	BUG_ON(stop > folio_size(folio) \|\| stop < length);
1528
1529	head = folio_buffers(folio);
1530	if (!head)
1531	return;
1532
1533	bh = head;
1534	do {
1535	size_t next_off = curr_off + bh->b_size;
1536	next = bh->b_this_page;
1537
1538	/*
1539	* Are we still fully in range ?
1540	*/
1541	if (next_off > stop)
1542	goto out;
1543
1544	/*
1545	* is this block fully invalidated?
1546	*/
1547	if (offset <= curr_off)
1548	discard_buffer(bh);
1549	curr_off = next_off;
1550	bh = next;
1551	} while (bh != head);
1552
1553	/*
1554	* We release buffers only if the entire folio is being invalidated.
1555	* The get_block cached value has been unconditionally invalidated,
1556	* so real IO is not possible anymore.
1557	*/
1558	if (length == folio_size(folio))
1559	filemap_release_folio(folio, 0);
1560	out:
1561	folio_clear_mappedtodisk(folio);
1562	}
1563	EXPORT_SYMBOL(block_invalidate_folio);
1564
1565	/*
1566	* We attach and possibly dirty the buffers atomically wrt
1567	* block_dirty_folio() via i_private_lock. try_to_free_buffers
1568	* is already excluded via the folio lock.
1569	*/
1570	struct buffer_head create_empty_buffers(struct folio folio,
1571	unsigned long blocksize, unsigned long b_state)
1572	{
1573	struct buffer_head bh, head, *tail;
1574	gfp_t gfp = GFP_NOFS \| __GFP_ACCOUNT \| __GFP_NOFAIL;
1575
1576	head = folio_alloc_buffers(folio, blocksize, gfp);
1577	bh = head;
1578	do {
1579	bh->b_state \|= b_state;
1580	tail = bh;
1581	bh = bh->b_this_page;
1582	} while (bh);
1583	tail->b_this_page = head;
1584
1585	spin_lock(&folio->mapping->i_private_lock);
1586	if (folio_test_uptodate(folio) \|\| folio_test_dirty(folio)) {
1587	bh = head;
1588	do {
1589	if (folio_test_dirty(folio))
1590	set_buffer_dirty(bh);
1591	if (folio_test_uptodate(folio))
1592	set_buffer_uptodate(bh);
1593	bh = bh->b_this_page;
1594	} while (bh != head);
1595	}
1596	folio_attach_private(folio, head);
1597	spin_unlock(&folio->mapping->i_private_lock);
1598
1599	return head;
1600	}
1601	EXPORT_SYMBOL(create_empty_buffers);
1602
1603	/**
1604	* clean_bdev_aliases: clean a range of buffers in block device
1605	* @bdev: Block device to clean buffers in
1606	* @block: Start of a range of blocks to clean
1607	* @len: Number of blocks to clean
1608	*
1609	* We are taking a range of blocks for data and we don't want writeback of any
1610	* buffer-cache aliases starting from return from this function and until the
1611	* moment when something will explicitly mark the buffer dirty (hopefully that
1612	* will not happen until we will free that block ;-) We don't even need to mark
1613	* it not-uptodate - nobody can expect anything from a newly allocated buffer
1614	* anyway. We used to use unmap_buffer() for such invalidation, but that was
1615	* wrong. We definitely don't want to mark the alias unmapped, for example - it
1616	* would confuse anyone who might pick it with bread() afterwards...
1617	*
1618	* Also.. Note that bforget() doesn't lock the buffer. So there can be
1619	* writeout I/O going on against recently-freed buffers. We don't wait on that
1620	* I/O in bforget() - it's more efficient to wait on the I/O only if we really
1621	* need to. That happens here.
1622	*/
1623	void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
1624	{
1625	struct address_space *bd_mapping = bdev->bd_mapping;
1626	const int blkbits = bd_mapping->host->i_blkbits;
1627	struct folio_batch fbatch;
1628	pgoff_t index = ((loff_t)block << blkbits) / PAGE_SIZE;
1629	pgoff_t end;
1630	int i, count;
1631	struct buffer_head *bh;
1632	struct buffer_head *head;
1633
1634	end = ((loff_t)(block + len - 1) << blkbits) / PAGE_SIZE;
1635	folio_batch_init(&fbatch);
1636	while (filemap_get_folios(bd_mapping, &index, end, &fbatch)) {
1637	count = folio_batch_count(&fbatch);
1638	for (i = 0; i < count; i++) {
1639	struct folio *folio = fbatch.folios[i];
1640
1641	if (!folio_buffers(folio))
1642	continue;
1643	/*
1644	* We use folio lock instead of bd_mapping->i_private_lock
1645	* to pin buffers here since we can afford to sleep and
1646	* it scales better than a global spinlock lock.
1647	*/
1648	folio_lock(folio);
1649	/* Recheck when the folio is locked which pins bhs */
1650	head = folio_buffers(folio);
1651	if (!head)
1652	goto unlock_page;
1653	bh = head;
1654	do {
1655	if (!buffer_mapped(bh) \|\| (bh->b_blocknr < block))
1656	goto next;
1657	if (bh->b_blocknr >= block + len)
1658	break;
1659	clear_buffer_dirty(bh);
1660	wait_on_buffer(bh);
1661	clear_buffer_req(bh);
1662	next:
1663	bh = bh->b_this_page;
1664	} while (bh != head);
1665	unlock_page:
1666	folio_unlock(folio);
1667	}
1668	folio_batch_release(&fbatch);
1669	cond_resched();
1670	/* End of range already reached? */
1671	if (index > end \|\| !index)
1672	break;
1673	}
1674	}
1675	EXPORT_SYMBOL(clean_bdev_aliases);
1676
1677	static struct buffer_head folio_create_buffers(struct folio folio,
1678	struct inode *inode,
1679	unsigned int b_state)
1680	{
1681	struct buffer_head *bh;
1682
1683	BUG_ON(!folio_test_locked(folio));
1684
1685	bh = folio_buffers(folio);
1686	if (!bh)
1687	bh = create_empty_buffers(folio,
1688	1 << READ_ONCE(inode->i_blkbits), b_state);
1689	return bh;
1690	}
1691
1692	/*
1693	* NOTE! All mapped/uptodate combinations are valid:
1694	*
1695	* Mapped Uptodate Meaning
1696	*
1697	* No No "unknown" - must do get_block()
1698	* No Yes "hole" - zero-filled
1699	* Yes No "allocated" - allocated on disk, not read in
1700	* Yes Yes "valid" - allocated and up-to-date in memory.
1701	*
1702	* "Dirty" is valid only with the last case (mapped+uptodate).
1703	*/
1704
1705	/*
1706	* While block_write_full_folio is writing back the dirty buffers under
1707	* the folio lock, whoever dirtied the buffers may decide to clean them
1708	* again at any time. We handle that by only looking at the buffer
1709	* state inside lock_buffer().
1710	*
1711	* If block_write_full_folio() is called for regular writeback
1712	* (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a folio which
1713	* has a locked buffer. This only can happen if someone has written
1714	* the buffer directly, with bh_submit(). At the address_space level
1715	* the folio writeback flag prevents this contention from occurring.
1716	*
1717	* If block_write_full_folio() is called with wbc->sync_mode ==
1718	* WB_SYNC_ALL, the writes are posted using REQ_SYNC; this
1719	* causes the writes to be flagged as synchronous writes.
1720	*/
1721	int __block_write_full_folio(struct inode inode, struct folio folio,
1722	get_block_t get_block, struct writeback_control wbc)
1723	{
1724	int err;
1725	sector_t block;
1726	sector_t last_block;
1727	struct buffer_head bh, head;
1728	size_t blocksize;
1729	int nr_underway = 0;
1730	blk_opf_t write_flags = wbc_to_write_flags(wbc);
1731
1732	head = folio_create_buffers(folio, inode,
1733	(1 << BH_Dirty) \| (1 << BH_Uptodate));
1734
1735	/*
1736	* Be very careful. We have no exclusion from block_dirty_folio
1737	* here, and the (potentially unmapped) buffers may become dirty at
1738	* any time. If a buffer becomes dirty here after we've inspected it
1739	* then we just miss that fact, and the folio stays dirty.
1740	*
1741	* Buffers outside i_size may be dirtied by block_dirty_folio;
1742	* handle that here by just cleaning them.
1743	*/
1744
1745	bh = head;
1746	blocksize = bh->b_size;
1747
1748	block = div_u64(folio_pos(folio), blocksize);
1749	last_block = div_u64(i_size_read(inode) - 1, blocksize);
1750
1751	/*
1752	* Get all the dirty buffers mapped to disk addresses and
1753	* handle any aliases from the underlying blockdev's mapping.
1754	*/
1755	do {
1756	if (block > last_block) {
1757	/*
1758	* mapped buffers outside i_size will occur, because
1759	* this folio can be outside i_size when there is a
1760	* truncate in progress.
1761	*/
1762	/*
1763	* The buffer was zeroed by block_write_full_folio()
1764	*/
1765	clear_buffer_dirty(bh);
1766	set_buffer_uptodate(bh);
1767	} else if ((!buffer_mapped(bh) \|\| buffer_delay(bh)) &&
1768	buffer_dirty(bh)) {
1769	WARN_ON(bh->b_size != blocksize);
1770	err = get_block(inode, block, bh, 1);
1771	if (err)
1772	goto recover;
1773	clear_buffer_delay(bh);
1774	if (buffer_new(bh)) {
1775	/* blockdev mappings never come here */
1776	clear_buffer_new(bh);
1777	clean_bdev_bh_alias(bh);
1778	}
1779	}
1780	bh = bh->b_this_page;
1781	block++;
1782	} while (bh != head);
1783
1784	do {
1785	if (!buffer_mapped(bh))
1786	continue;
1787	/*
1788	* If it's a fully non-blocking write attempt and we cannot
1789	* lock the buffer then redirty the folio. Note that this can
1790	* potentially cause a busy-wait loop from writeback threads
1791	* and kswapd activity, but those code paths have their own
1792	* higher-level throttling.
1793	*/
1794	if (wbc->sync_mode != WB_SYNC_NONE) {
1795	lock_buffer(bh);
1796	} else if (!trylock_buffer(bh)) {
1797	folio_redirty_for_writepage(wbc, folio);
1798	continue;
1799	}
1800	if (test_clear_buffer_dirty(bh)) {
1801	set_buffer_async_write(bh);
1802	} else {
1803	unlock_buffer(bh);
1804	}
1805	} while ((bh = bh->b_this_page) != head);
1806
1807	/*
1808	* The folio and its buffers are protected by the writeback flag,
1809	* so we can drop the bh refcounts early.
1810	*/
1811	BUG_ON(folio_test_writeback(folio));
1812	folio_start_writeback(folio);
1813
1814	do {
1815	struct buffer_head *next = bh->b_this_page;
1816	if (buffer_async_write(bh)) {
1817	__bh_submit(bh, REQ_OP_WRITE \| write_flags,
1818	inode->i_write_hint, wbc,
1819	bh_end_async_write);
1820	nr_underway++;
1821	}
1822	bh = next;
1823	} while (bh != head);
1824	folio_unlock(folio);
1825
1826	err = 0;
1827	done:
1828	if (nr_underway == 0) {
1829	/*
1830	* The folio was marked dirty, but the buffers were
1831	* clean. Someone wrote them back by hand with
1832	* write_dirty_buffer/bh_submit. A rare case.
1833	*/
1834	folio_end_writeback(folio);
1835
1836	/*
1837	* The folio and buffer_heads can be released at any time from
1838	* here on.
1839	*/
1840	}
1841	return err;
1842
1843	recover:
1844	/*
1845	* ENOSPC, or some other error. We may already have added some
1846	* blocks to the file, so we need to write these out to avoid
1847	* exposing stale data.
1848	* The folio is currently locked and not marked for writeback
1849	*/
1850	bh = head;
1851	/* Recovery: lock and submit the mapped buffers */
1852	do {
1853	if (buffer_mapped(bh) && buffer_dirty(bh) &&
1854	!buffer_delay(bh)) {
1855	lock_buffer(bh);
1856	set_buffer_async_write(bh);
1857	} else {
1858	/*
1859	* The buffer may have been set dirty during
1860	* attachment to a dirty folio.
1861	*/
1862	clear_buffer_dirty(bh);
1863	}
1864	} while ((bh = bh->b_this_page) != head);
1865	BUG_ON(folio_test_writeback(folio));
1866	mapping_set_error(folio->mapping, err);
1867	folio_start_writeback(folio);
1868	do {
1869	struct buffer_head *next = bh->b_this_page;
1870	if (buffer_async_write(bh)) {
1871	clear_buffer_dirty(bh);
1872	__bh_submit(bh, REQ_OP_WRITE \| write_flags,
1873	inode->i_write_hint, wbc,
1874	bh_end_async_write);
1875	nr_underway++;
1876	}
1877	bh = next;
1878	} while (bh != head);
1879	folio_unlock(folio);
1880	goto done;
1881	}
1882	EXPORT_SYMBOL(__block_write_full_folio);
1883
1884	/*
1885	* If a folio has any new buffers, zero them out here, and mark them uptodate
1886	* and dirty so they'll be written out (in order to prevent uninitialised
1887	* block data from leaking). And clear the new bit.
1888	*/
1889	void folio_zero_new_buffers(struct folio *folio, size_t from, size_t to)
1890	{
1891	size_t block_start, block_end;
1892	struct buffer_head head, bh;
1893
1894	BUG_ON(!folio_test_locked(folio));
1895	head = folio_buffers(folio);
1896	if (!head)
1897	return;
1898
1899	bh = head;
1900	block_start = 0;
1901	do {
1902	block_end = block_start + bh->b_size;
1903
1904	if (buffer_new(bh)) {
1905	if (block_end > from && block_start < to) {
1906	if (!folio_test_uptodate(folio)) {
1907	size_t start, xend;
1908
1909	start = max(from, block_start);
1910	xend = min(to, block_end);
1911
1912	folio_zero_segment(folio, start, xend);
1913	set_buffer_uptodate(bh);
1914	}
1915
1916	clear_buffer_new(bh);
1917	mark_buffer_dirty(bh);
1918	}
1919	}
1920
1921	block_start = block_end;
1922	bh = bh->b_this_page;
1923	} while (bh != head);
1924	}
1925	EXPORT_SYMBOL(folio_zero_new_buffers);
1926
1927	static int
1928	iomap_to_bh(struct inode inode, sector_t block, struct buffer_head bh,
1929	const struct iomap *iomap)
1930	{
1931	loff_t offset = (loff_t)block << inode->i_blkbits;
1932
1933	bh->b_bdev = iomap->bdev;
1934
1935	/*
1936	* Block points to offset in file we need to map, iomap contains
1937	* the offset at which the map starts. If the map ends before the
1938	* current block, then do not map the buffer and let the caller
1939	* handle it.
1940	*/
1941	if (offset >= iomap->offset + iomap->length)
1942	return -EIO;
1943
1944	switch (iomap->type) {
1945	case IOMAP_HOLE:
1946	/*
1947	* If the buffer is not up to date or beyond the current EOF,
1948	* we need to mark it as new to ensure sub-block zeroing is
1949	* executed if necessary.
1950	*/
1951	if (!buffer_uptodate(bh) \|\|
1952	(offset >= i_size_read(inode)))
1953	set_buffer_new(bh);
1954	return 0;
1955	case IOMAP_DELALLOC:
1956	if (!buffer_uptodate(bh) \|\|
1957	(offset >= i_size_read(inode)))
1958	set_buffer_new(bh);
1959	set_buffer_uptodate(bh);
1960	set_buffer_mapped(bh);
1961	set_buffer_delay(bh);
1962	return 0;
1963	case IOMAP_UNWRITTEN:
1964	/*
1965	* For unwritten regions, we always need to ensure that regions
1966	* in the block we are not writing to are zeroed. Mark the
1967	* buffer as new to ensure this.
1968	*/
1969	set_buffer_new(bh);
1970	set_buffer_unwritten(bh);
1971	fallthrough;
1972	case IOMAP_MAPPED:
1973	if ((iomap->flags & IOMAP_F_NEW) \|\|
1974	offset >= i_size_read(inode)) {
1975	/*
1976	* This can happen if truncating the block device races
1977	* with the check in the caller as i_size updates on
1978	* block devices aren't synchronized by i_rwsem for
1979	* block devices.
1980	*/
1981	if (S_ISBLK(inode->i_mode))
1982	return -EIO;
1983	set_buffer_new(bh);
1984	}
1985	bh->b_blocknr = (iomap->addr + offset - iomap->offset) >>
1986	inode->i_blkbits;
1987	set_buffer_mapped(bh);
1988	return 0;
1989	default:
1990	WARN_ON_ONCE(1);
1991	return -EIO;
1992	}
1993	}
1994
1995	int __block_write_begin_int(struct folio *folio, loff_t pos, unsigned len,
1996	get_block_t get_block, const struct iomap iomap)
1997	{
1998	size_t from = offset_in_folio(folio, pos);
1999	size_t to = from + len;
2000	struct inode *inode = folio->mapping->host;
2001	size_t block_start, block_end;
2002	sector_t block;
2003	int err = 0;
2004	size_t blocksize;
2005	struct buffer_head bh, head, wait[2], *wait_bh=wait;
2006
2007	BUG_ON(!folio_test_locked(folio));
2008	BUG_ON(to > folio_size(folio));
2009	BUG_ON(from > to);
2010
2011	head = folio_create_buffers(folio, inode, 0);
2012	blocksize = head->b_size;
2013	block = div_u64(folio_pos(folio), blocksize);
2014
2015	for (bh = head, block_start = 0; bh != head \|\| !block_start;
2016	block++, block_start=block_end, bh = bh->b_this_page) {
2017	block_end = block_start + blocksize;
2018	if (block_end <= from \|\| block_start >= to) {
2019	if (folio_test_uptodate(folio)) {
2020	if (!buffer_uptodate(bh))
2021	set_buffer_uptodate(bh);
2022	}
2023	continue;
2024	}
2025	if (buffer_new(bh))
2026	clear_buffer_new(bh);
2027	if (!buffer_mapped(bh)) {
2028	WARN_ON(bh->b_size != blocksize);
2029	if (get_block)
2030	err = get_block(inode, block, bh, 1);
2031	else
2032	err = iomap_to_bh(inode, block, bh, iomap);
2033	if (err)
2034	break;
2035
2036	if (buffer_new(bh)) {
2037	clean_bdev_bh_alias(bh);
2038	if (folio_test_uptodate(folio)) {
2039	clear_buffer_new(bh);
2040	set_buffer_uptodate(bh);
2041	mark_buffer_dirty(bh);
2042	continue;
2043	}
2044	if (block_end > to \|\| block_start < from)
2045	folio_zero_segments(folio,
2046	to, block_end,
2047	block_start, from);
2048	continue;
2049	}
2050	}
2051	if (folio_test_uptodate(folio)) {
2052	if (!buffer_uptodate(bh))
2053	set_buffer_uptodate(bh);
2054	continue;
2055	}
2056	if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
2057	!buffer_unwritten(bh) &&
2058	(block_start < from \|\| block_end > to)) {
2059	bh_read_nowait(bh, 0);
2060	*wait_bh++=bh;
2061	}
2062	}
2063	/*
2064	* If we issued read requests - let them complete.
2065	*/
2066	while(wait_bh > wait) {
2067	wait_on_buffer(*--wait_bh);
2068	if (!buffer_uptodate(*wait_bh))
2069	err = -EIO;
2070	}
2071	if (unlikely(err))
2072	folio_zero_new_buffers(folio, from, to);
2073	return err;
2074	}
2075
2076	int __block_write_begin(struct folio *folio, loff_t pos, unsigned len,
2077	get_block_t *get_block)
2078	{
2079	return __block_write_begin_int(folio, pos, len, get_block, NULL);
2080	}
2081	EXPORT_SYMBOL(__block_write_begin);
2082
2083	void block_commit_write(struct folio *folio, size_t from, size_t to)
2084	{
2085	size_t block_start, block_end;
2086	bool partial = false;
2087	bool uptodate = folio_test_uptodate(folio);
2088	unsigned blocksize;
2089	struct buffer_head bh, head;
2090
2091	bh = head = folio_buffers(folio);
2092	if (!bh)
2093	return;
2094	blocksize = bh->b_size;
2095
2096	block_start = 0;
2097	do {
2098	block_end = block_start + blocksize;
2099	if (block_end <= from \|\| block_start >= to) {
2100	if (!buffer_uptodate(bh))
2101	partial = true;
2102	} else {
2103	set_buffer_uptodate(bh);
2104	mark_buffer_dirty(bh);
2105	}
2106	if (buffer_new(bh))
2107	clear_buffer_new(bh);
2108
2109	block_start = block_end;
2110	if (uptodate && block_start >= to)
2111	break;
2112	bh = bh->b_this_page;
2113	} while (bh != head);
2114
2115	/*
2116	* If this is a partial write which happened to make all buffers
2117	* uptodate then we can optimize away a bogus read_folio() for
2118	* the next read(). Here we 'discover' whether the folio went
2119	* uptodate as a result of this (potentially partial) write.
2120	*/
2121	if (!partial)
2122	folio_mark_uptodate(folio);
2123	}
2124	EXPORT_SYMBOL(block_commit_write);
2125
2126	/*
2127	* block_write_begin takes care of the basic task of block allocation and
2128	* bringing partial write blocks uptodate first.
2129	*
2130	* The filesystem needs to handle block truncation upon failure.
2131	*/
2132	int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
2133	struct folio *foliop, get_block_t get_block)
2134	{
2135	pgoff_t index = pos >> PAGE_SHIFT;
2136	struct folio *folio;
2137	int status;
2138
2139	folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN,
2140	mapping_gfp_mask(mapping));
2141	if (IS_ERR(folio))
2142	return PTR_ERR(folio);
2143
2144	status = __block_write_begin_int(folio, pos, len, get_block, NULL);
2145	if (unlikely(status)) {
2146	folio_unlock(folio);
2147	folio_put(folio);
2148	folio = NULL;
2149	}
2150
2151	*foliop = folio;
2152	return status;
2153	}
2154	EXPORT_SYMBOL(block_write_begin);
2155
2156	int block_write_end(loff_t pos, unsigned len, unsigned copied,
2157	struct folio *folio)
2158	{
2159	size_t start = pos - folio_pos(folio);
2160
2161	if (unlikely(copied < len)) {
2162	/*
2163	* The buffers that were written will now be uptodate, so
2164	* we don't have to worry about a read_folio reading them
2165	* and overwriting a partial write. However if we have
2166	* encountered a short write and only partially written
2167	* into a buffer, it will not be marked uptodate, so a
2168	* read_folio might come in and destroy our partial write.
2169	*
2170	* Do the simplest thing, and just treat any short write to a
2171	* non uptodate folio as a zero-length write, and force the
2172	* caller to redo the whole thing.
2173	*/
2174	if (!folio_test_uptodate(folio))
2175	copied = 0;
2176
2177	folio_zero_new_buffers(folio, start+copied, start+len);
2178	}
2179	flush_dcache_folio(folio);
2180
2181	/* This could be a short (even 0-length) commit */
2182	block_commit_write(folio, start, start + copied);
2183
2184	return copied;
2185	}
2186	EXPORT_SYMBOL(block_write_end);
2187
2188	int generic_write_end(const struct kiocb iocb, struct address_space mapping,
2189	loff_t pos, unsigned len, unsigned copied,
2190	struct folio folio, void fsdata)
2191	{
2192	struct inode *inode = mapping->host;
2193	loff_t old_size = inode->i_size;
2194	bool i_size_changed = false;
2195
2196	copied = block_write_end(pos, len, copied, folio);
2197
2198	/*
2199	* No need to use i_size_read() here, the i_size cannot change under us
2200	* because we hold i_rwsem.
2201	*
2202	* But it's important to update i_size while still holding folio lock:
2203	* page writeout could otherwise come in and zero beyond i_size.
2204	*/
2205	if (pos + copied > inode->i_size) {
2206	i_size_write(inode, pos + copied);
2207	i_size_changed = true;
2208	}
2209
2210	folio_unlock(folio);
2211	folio_put(folio);
2212
2213	if (old_size < pos)
2214	pagecache_isize_extended(inode, old_size, pos);
2215	/*
2216	* Don't mark the inode dirty under page lock. First, it unnecessarily
2217	* makes the holding time of page lock longer. Second, it forces lock
2218	* ordering of page lock and transaction start for journaling
2219	* filesystems.
2220	*/
2221	if (i_size_changed)
2222	mark_inode_dirty(inode);
2223	return copied;
2224	}
2225	EXPORT_SYMBOL(generic_write_end);
2226
2227	/*
2228	* block_is_partially_uptodate checks whether buffers within a folio are
2229	* uptodate or not.
2230	*
2231	* Returns true if all buffers which correspond to the specified part
2232	* of the folio are uptodate.
2233	*/
2234	bool block_is_partially_uptodate(struct folio *folio, size_t from, size_t count)
2235	{
2236	unsigned block_start, block_end, blocksize;
2237	unsigned to;
2238	struct buffer_head bh, head;
2239	bool ret = true;
2240
2241	head = folio_buffers(folio);
2242	if (!head)
2243	return false;
2244	blocksize = head->b_size;
2245	to = min(folio_size(folio) - from, count);
2246	to = from + to;
2247	if (from < blocksize && to > folio_size(folio) - blocksize)
2248	return false;
2249
2250	bh = head;
2251	block_start = 0;
2252	do {
2253	block_end = block_start + blocksize;
2254	if (block_end > from && block_start < to) {
2255	if (!buffer_uptodate(bh)) {
2256	ret = false;
2257	break;
2258	}
2259	if (block_end >= to)
2260	break;
2261	}
2262	block_start = block_end;
2263	bh = bh->b_this_page;
2264	} while (bh != head);
2265
2266	return ret;
2267	}
2268	EXPORT_SYMBOL(block_is_partially_uptodate);
2269
2270	/*
2271	* Generic "read_folio" function for block devices that have the normal
2272	* get_block functionality. This is most of the block device filesystems.
2273	* Reads the folio asynchronously --- the unlock_buffer() and
2274	* set/clear_buffer_uptodate() functions propagate buffer state into the
2275	* folio once IO has completed.
2276	*/
2277	int block_read_full_folio(struct folio folio, get_block_t get_block)
2278	{
2279	struct inode *inode = folio->mapping->host;
2280	sector_t iblock, lblock;
2281	struct buffer_head bh, head, *prev = NULL;
2282	size_t blocksize;
2283	int fully_mapped = 1;
2284	bool page_error = false;
2285	loff_t limit = i_size_read(inode);
2286
2287	/* This is needed for ext4. */
2288	if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode))
2289	limit = inode->i_sb->s_maxbytes;
2290
2291	head = folio_create_buffers(folio, inode, 0);
2292	blocksize = head->b_size;
2293
2294	iblock = div_u64(folio_pos(folio), blocksize);
2295	lblock = div_u64(limit + blocksize - 1, blocksize);
2296	bh = head;
2297
2298	do {
2299	if (buffer_uptodate(bh))
2300	continue;
2301
2302	if (!buffer_mapped(bh)) {
2303	int err = 0;
2304
2305	fully_mapped = 0;
2306	if (iblock < lblock) {
2307	WARN_ON(bh->b_size != blocksize);
2308	err = get_block(inode, iblock, bh, 0);
2309	if (err)
2310	page_error = true;
2311	}
2312	if (!buffer_mapped(bh)) {
2313	folio_zero_range(folio, bh_offset(bh),
2314	blocksize);
2315	if (!err)
2316	set_buffer_uptodate(bh);
2317	continue;
2318	}
2319	/*
2320	* get_block() might have updated the buffer
2321	* synchronously
2322	*/
2323	if (buffer_uptodate(bh))
2324	continue;
2325	}
2326
2327	lock_buffer(bh);
2328	if (buffer_uptodate(bh)) {
2329	unlock_buffer(bh);
2330	continue;
2331	}
2332
2333	/*
2334	* If a folio's buffers are under async readin
2335	* (end_buffer_async_read completion) then there is a
2336	* possibility that another thread of control could lock
2337	* one of the buffers after it has completed but while
2338	* some of the other buffers have not completed. This
2339	* locked buffer would confuse end_buffer_async_read()
2340	* into not unlocking the folio. So the absence of
2341	* BH_Async_Read tells end_buffer_async_read() that this
2342	* buffer is not under async I/O.
2343	*
2344	* The folio comes unlocked when it has no locked
2345	* buffer_async buffers left.
2346	*
2347	* The folio lock prevents anyone starting new async
2348	* I/O reads into any of the buffers.
2349	*
2350	* The writeback flag is used to prevent simultaneous
2351	* writeout of the same folio.
2352	*
2353	* The folio lock prevents anyone from starting writeback
2354	* of a folio which is under read I/O (the writeback
2355	* flag is only ever set on a locked folio).
2356	*/
2357	set_buffer_async_read(bh);
2358	if (prev)
2359	bh_submit(prev, REQ_OP_READ, bh_end_async_read);
2360	prev = bh;
2361	} while (iblock++, (bh = bh->b_this_page) != head);
2362
2363	if (fully_mapped)
2364	folio_set_mappedtodisk(folio);
2365
2366	/*
2367	* All buffers are uptodate or get_block() returned an error
2368	* when trying to map them - we must finish the read because
2369	* end_buffer_async_read() will never be called on any buffer
2370	* in this folio.
2371	*/
2372	if (prev)
2373	bh_submit(prev, REQ_OP_READ, bh_end_async_read);
2374	else
2375	folio_end_read(folio, !page_error);
2376
2377	return 0;
2378	}
2379	EXPORT_SYMBOL(block_read_full_folio);
2380
2381	/* utility function for filesystems that need to do work on expanding
2382	* truncates. Uses filesystem pagecache writes to allow the filesystem to
2383	* deal with the hole.
2384	*/
2385	int generic_cont_expand_simple(struct inode *inode, loff_t size)
2386	{
2387	struct address_space *mapping = inode->i_mapping;
2388	const struct address_space_operations *aops = mapping->a_ops;
2389	struct folio *folio;
2390	void *fsdata = NULL;
2391	int err;
2392
2393	err = inode_newsize_ok(inode, size);
2394	if (err)
2395	goto out;
2396
2397	err = aops->write_begin(NULL, mapping, size, 0, &folio, &fsdata);
2398	if (err)
2399	goto out;
2400
2401	err = aops->write_end(NULL, mapping, size, 0, 0, folio, fsdata);
2402	BUG_ON(err > 0);
2403
2404	out:
2405	return err;
2406	}
2407	EXPORT_SYMBOL(generic_cont_expand_simple);
2408
2409	static int cont_expand_zero(const struct kiocb *iocb,
2410	struct address_space *mapping,
2411	loff_t pos, loff_t *bytes)
2412	{
2413	struct inode *inode = mapping->host;
2414	const struct address_space_operations *aops = mapping->a_ops;
2415	unsigned int blocksize = i_blocksize(inode);
2416	struct folio *folio;
2417	void *fsdata = NULL;
2418	pgoff_t index, curidx;
2419	loff_t curpos;
2420	unsigned zerofrom, offset, len;
2421	int err = 0;
2422
2423	index = pos >> PAGE_SHIFT;
2424	offset = pos & ~PAGE_MASK;
2425
2426	while (index > (curidx = (curpos = *bytes)>>PAGE_SHIFT)) {
2427	zerofrom = curpos & ~PAGE_MASK;
2428	if (zerofrom & (blocksize-1)) {
2429	*bytes \|= (blocksize-1);
2430	(*bytes)++;
2431	}
2432	len = PAGE_SIZE - zerofrom;
2433
2434	err = aops->write_begin(iocb, mapping, curpos, len,
2435	&folio, &fsdata);
2436	if (err)
2437	goto out;
2438	folio_zero_range(folio, offset_in_folio(folio, curpos), len);
2439	err = aops->write_end(iocb, mapping, curpos, len, len,
2440	folio, fsdata);
2441	if (err < 0)
2442	goto out;
2443	BUG_ON(err != len);
2444	err = 0;
2445
2446	balance_dirty_pages_ratelimited(mapping);
2447
2448	if (fatal_signal_pending(current)) {
2449	err = -EINTR;
2450	goto out;
2451	}
2452	}
2453
2454	/* page covers the boundary, find the boundary offset */
2455	if (index == curidx) {
2456	zerofrom = curpos & ~PAGE_MASK;
2457	/* if we will expand the thing last block will be filled */
2458	if (offset <= zerofrom) {
2459	goto out;
2460	}
2461	if (zerofrom & (blocksize-1)) {
2462	*bytes \|= (blocksize-1);
2463	(*bytes)++;
2464	}
2465	len = offset - zerofrom;
2466
2467	err = aops->write_begin(iocb, mapping, curpos, len,
2468	&folio, &fsdata);
2469	if (err)
2470	goto out;
2471	folio_zero_range(folio, offset_in_folio(folio, curpos), len);
2472	err = aops->write_end(iocb, mapping, curpos, len, len,
2473	folio, fsdata);
2474	if (err < 0)
2475	goto out;
2476	BUG_ON(err != len);
2477	err = 0;
2478	}
2479	out:
2480	return err;
2481	}
2482
2483	/*
2484	* For moronic filesystems that do not allow holes in file.
2485	* We may have to extend the file.
2486	*/
2487	int cont_write_begin(const struct kiocb iocb, struct address_space mapping,
2488	loff_t pos, unsigned len, struct folio **foliop,
2489	void *fsdata, get_block_t get_block, loff_t *bytes)
2490	{
2491	struct inode *inode = mapping->host;
2492	unsigned int blocksize = i_blocksize(inode);
2493	unsigned int zerofrom;
2494	int err;
2495
2496	err = cont_expand_zero(iocb, mapping, pos, bytes);
2497	if (err)
2498	return err;
2499
2500	zerofrom = *bytes & ~PAGE_MASK;
2501	if (pos+len > *bytes && zerofrom & (blocksize-1)) {
2502	*bytes \|= (blocksize-1);
2503	(*bytes)++;
2504	}
2505
2506	return block_write_begin(mapping, pos, len, foliop, get_block);
2507	}
2508	EXPORT_SYMBOL(cont_write_begin);
2509
2510	/*
2511	* block_page_mkwrite() is not allowed to change the file size as it gets
2512	* called from a page fault handler when a page is first dirtied. Hence we must
2513	* be careful to check for EOF conditions here. We set the page up correctly
2514	* for a written page which means we get ENOSPC checking when writing into
2515	* holes and correct delalloc and unwritten extent mapping on filesystems that
2516	* support these features.
2517	*
2518	* We are not allowed to take the i_rwsem here so we have to play games to
2519	* protect against truncate races as the page could now be beyond EOF. Because
2520	* truncate writes the inode size before removing pages, once we have the
2521	* page lock we can determine safely if the page is beyond EOF. If it is not
2522	* beyond EOF, then the page is guaranteed safe against truncation until we
2523	* unlock the page.
2524	*
2525	* Direct callers of this function should protect against filesystem freezing
2526	* using sb_start_pagefault() - sb_end_pagefault() functions.
2527	*/
2528	int block_page_mkwrite(struct vm_area_struct vma, struct vm_fault vmf,
2529	get_block_t get_block)
2530	{
2531	struct folio *folio = page_folio(vmf->page);
2532	struct inode *inode = file_inode(vma->vm_file);
2533	unsigned long end;
2534	loff_t size;
2535	int ret;
2536
2537	folio_lock(folio);
2538	size = i_size_read(inode);
2539	if ((folio->mapping != inode->i_mapping) \|\|
2540	(folio_pos(folio) >= size)) {
2541	/* We overload EFAULT to mean page got truncated */
2542	ret = -EFAULT;
2543	goto out_unlock;
2544	}
2545
2546	end = folio_size(folio);
2547	/* folio is wholly or partially inside EOF */
2548	if (folio_pos(folio) + end > size)
2549	end = size - folio_pos(folio);
2550
2551	ret = __block_write_begin_int(folio, 0, end, get_block, NULL);
2552	if (unlikely(ret))
2553	goto out_unlock;
2554
2555	block_commit_write(folio, 0, end);
2556
2557	folio_mark_dirty(folio);
2558	folio_wait_stable(folio);
2559	return 0;
2560	out_unlock:
2561	folio_unlock(folio);
2562	return ret;
2563	}
2564	EXPORT_SYMBOL(block_page_mkwrite);
2565
2566	int block_truncate_page(struct address_space *mapping,
2567	loff_t from, get_block_t *get_block)
2568	{
2569	pgoff_t index = from >> PAGE_SHIFT;
2570	unsigned blocksize;
2571	sector_t iblock;
2572	size_t offset, length, pos;
2573	struct inode *inode = mapping->host;
2574	struct folio *folio;
2575	struct buffer_head *bh;
2576	int err = 0;
2577
2578	blocksize = i_blocksize(inode);
2579	length = from & (blocksize - 1);
2580
2581	/* Block boundary? Nothing to do */
2582	if (!length)
2583	return 0;
2584
2585	length = blocksize - length;
2586	iblock = ((loff_t)index * PAGE_SIZE) >> inode->i_blkbits;
2587
2588	folio = filemap_grab_folio(mapping, index);
2589	if (IS_ERR(folio))
2590	return PTR_ERR(folio);
2591
2592	bh = folio_buffers(folio);
2593	if (!bh)
2594	bh = create_empty_buffers(folio, blocksize, 0);
2595
2596	/* Find the buffer that contains "offset" */
2597	offset = offset_in_folio(folio, from);
2598	pos = blocksize;
2599	while (offset >= pos) {
2600	bh = bh->b_this_page;
2601	iblock++;
2602	pos += blocksize;
2603	}
2604
2605	if (!buffer_mapped(bh)) {
2606	WARN_ON(bh->b_size != blocksize);
2607	err = get_block(inode, iblock, bh, 0);
2608	if (err)
2609	goto unlock;
2610	/* unmapped? It's a hole - nothing to do */
2611	if (!buffer_mapped(bh))
2612	goto unlock;
2613	}
2614
2615	/* Ok, it's mapped. Make sure it's up-to-date */
2616	if (folio_test_uptodate(folio))
2617	set_buffer_uptodate(bh);
2618
2619	if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
2620	err = bh_read(bh, 0);
2621	/* Uhhuh. Read error. Complain and punt. */
2622	if (err < 0)
2623	goto unlock;
2624	}
2625
2626	folio_zero_range(folio, offset, length);
2627	mark_buffer_dirty(bh);
2628
2629	unlock:
2630	folio_unlock(folio);
2631	folio_put(folio);
2632
2633	return err;
2634	}
2635	EXPORT_SYMBOL(block_truncate_page);
2636
2637	/*
2638	* The generic write folio function for buffer-backed address_spaces
2639	*/
2640	int block_write_full_folio(struct folio folio, struct writeback_control wbc,
2641	void *get_block)
2642	{
2643	struct inode * const inode = folio->mapping->host;
2644	loff_t i_size = i_size_read(inode);
2645
2646	/* Is the folio fully inside i_size? */
2647	if (folio_next_pos(folio) <= i_size)
2648	return __block_write_full_folio(inode, folio, get_block, wbc);
2649
2650	/* Is the folio fully outside i_size? (truncate in progress) */
2651	if (folio_pos(folio) >= i_size) {
2652	folio_unlock(folio);
2653	return 0; /* don't care */
2654	}
2655
2656	/*
2657	* The folio straddles i_size. It must be zeroed out on each and every
2658	* writeback invocation because it may be mmapped. "A file is mapped
2659	* in multiples of the page size. For a file that is not a multiple of
2660	* the page size, the remaining memory is zeroed when mapped, and
2661	* writes to that region are not written out to the file."
2662	*/
2663	folio_zero_segment(folio, offset_in_folio(folio, i_size),
2664	folio_size(folio));
2665	return __block_write_full_folio(inode, folio, get_block, wbc);
2666	}
2667
2668	sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2669	get_block_t *get_block)
2670	{
2671	struct inode *inode = mapping->host;
2672	struct buffer_head tmp = {
2673	.b_size = i_blocksize(inode),
2674	};
2675
2676	get_block(inode, block, &tmp, 0);
2677	return tmp.b_blocknr;
2678	}
2679	EXPORT_SYMBOL(generic_block_bmap);
2680
2681	void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags)
2682	{
2683	lock_buffer(bh);
2684	if (!test_clear_buffer_dirty(bh)) {
2685	unlock_buffer(bh);
2686	return;
2687	}
2688	bh_submit(bh, REQ_OP_WRITE \| op_flags, bh_end_write);
2689	}
2690	EXPORT_SYMBOL(write_dirty_buffer);
2691
2692	/*
2693	* For a data-integrity writeout, we need to wait upon any in-progress I/O
2694	* and then start new I/O and then wait upon it. The caller must have a ref on
2695	* the buffer_head.
2696	*/
2697	int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags)
2698	{
2699	WARN_ON(atomic_read(&bh->b_count) < 1);
2700	lock_buffer(bh);
2701	if (test_clear_buffer_dirty(bh)) {
2702	/*
2703	* The bh should be mapped, but it might not be if the
2704	* device was hot-removed. Not much we can do but fail the I/O.
2705	*/
2706	if (!buffer_mapped(bh)) {
2707	unlock_buffer(bh);
2708	return -EIO;
2709	}
2710
2711	bh_submit(bh, REQ_OP_WRITE \| op_flags, bh_end_write);
2712	wait_on_buffer(bh);
2713	if (!buffer_uptodate(bh))
2714	return -EIO;
2715	} else {
2716	unlock_buffer(bh);
2717	}
2718	return 0;
2719	}
2720	EXPORT_SYMBOL(__sync_dirty_buffer);
2721
2722	int sync_dirty_buffer(struct buffer_head *bh)
2723	{
2724	return __sync_dirty_buffer(bh, REQ_SYNC);
2725	}
2726	EXPORT_SYMBOL(sync_dirty_buffer);
2727
2728	static inline int buffer_busy(struct buffer_head *bh)
2729	{
2730	return atomic_read(&bh->b_count) \|
2731	(bh->b_state & ((1 << BH_Dirty) \| (1 << BH_Lock)));
2732	}
2733
2734	static bool
2735	drop_buffers(struct folio folio, struct buffer_head *buffers_to_free)
2736	{
2737	struct buffer_head *head = folio_buffers(folio);
2738	struct buffer_head *bh;
2739
2740	bh = head;
2741	do {
2742	if (buffer_busy(bh))
2743	goto failed;
2744	bh = bh->b_this_page;
2745	} while (bh != head);
2746
2747	do {
2748	struct buffer_head *next = bh->b_this_page;
2749
2750	remove_assoc_queue(bh);
2751	bh = next;
2752	} while (bh != head);
2753	*buffers_to_free = head;
2754	folio_detach_private(folio);
2755	return true;
2756	failed:
2757	return false;
2758	}
2759
2760	/**
2761	* try_to_free_buffers - Release buffers attached to this folio.
2762	* @folio: The folio.
2763	*
2764	* If any buffers are in use (dirty, under writeback, elevated refcount),
2765	* no buffers will be freed.
2766	*
2767	* If the folio is dirty but all the buffers are clean then we need to
2768	* be sure to mark the folio clean as well. This is because the folio
2769	* may be against a block device, and a later reattachment of buffers
2770	* to a dirty folio will set all buffers dirty. Which would corrupt
2771	* filesystem data on the same device.
2772	*
2773	* The same applies to regular filesystem folios: if all the buffers are
2774	* clean then we set the folio clean and proceed. To do that, we require
2775	* total exclusion from block_dirty_folio(). That is obtained with
2776	* i_private_lock.
2777	*
2778	* Exclusion against try_to_free_buffers may be obtained by either
2779	* locking the folio or by holding its mapping's i_private_lock.
2780	*
2781	* Context: Process context. @folio must be locked. Will not sleep.
2782	* Return: true if all buffers attached to this folio were freed.
2783	*/
2784	bool try_to_free_buffers(struct folio *folio)
2785	{
2786	struct address_space * const mapping = folio->mapping;
2787	struct buffer_head *buffers_to_free = NULL;
2788	bool ret = 0;
2789
2790	BUG_ON(!folio_test_locked(folio));
2791	if (folio_test_writeback(folio))
2792	return false;
2793
2794	/* Misconfigured folio check */
2795	if (WARN_ON_ONCE(!folio_buffers(folio)))
2796	return true;
2797
2798	if (mapping == NULL) { /* can this still happen? */
2799	ret = drop_buffers(folio, &buffers_to_free);
2800	goto out;
2801	}
2802
2803	spin_lock(&mapping->i_private_lock);
2804	ret = drop_buffers(folio, &buffers_to_free);
2805
2806	/*
2807	* If the filesystem writes its buffers by hand (eg ext3)
2808	* then we can have clean buffers against a dirty folio. We
2809	* clean the folio here; otherwise the VM will never notice
2810	* that the filesystem did any IO at all.
2811	*
2812	* Also, during truncate, discard_buffer will have marked all
2813	* the folio's buffers clean. We discover that here and clean
2814	* the folio also.
2815	*
2816	* i_private_lock must be held over this entire operation in order
2817	* to synchronise against block_dirty_folio and prevent the
2818	* dirty bit from being lost.
2819	*/
2820	if (ret)
2821	folio_cancel_dirty(folio);
2822	spin_unlock(&mapping->i_private_lock);
2823	out:
2824	if (buffers_to_free) {
2825	struct buffer_head *bh = buffers_to_free;
2826
2827	do {
2828	struct buffer_head *next = bh->b_this_page;
2829	free_buffer_head(bh);
2830	bh = next;
2831	} while (bh != buffers_to_free);
2832	}
2833	return ret;
2834	}
2835	EXPORT_SYMBOL(try_to_free_buffers);
2836
2837	/*
2838	* Buffer-head allocation
2839	*/
2840	static struct kmem_cache *bh_cachep __ro_after_init;
2841
2842	/*
2843	* Once the number of bh's in the machine exceeds this level, we start
2844	* stripping them in writeback.
2845	*/
2846	static unsigned long max_buffer_heads __ro_after_init;
2847
2848	int buffer_heads_over_limit;
2849
2850	struct bh_accounting {
2851	int nr; /* Number of live bh's */
2852	int ratelimit; /* Limit cacheline bouncing */
2853	};
2854
2855	static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
2856
2857	static void recalc_bh_state(void)
2858	{
2859	int i;
2860	int tot = 0;
2861
2862	if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
2863	return;
2864	__this_cpu_write(bh_accounting.ratelimit, 0);
2865	for_each_online_cpu(i)
2866	tot += per_cpu(bh_accounting, i).nr;
2867	buffer_heads_over_limit = (tot > max_buffer_heads);
2868	}
2869
2870	struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
2871	{
2872	struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
2873	if (ret) {
2874	INIT_LIST_HEAD(&ret->b_assoc_buffers);
2875	spin_lock_init(&ret->b_uptodate_lock);
2876	preempt_disable();
2877	__this_cpu_inc(bh_accounting.nr);
2878	recalc_bh_state();
2879	preempt_enable();
2880	}
2881	return ret;
2882	}
2883	EXPORT_SYMBOL(alloc_buffer_head);
2884
2885	void free_buffer_head(struct buffer_head *bh)
2886	{
2887	BUG_ON(!list_empty(&bh->b_assoc_buffers));
2888	kmem_cache_free(bh_cachep, bh);
2889	preempt_disable();
2890	__this_cpu_dec(bh_accounting.nr);
2891	recalc_bh_state();
2892	preempt_enable();
2893	}
2894	EXPORT_SYMBOL(free_buffer_head);
2895
2896	static int buffer_exit_cpu_dead(unsigned int cpu)
2897	{
2898	int i;
2899	struct bh_lru *b = &per_cpu(bh_lrus, cpu);
2900
2901	for (i = 0; i < BH_LRU_SIZE; i++) {
2902	brelse(b->bhs[i]);
2903	b->bhs[i] = NULL;
2904	}
2905	this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
2906	per_cpu(bh_accounting, cpu).nr = 0;
2907	return 0;
2908	}
2909
2910	/**
2911	* bh_uptodate_or_lock - Test whether the buffer is uptodate
2912	* @bh: struct buffer_head
2913	*
2914	* Return true if the buffer is up-to-date and false,
2915	* with the buffer locked, if not.
2916	*/
2917	int bh_uptodate_or_lock(struct buffer_head *bh)
2918	{
2919	if (!buffer_uptodate(bh)) {
2920	lock_buffer(bh);
2921	if (!buffer_uptodate(bh))
2922	return 0;
2923	unlock_buffer(bh);
2924	}
2925	return 1;
2926	}
2927	EXPORT_SYMBOL(bh_uptodate_or_lock);
2928
2929	/**
2930	* __bh_read - Submit read for a locked buffer
2931	* @bh: struct buffer_head
2932	* @op_flags: appending REQ_OP_* flags besides REQ_OP_READ
2933	* @wait: wait until reading finish
2934	*
2935	* Returns zero on success or don't wait, and -EIO on error.
2936	*/
2937	int __bh_read(struct buffer_head *bh, blk_opf_t op_flags, bool wait)
2938	{
2939	int ret = 0;
2940
2941	BUG_ON(!buffer_locked(bh));
2942
2943	bh_submit(bh, REQ_OP_READ \| op_flags, bh_end_read);
2944	if (wait) {
2945	wait_on_buffer(bh);
2946	if (!buffer_uptodate(bh))
2947	ret = -EIO;
2948	}
2949	return ret;
2950	}
2951	EXPORT_SYMBOL(__bh_read);
2952
2953	/**
2954	* __bh_read_batch - Submit read for a batch of unlocked buffers
2955	* @nr: entry number of the buffer batch
2956	* @bhs: a batch of struct buffer_head
2957	* @op_flags: appending REQ_OP_* flags besides REQ_OP_READ
2958	* @force_lock: force to get a lock on the buffer if set, otherwise drops any
2959	* buffer that cannot lock.
2960	*
2961	* Returns zero on success or don't wait, and -EIO on error.
2962	*/
2963	void __bh_read_batch(int nr, struct buffer_head *bhs[],
2964	blk_opf_t op_flags, bool force_lock)
2965	{
2966	int i;
2967
2968	for (i = 0; i < nr; i++) {
2969	struct buffer_head *bh = bhs[i];
2970
2971	if (buffer_uptodate(bh))
2972	continue;
2973
2974	if (force_lock)
2975	lock_buffer(bh);
2976	else
2977	if (!trylock_buffer(bh))
2978	continue;
2979
2980	if (buffer_uptodate(bh)) {
2981	unlock_buffer(bh);
2982	continue;
2983	}
2984
2985	bh_submit(bh, REQ_OP_READ \| op_flags, bh_end_read);
2986	}
2987	}
2988	EXPORT_SYMBOL(__bh_read_batch);
2989
2990	void __init buffer_init(void)
2991	{
2992	unsigned long nrpages;
2993	int ret;
2994
2995	bh_cachep = KMEM_CACHE(buffer_head,
2996	SLAB_RECLAIM_ACCOUNT\|SLAB_PANIC);
2997	/*
2998	* Limit the bh occupancy to 10% of ZONE_NORMAL
2999	*/
3000	nrpages = (nr_free_buffer_pages() * 10) / 100;
3001	max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3002	ret = cpuhp_setup_state_nocalls(CPUHP_FS_BUFF_DEAD, "fs/buffer:dead",
3003	NULL, buffer_exit_cpu_dead);
3004	WARN_ON(ret < 0);
3005	}