linux - fs/buffer.c

3169 lines · 83.5 KiB

1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* linux/fs/buffer.c
4	*
5	* Copyright (C) 1991, 1992, 2002 Linus Torvalds
6	*/
7
8	/*
9	* Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
10	*
11	* Removed a lot of unnecessary code and simplified things now that
12	* the buffer cache isn't our primary cache - Andrew Tridgell 12/96
13	*
14	* Speed up hash, lru, and free list operations. Use gfp() for allocating
15	* hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
16	*
17	* Added 32k buffer block sizes - these are required older ARM systems. - RMK
18	*
19	* async buffer flushing, 1999 Andrea Arcangeli <[email protected]>
20	*/
21
22	#include <linux/kernel.h>
23	#include <linux/sched/signal.h>
24	#include <linux/syscalls.h>
25	#include <linux/fs.h>
26	#include <linux/iomap.h>
27	#include <linux/mm.h>
28	#include <linux/percpu.h>
29	#include <linux/slab.h>
30	#include <linux/capability.h>
31	#include <linux/blkdev.h>
32	#include <linux/file.h>
33	#include <linux/quotaops.h>
34	#include <linux/highmem.h>
35	#include <linux/export.h>
36	#include <linux/backing-dev.h>
37	#include <linux/writeback.h>
38	#include <linux/hash.h>
39	#include <linux/suspend.h>
40	#include <linux/buffer_head.h>
41	#include <linux/task_io_accounting_ops.h>
42	#include <linux/bio.h>
43	#include <linux/cpu.h>
44	#include <linux/bitops.h>
45	#include <linux/mpage.h>
46	#include <linux/bit_spinlock.h>
47	#include <linux/pagevec.h>
48	#include <linux/sched/mm.h>
49	#include <trace/events/block.h>
50	#include <linux/fscrypt.h>
51	#include <linux/fsverity.h>
52	#include <linux/sched/isolation.h>
53
54	#include "internal.h"
55
56	static int fsync_buffers_list(spinlock_t lock, struct list_head list);
57	static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh,
58	enum rw_hint hint, struct writeback_control *wbc);
59
60	#define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
61
62	inline void touch_buffer(struct buffer_head *bh)
63	{
64	trace_block_touch_buffer(bh);
65	folio_mark_accessed(bh->b_folio);
66	}
67	EXPORT_SYMBOL(touch_buffer);
68
69	void __lock_buffer(struct buffer_head *bh)
70	{
71	wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
72	}
73	EXPORT_SYMBOL(__lock_buffer);
74
75	void unlock_buffer(struct buffer_head *bh)
76	{
77	clear_bit_unlock(BH_Lock, &bh->b_state);
78	smp_mb__after_atomic();
79	wake_up_bit(&bh->b_state, BH_Lock);
80	}
81	EXPORT_SYMBOL(unlock_buffer);
82
83	/*
84	* Returns if the folio has dirty or writeback buffers. If all the buffers
85	* are unlocked and clean then the folio_test_dirty information is stale. If
86	* any of the buffers are locked, it is assumed they are locked for IO.
87	*/
88	void buffer_check_dirty_writeback(struct folio *folio,
89	bool dirty, bool writeback)
90	{
91	struct buffer_head head, bh;
92	*dirty = false;
93	*writeback = false;
94
95	BUG_ON(!folio_test_locked(folio));
96
97	head = folio_buffers(folio);
98	if (!head)
99	return;
100
101	if (folio_test_writeback(folio))
102	*writeback = true;
103
104	bh = head;
105	do {
106	if (buffer_locked(bh))
107	*writeback = true;
108
109	if (buffer_dirty(bh))
110	*dirty = true;
111
112	bh = bh->b_this_page;
113	} while (bh != head);
114	}
115
116	/*
117	* Block until a buffer comes unlocked. This doesn't stop it
118	* from becoming locked again - you have to lock it yourself
119	* if you want to preserve its state.
120	*/
121	void __wait_on_buffer(struct buffer_head * bh)
122	{
123	wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
124	}
125	EXPORT_SYMBOL(__wait_on_buffer);
126
127	static void buffer_io_error(struct buffer_head bh, char msg)
128	{
129	if (!test_bit(BH_Quiet, &bh->b_state))
130	printk_ratelimited(KERN_ERR
131	"Buffer I/O error on dev %pg, logical block %llu%s\n",
132	bh->b_bdev, (unsigned long long)bh->b_blocknr, msg);
133	}
134
135	/*
136	* End-of-IO handler helper function which does not touch the bh after
137	* unlocking it.
138	* Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
139	* a race there is benign: unlock_buffer() only use the bh's address for
140	* hashing after unlocking the buffer, so it doesn't actually touch the bh
141	* itself.
142	*/
143	static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
144	{
145	if (uptodate) {
146	set_buffer_uptodate(bh);
147	} else {
148	/* This happens, due to failed read-ahead attempts. */
149	clear_buffer_uptodate(bh);
150	}
151	unlock_buffer(bh);
152	}
153
154	/*
155	* Default synchronous end-of-IO handler.. Just mark it up-to-date and
156	* unlock the buffer.
157	*/
158	void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
159	{
160	__end_buffer_read_notouch(bh, uptodate);
161	put_bh(bh);
162	}
163	EXPORT_SYMBOL(end_buffer_read_sync);
164
165	void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
166	{
167	if (uptodate) {
168	set_buffer_uptodate(bh);
169	} else {
170	buffer_io_error(bh, ", lost sync page write");
171	mark_buffer_write_io_error(bh);
172	clear_buffer_uptodate(bh);
173	}
174	unlock_buffer(bh);
175	put_bh(bh);
176	}
177	EXPORT_SYMBOL(end_buffer_write_sync);
178
179	static struct buffer_head *
180	__find_get_block_slow(struct block_device *bdev, sector_t block, bool atomic)
181	{
182	struct address_space *bd_mapping = bdev->bd_mapping;
183	const int blkbits = bd_mapping->host->i_blkbits;
184	struct buffer_head *ret = NULL;
185	pgoff_t index;
186	struct buffer_head *bh;
187	struct buffer_head *head;
188	struct folio *folio;
189	int all_mapped = 1;
190	static DEFINE_RATELIMIT_STATE(last_warned, HZ, 1);
191
192	index = ((loff_t)block << blkbits) / PAGE_SIZE;
193	folio = __filemap_get_folio(bd_mapping, index, FGP_ACCESSED, 0);
194	if (IS_ERR(folio))
195	goto out;
196
197	/*
198	* Folio lock protects the buffers. Callers that cannot block
199	* will fallback to serializing vs try_to_free_buffers() via
200	* the i_private_lock.
201	*/
202	if (atomic)
203	spin_lock(&bd_mapping->i_private_lock);
204	else
205	folio_lock(folio);
206
207	head = folio_buffers(folio);
208	if (!head)
209	goto out_unlock;
210	/*
211	* Upon a noref migration, the folio lock serializes here;
212	* otherwise bail.
213	*/
214	if (test_bit_acquire(BH_Migrate, &head->b_state)) {
215	WARN_ON(!atomic);
216	goto out_unlock;
217	}
218
219	bh = head;
220	do {
221	if (!buffer_mapped(bh))
222	all_mapped = 0;
223	else if (bh->b_blocknr == block) {
224	ret = bh;
225	get_bh(bh);
226	goto out_unlock;
227	}
228	bh = bh->b_this_page;
229	} while (bh != head);
230
231	/* we might be here because some of the buffers on this page are
232	* not mapped. This is due to various races between
233	* file io on the block device and getblk. It gets dealt with
234	* elsewhere, don't buffer_error if we had some unmapped buffers
235	*/
236	ratelimit_set_flags(&last_warned, RATELIMIT_MSG_ON_RELEASE);
237	if (all_mapped && __ratelimit(&last_warned)) {
238	printk("__find_get_block_slow() failed. block=%llu, "
239	"b_blocknr=%llu, b_state=0x%08lx, b_size=%zu, "
240	"device %pg blocksize: %d\n",
241	(unsigned long long)block,
242	(unsigned long long)bh->b_blocknr,
243	bh->b_state, bh->b_size, bdev,
244	1 << blkbits);
245	}
246	out_unlock:
247	if (atomic)
248	spin_unlock(&bd_mapping->i_private_lock);
249	else
250	folio_unlock(folio);
251	folio_put(folio);
252	out:
253	return ret;
254	}
255
256	static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
257	{
258	unsigned long flags;
259	struct buffer_head *first;
260	struct buffer_head *tmp;
261	struct folio *folio;
262	int folio_uptodate = 1;
263
264	BUG_ON(!buffer_async_read(bh));
265
266	folio = bh->b_folio;
267	if (uptodate) {
268	set_buffer_uptodate(bh);
269	} else {
270	clear_buffer_uptodate(bh);
271	buffer_io_error(bh, ", async page read");
272	}
273
274	/*
275	* Be _very_ careful from here on. Bad things can happen if
276	* two buffer heads end IO at almost the same time and both
277	* decide that the page is now completely done.
278	*/
279	first = folio_buffers(folio);
280	spin_lock_irqsave(&first->b_uptodate_lock, flags);
281	clear_buffer_async_read(bh);
282	unlock_buffer(bh);
283	tmp = bh;
284	do {
285	if (!buffer_uptodate(tmp))
286	folio_uptodate = 0;
287	if (buffer_async_read(tmp)) {
288	BUG_ON(!buffer_locked(tmp));
289	goto still_busy;
290	}
291	tmp = tmp->b_this_page;
292	} while (tmp != bh);
293	spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
294
295	folio_end_read(folio, folio_uptodate);
296	return;
297
298	still_busy:
299	spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
300	}
301
302	struct postprocess_bh_ctx {
303	struct work_struct work;
304	struct buffer_head *bh;
305	};
306
307	static void verify_bh(struct work_struct *work)
308	{
309	struct postprocess_bh_ctx *ctx =
310	container_of(work, struct postprocess_bh_ctx, work);
311	struct buffer_head *bh = ctx->bh;
312	bool valid;
313
314	valid = fsverity_verify_blocks(bh->b_folio, bh->b_size, bh_offset(bh));
315	end_buffer_async_read(bh, valid);
316	kfree(ctx);
317	}
318
319	static bool need_fsverity(struct buffer_head *bh)
320	{
321	struct folio *folio = bh->b_folio;
322	struct inode *inode = folio->mapping->host;
323
324	return fsverity_active(inode) &&
325	/* needed by ext4 */
326	folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
327	}
328
329	static void decrypt_bh(struct work_struct *work)
330	{
331	struct postprocess_bh_ctx *ctx =
332	container_of(work, struct postprocess_bh_ctx, work);
333	struct buffer_head *bh = ctx->bh;
334	int err;
335
336	err = fscrypt_decrypt_pagecache_blocks(bh->b_folio, bh->b_size,
337	bh_offset(bh));
338	if (err == 0 && need_fsverity(bh)) {
339	/*
340	* We use different work queues for decryption and for verity
341	* because verity may require reading metadata pages that need
342	* decryption, and we shouldn't recurse to the same workqueue.
343	*/
344	INIT_WORK(&ctx->work, verify_bh);
345	fsverity_enqueue_verify_work(&ctx->work);
346	return;
347	}
348	end_buffer_async_read(bh, err == 0);
349	kfree(ctx);
350	}
351
352	/*
353	* I/O completion handler for block_read_full_folio() - pages
354	* which come unlocked at the end of I/O.
355	*/
356	static void end_buffer_async_read_io(struct buffer_head *bh, int uptodate)
357	{
358	struct inode *inode = bh->b_folio->mapping->host;
359	bool decrypt = fscrypt_inode_uses_fs_layer_crypto(inode);
360	bool verify = need_fsverity(bh);
361
362	/* Decrypt (with fscrypt) and/or verify (with fsverity) if needed. */
363	if (uptodate && (decrypt \|\| verify)) {
364	struct postprocess_bh_ctx *ctx =
365	kmalloc(sizeof(*ctx), GFP_ATOMIC);
366
367	if (ctx) {
368	ctx->bh = bh;
369	if (decrypt) {
370	INIT_WORK(&ctx->work, decrypt_bh);
371	fscrypt_enqueue_decrypt_work(&ctx->work);
372	} else {
373	INIT_WORK(&ctx->work, verify_bh);
374	fsverity_enqueue_verify_work(&ctx->work);
375	}
376	return;
377	}
378	uptodate = 0;
379	}
380	end_buffer_async_read(bh, uptodate);
381	}
382
383	/*
384	* Completion handler for block_write_full_folio() - folios which are unlocked
385	* during I/O, and which have the writeback flag cleared upon I/O completion.
386	*/
387	static void end_buffer_async_write(struct buffer_head *bh, int uptodate)
388	{
389	unsigned long flags;
390	struct buffer_head *first;
391	struct buffer_head *tmp;
392	struct folio *folio;
393
394	BUG_ON(!buffer_async_write(bh));
395
396	folio = bh->b_folio;
397	if (uptodate) {
398	set_buffer_uptodate(bh);
399	} else {
400	buffer_io_error(bh, ", lost async page write");
401	mark_buffer_write_io_error(bh);
402	clear_buffer_uptodate(bh);
403	}
404
405	first = folio_buffers(folio);
406	spin_lock_irqsave(&first->b_uptodate_lock, flags);
407
408	clear_buffer_async_write(bh);
409	unlock_buffer(bh);
410	tmp = bh->b_this_page;
411	while (tmp != bh) {
412	if (buffer_async_write(tmp)) {
413	BUG_ON(!buffer_locked(tmp));
414	goto still_busy;
415	}
416	tmp = tmp->b_this_page;
417	}
418	spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
419	folio_end_writeback(folio);
420	return;
421
422	still_busy:
423	spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
424	}
425
426	/*
427	* If a page's buffers are under async readin (end_buffer_async_read
428	* completion) then there is a possibility that another thread of
429	* control could lock one of the buffers after it has completed
430	* but while some of the other buffers have not completed. This
431	* locked buffer would confuse end_buffer_async_read() into not unlocking
432	* the page. So the absence of BH_Async_Read tells end_buffer_async_read()
433	* that this buffer is not under async I/O.
434	*
435	* The page comes unlocked when it has no locked buffer_async buffers
436	* left.
437	*
438	* PageLocked prevents anyone starting new async I/O reads any of
439	* the buffers.
440	*
441	* PageWriteback is used to prevent simultaneous writeout of the same
442	* page.
443	*
444	* PageLocked prevents anyone from starting writeback of a page which is
445	* under read I/O (PageWriteback is only ever set against a locked page).
446	*/
447	static void mark_buffer_async_read(struct buffer_head *bh)
448	{
449	bh->b_end_io = end_buffer_async_read_io;
450	set_buffer_async_read(bh);
451	}
452
453	static void mark_buffer_async_write_endio(struct buffer_head *bh,
454	bh_end_io_t *handler)
455	{
456	bh->b_end_io = handler;
457	set_buffer_async_write(bh);
458	}
459
460	void mark_buffer_async_write(struct buffer_head *bh)
461	{
462	mark_buffer_async_write_endio(bh, end_buffer_async_write);
463	}
464	EXPORT_SYMBOL(mark_buffer_async_write);
465
466
467	/*
468	* fs/buffer.c contains helper functions for buffer-backed address space's
469	* fsync functions. A common requirement for buffer-based filesystems is
470	* that certain data from the backing blockdev needs to be written out for
471	* a successful fsync(). For example, ext2 indirect blocks need to be
472	* written back and waited upon before fsync() returns.
473	*
474	* The functions mark_buffer_dirty_inode(), fsync_inode_buffers(),
475	* inode_has_buffers() and invalidate_inode_buffers() are provided for the
476	* management of a list of dependent buffers at ->i_mapping->i_private_list.
477	*
478	* Locking is a little subtle: try_to_free_buffers() will remove buffers
479	* from their controlling inode's queue when they are being freed. But
480	* try_to_free_buffers() will be operating against the blockdev mapping
481	* at the time, not against the S_ISREG file which depends on those buffers.
482	* So the locking for i_private_list is via the i_private_lock in the address_space
483	* which backs the buffers. Which is different from the address_space
484	* against which the buffers are listed. So for a particular address_space,
485	* mapping->i_private_lock does not protect mapping->i_private_list! In fact,
486	* mapping->i_private_list will always be protected by the backing blockdev's
487	* ->i_private_lock.
488	*
489	* Which introduces a requirement: all buffers on an address_space's
490	* ->i_private_list must be from the same address_space: the blockdev's.
491	*
492	* address_spaces which do not place buffers at ->i_private_list via these
493	* utility functions are free to use i_private_lock and i_private_list for
494	* whatever they want. The only requirement is that list_empty(i_private_list)
495	* be true at clear_inode() time.
496	*
497	* FIXME: clear_inode should not call invalidate_inode_buffers(). The
498	* filesystems should do that. invalidate_inode_buffers() should just go
499	* BUG_ON(!list_empty).
500	*
501	* FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
502	* take an address_space, not an inode. And it should be called
503	* mark_buffer_dirty_fsync() to clearly define why those buffers are being
504	* queued up.
505	*
506	* FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
507	* list if it is already on a list. Because if the buffer is on a list,
508	* it must already be on the right one. If not, the filesystem is being
509	* silly. This will save a ton of locking. But first we have to ensure
510	* that buffers are taken off the old inode's list when they are freed
511	* (presumably in truncate). That requires careful auditing of all
512	* filesystems (do it inside bforget()). It could also be done by bringing
513	* b_inode back.
514	*/
515
516	/*
517	* The buffer's backing address_space's i_private_lock must be held
518	*/
519	static void __remove_assoc_queue(struct buffer_head *bh)
520	{
521	list_del_init(&bh->b_assoc_buffers);
522	WARN_ON(!bh->b_assoc_map);
523	bh->b_assoc_map = NULL;
524	}
525
526	int inode_has_buffers(struct inode *inode)
527	{
528	return !list_empty(&inode->i_data.i_private_list);
529	}
530
531	/*
532	* osync is designed to support O_SYNC io. It waits synchronously for
533	* all already-submitted IO to complete, but does not queue any new
534	* writes to the disk.
535	*
536	* To do O_SYNC writes, just queue the buffer writes with write_dirty_buffer
537	* as you dirty the buffers, and then use osync_inode_buffers to wait for
538	* completion. Any other dirty buffers which are not yet queued for
539	* write will not be flushed to disk by the osync.
540	*/
541	static int osync_buffers_list(spinlock_t lock, struct list_head list)
542	{
543	struct buffer_head *bh;
544	struct list_head *p;
545	int err = 0;
546
547	spin_lock(lock);
548	repeat:
549	list_for_each_prev(p, list) {
550	bh = BH_ENTRY(p);
551	if (buffer_locked(bh)) {
552	get_bh(bh);
553	spin_unlock(lock);
554	wait_on_buffer(bh);
555	if (!buffer_uptodate(bh))
556	err = -EIO;
557	brelse(bh);
558	spin_lock(lock);
559	goto repeat;
560	}
561	}
562	spin_unlock(lock);
563	return err;
564	}
565
566	/**
567	* sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
568	* @mapping: the mapping which wants those buffers written
569	*
570	* Starts I/O against the buffers at mapping->i_private_list, and waits upon
571	* that I/O.
572	*
573	* Basically, this is a convenience function for fsync().
574	* @mapping is a file or directory which needs those buffers to be written for
575	* a successful fsync().
576	*/
577	int sync_mapping_buffers(struct address_space *mapping)
578	{
579	struct address_space *buffer_mapping = mapping->i_private_data;
580
581	if (buffer_mapping == NULL \|\| list_empty(&mapping->i_private_list))
582	return 0;
583
584	return fsync_buffers_list(&buffer_mapping->i_private_lock,
585	&mapping->i_private_list);
586	}
587	EXPORT_SYMBOL(sync_mapping_buffers);
588
589	/**
590	* generic_buffers_fsync_noflush - generic buffer fsync implementation
591	* for simple filesystems with no inode lock
592	*
593	* @file: file to synchronize
594	* @start: start offset in bytes
595	* @end: end offset in bytes (inclusive)
596	* @datasync: only synchronize essential metadata if true
597	*
598	* This is a generic implementation of the fsync method for simple
599	* filesystems which track all non-inode metadata in the buffers list
600	* hanging off the address_space structure.
601	*/
602	int generic_buffers_fsync_noflush(struct file *file, loff_t start, loff_t end,
603	bool datasync)
604	{
605	struct inode *inode = file->f_mapping->host;
606	int err;
607	int ret;
608
609	err = file_write_and_wait_range(file, start, end);
610	if (err)
611	return err;
612
613	ret = sync_mapping_buffers(inode->i_mapping);
614	if (!(inode->i_state & I_DIRTY_ALL))
615	goto out;
616	if (datasync && !(inode->i_state & I_DIRTY_DATASYNC))
617	goto out;
618
619	err = sync_inode_metadata(inode, 1);
620	if (ret == 0)
621	ret = err;
622
623	out:
624	/* check and advance again to catch errors after syncing out buffers */
625	err = file_check_and_advance_wb_err(file);
626	if (ret == 0)
627	ret = err;
628	return ret;
629	}
630	EXPORT_SYMBOL(generic_buffers_fsync_noflush);
631
632	/**
633	* generic_buffers_fsync - generic buffer fsync implementation
634	* for simple filesystems with no inode lock
635	*
636	* @file: file to synchronize
637	* @start: start offset in bytes
638	* @end: end offset in bytes (inclusive)
639	* @datasync: only synchronize essential metadata if true
640	*
641	* This is a generic implementation of the fsync method for simple
642	* filesystems which track all non-inode metadata in the buffers list
643	* hanging off the address_space structure. This also makes sure that
644	* a device cache flush operation is called at the end.
645	*/
646	int generic_buffers_fsync(struct file *file, loff_t start, loff_t end,
647	bool datasync)
648	{
649	struct inode *inode = file->f_mapping->host;
650	int ret;
651
652	ret = generic_buffers_fsync_noflush(file, start, end, datasync);
653	if (!ret)
654	ret = blkdev_issue_flush(inode->i_sb->s_bdev);
655	return ret;
656	}
657	EXPORT_SYMBOL(generic_buffers_fsync);
658
659	/*
660	* Called when we've recently written block `bblock', and it is known that
661	* `bblock' was for a buffer_boundary() buffer. This means that the block at
662	* `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
663	* dirty, schedule it for IO. So that indirects merge nicely with their data.
664	*/
665	void write_boundary_block(struct block_device *bdev,
666	sector_t bblock, unsigned blocksize)
667	{
668	struct buffer_head *bh;
669
670	bh = __find_get_block_nonatomic(bdev, bblock + 1, blocksize);
671	if (bh) {
672	if (buffer_dirty(bh))
673	write_dirty_buffer(bh, 0);
674	put_bh(bh);
675	}
676	}
677
678	void mark_buffer_dirty_inode(struct buffer_head bh, struct inode inode)
679	{
680	struct address_space *mapping = inode->i_mapping;
681	struct address_space *buffer_mapping = bh->b_folio->mapping;
682
683	mark_buffer_dirty(bh);
684	if (!mapping->i_private_data) {
685	mapping->i_private_data = buffer_mapping;
686	} else {
687	BUG_ON(mapping->i_private_data != buffer_mapping);
688	}
689	if (!bh->b_assoc_map) {
690	spin_lock(&buffer_mapping->i_private_lock);
691	list_move_tail(&bh->b_assoc_buffers,
692	&mapping->i_private_list);
693	bh->b_assoc_map = mapping;
694	spin_unlock(&buffer_mapping->i_private_lock);
695	}
696	}
697	EXPORT_SYMBOL(mark_buffer_dirty_inode);
698
699	/**
700	* block_dirty_folio - Mark a folio as dirty.
701	* @mapping: The address space containing this folio.
702	* @folio: The folio to mark dirty.
703	*
704	* Filesystems which use buffer_heads can use this function as their
705	* ->dirty_folio implementation. Some filesystems need to do a little
706	* work before calling this function. Filesystems which do not use
707	* buffer_heads should call filemap_dirty_folio() instead.
708	*
709	* If the folio has buffers, the uptodate buffers are set dirty, to
710	* preserve dirty-state coherency between the folio and the buffers.
711	* Buffers added to a dirty folio are created dirty.
712	*
713	* The buffers are dirtied before the folio is dirtied. There's a small
714	* race window in which writeback may see the folio cleanness but not the
715	* buffer dirtiness. That's fine. If this code were to set the folio
716	* dirty before the buffers, writeback could clear the folio dirty flag,
717	* see a bunch of clean buffers and we'd end up with dirty buffers/clean
718	* folio on the dirty folio list.
719	*
720	* We use i_private_lock to lock against try_to_free_buffers() while
721	* using the folio's buffer list. This also prevents clean buffers
722	* being added to the folio after it was set dirty.
723	*
724	* Context: May only be called from process context. Does not sleep.
725	* Caller must ensure that @folio cannot be truncated during this call,
726	* typically by holding the folio lock or having a page in the folio
727	* mapped and holding the page table lock.
728	*
729	* Return: True if the folio was dirtied; false if it was already dirtied.
730	*/
731	bool block_dirty_folio(struct address_space mapping, struct folio folio)
732	{
733	struct buffer_head *head;
734	bool newly_dirty;
735
736	spin_lock(&mapping->i_private_lock);
737	head = folio_buffers(folio);
738	if (head) {
739	struct buffer_head *bh = head;
740
741	do {
742	set_buffer_dirty(bh);
743	bh = bh->b_this_page;
744	} while (bh != head);
745	}
746	/*
747	* Lock out page's memcg migration to keep PageDirty
748	* synchronized with per-memcg dirty page counters.
749	*/
750	newly_dirty = !folio_test_set_dirty(folio);
751	spin_unlock(&mapping->i_private_lock);
752
753	if (newly_dirty)
754	__folio_mark_dirty(folio, mapping, 1);
755
756	if (newly_dirty)
757	__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
758
759	return newly_dirty;
760	}
761	EXPORT_SYMBOL(block_dirty_folio);
762
763	/*
764	* Write out and wait upon a list of buffers.
765	*
766	* We have conflicting pressures: we want to make sure that all
767	* initially dirty buffers get waited on, but that any subsequently
768	* dirtied buffers don't. After all, we don't want fsync to last
769	* forever if somebody is actively writing to the file.
770	*
771	* Do this in two main stages: first we copy dirty buffers to a
772	* temporary inode list, queueing the writes as we go. Then we clean
773	* up, waiting for those writes to complete.
774	*
775	* During this second stage, any subsequent updates to the file may end
776	* up refiling the buffer on the original inode's dirty list again, so
777	* there is a chance we will end up with a buffer queued for write but
778	* not yet completed on that list. So, as a final cleanup we go through
779	* the osync code to catch these locked, dirty buffers without requeuing
780	* any newly dirty buffers for write.
781	*/
782	static int fsync_buffers_list(spinlock_t lock, struct list_head list)
783	{
784	struct buffer_head *bh;
785	struct address_space *mapping;
786	int err = 0, err2;
787	struct blk_plug plug;
788	LIST_HEAD(tmp);
789
790	blk_start_plug(&plug);
791
792	spin_lock(lock);
793	while (!list_empty(list)) {
794	bh = BH_ENTRY(list->next);
795	mapping = bh->b_assoc_map;
796	__remove_assoc_queue(bh);
797	/* Avoid race with mark_buffer_dirty_inode() which does
798	* a lockless check and we rely on seeing the dirty bit */
799	smp_mb();
800	if (buffer_dirty(bh) \|\| buffer_locked(bh)) {
801	list_add(&bh->b_assoc_buffers, &tmp);
802	bh->b_assoc_map = mapping;
803	if (buffer_dirty(bh)) {
804	get_bh(bh);
805	spin_unlock(lock);
806	/*
807	* Ensure any pending I/O completes so that
808	* write_dirty_buffer() actually writes the
809	* current contents - it is a noop if I/O is
810	* still in flight on potentially older
811	* contents.
812	*/
813	write_dirty_buffer(bh, REQ_SYNC);
814
815	/*
816	* Kick off IO for the previous mapping. Note
817	* that we will not run the very last mapping,
818	* wait_on_buffer() will do that for us
819	* through sync_buffer().
820	*/
821	brelse(bh);
822	spin_lock(lock);
823	}
824	}
825	}
826
827	spin_unlock(lock);
828	blk_finish_plug(&plug);
829	spin_lock(lock);
830
831	while (!list_empty(&tmp)) {
832	bh = BH_ENTRY(tmp.prev);
833	get_bh(bh);
834	mapping = bh->b_assoc_map;
835	__remove_assoc_queue(bh);
836	/* Avoid race with mark_buffer_dirty_inode() which does
837	* a lockless check and we rely on seeing the dirty bit */
838	smp_mb();
839	if (buffer_dirty(bh)) {
840	list_add(&bh->b_assoc_buffers,
841	&mapping->i_private_list);
842	bh->b_assoc_map = mapping;
843	}
844	spin_unlock(lock);
845	wait_on_buffer(bh);
846	if (!buffer_uptodate(bh))
847	err = -EIO;
848	brelse(bh);
849	spin_lock(lock);
850	}
851
852	spin_unlock(lock);
853	err2 = osync_buffers_list(lock, list);
854	if (err)
855	return err;
856	else
857	return err2;
858	}
859
860	/*
861	* Invalidate any and all dirty buffers on a given inode. We are
862	* probably unmounting the fs, but that doesn't mean we have already
863	* done a sync(). Just drop the buffers from the inode list.
864	*
865	* NOTE: we take the inode's blockdev's mapping's i_private_lock. Which
866	* assumes that all the buffers are against the blockdev.
867	*/
868	void invalidate_inode_buffers(struct inode *inode)
869	{
870	if (inode_has_buffers(inode)) {
871	struct address_space *mapping = &inode->i_data;
872	struct list_head *list = &mapping->i_private_list;
873	struct address_space *buffer_mapping = mapping->i_private_data;
874
875	spin_lock(&buffer_mapping->i_private_lock);
876	while (!list_empty(list))
877	__remove_assoc_queue(BH_ENTRY(list->next));
878	spin_unlock(&buffer_mapping->i_private_lock);
879	}
880	}
881	EXPORT_SYMBOL(invalidate_inode_buffers);
882
883	/*
884	* Remove any clean buffers from the inode's buffer list. This is called
885	* when we're trying to free the inode itself. Those buffers can pin it.
886	*
887	* Returns true if all buffers were removed.
888	*/
889	int remove_inode_buffers(struct inode *inode)
890	{
891	int ret = 1;
892
893	if (inode_has_buffers(inode)) {
894	struct address_space *mapping = &inode->i_data;
895	struct list_head *list = &mapping->i_private_list;
896	struct address_space *buffer_mapping = mapping->i_private_data;
897
898	spin_lock(&buffer_mapping->i_private_lock);
899	while (!list_empty(list)) {
900	struct buffer_head *bh = BH_ENTRY(list->next);
901	if (buffer_dirty(bh)) {
902	ret = 0;
903	break;
904	}
905	__remove_assoc_queue(bh);
906	}
907	spin_unlock(&buffer_mapping->i_private_lock);
908	}
909	return ret;
910	}
911
912	/*
913	* Create the appropriate buffers when given a folio for data area and
914	* the size of each buffer.. Use the bh->b_this_page linked list to
915	* follow the buffers created. Return NULL if unable to create more
916	* buffers.
917	*
918	* The retry flag is used to differentiate async IO (paging, swapping)
919	* which may not fail from ordinary buffer allocations.
920	*/
921	struct buffer_head folio_alloc_buffers(struct folio folio, unsigned long size,
922	gfp_t gfp)
923	{
924	struct buffer_head bh, head;
925	long offset;
926	struct mem_cgroup memcg, old_memcg;
927
928	/* The folio lock pins the memcg */
929	memcg = folio_memcg(folio);
930	old_memcg = set_active_memcg(memcg);
931
932	head = NULL;
933	offset = folio_size(folio);
934	while ((offset -= size) >= 0) {
935	bh = alloc_buffer_head(gfp);
936	if (!bh)
937	goto no_grow;
938
939	bh->b_this_page = head;
940	bh->b_blocknr = -1;
941	head = bh;
942
943	bh->b_size = size;
944
945	/* Link the buffer to its folio */
946	folio_set_bh(bh, folio, offset);
947	}
948	out:
949	set_active_memcg(old_memcg);
950	return head;
951	/*
952	* In case anything failed, we just free everything we got.
953	*/
954	no_grow:
955	if (head) {
956	do {
957	bh = head;
958	head = head->b_this_page;
959	free_buffer_head(bh);
960	} while (head);
961	}
962
963	goto out;
964	}
965	EXPORT_SYMBOL_GPL(folio_alloc_buffers);
966
967	struct buffer_head alloc_page_buffers(struct page page, unsigned long size)
968	{
969	gfp_t gfp = GFP_NOFS \| __GFP_ACCOUNT;
970
971	return folio_alloc_buffers(page_folio(page), size, gfp);
972	}
973	EXPORT_SYMBOL_GPL(alloc_page_buffers);
974
975	static inline void link_dev_buffers(struct folio *folio,
976	struct buffer_head *head)
977	{
978	struct buffer_head bh, tail;
979
980	bh = head;
981	do {
982	tail = bh;
983	bh = bh->b_this_page;
984	} while (bh);
985	tail->b_this_page = head;
986	folio_attach_private(folio, head);
987	}
988
989	static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
990	{
991	sector_t retval = ~((sector_t)0);
992	loff_t sz = bdev_nr_bytes(bdev);
993
994	if (sz) {
995	unsigned int sizebits = blksize_bits(size);
996	retval = (sz >> sizebits);
997	}
998	return retval;
999	}
1000
1001	/*
1002	* Initialise the state of a blockdev folio's buffers.
1003	*/
1004	static sector_t folio_init_buffers(struct folio *folio,
1005	struct block_device *bdev, unsigned size)
1006	{
1007	struct buffer_head *head = folio_buffers(folio);
1008	struct buffer_head *bh = head;
1009	bool uptodate = folio_test_uptodate(folio);
1010	sector_t block = div_u64(folio_pos(folio), size);
1011	sector_t end_block = blkdev_max_block(bdev, size);
1012
1013	do {
1014	if (!buffer_mapped(bh)) {
1015	bh->b_end_io = NULL;
1016	bh->b_private = NULL;
1017	bh->b_bdev = bdev;
1018	bh->b_blocknr = block;
1019	if (uptodate)
1020	set_buffer_uptodate(bh);
1021	if (block < end_block)
1022	set_buffer_mapped(bh);
1023	}
1024	block++;
1025	bh = bh->b_this_page;
1026	} while (bh != head);
1027
1028	/*
1029	* Caller needs to validate requested block against end of device.
1030	*/
1031	return end_block;
1032	}
1033
1034	/*
1035	* Create the page-cache folio that contains the requested block.
1036	*
1037	* This is used purely for blockdev mappings.
1038	*
1039	* Returns false if we have a failure which cannot be cured by retrying
1040	* without sleeping. Returns true if we succeeded, or the caller should retry.
1041	*/
1042	static bool grow_dev_folio(struct block_device *bdev, sector_t block,
1043	pgoff_t index, unsigned size, gfp_t gfp)
1044	{
1045	struct address_space *mapping = bdev->bd_mapping;
1046	struct folio *folio;
1047	struct buffer_head *bh;
1048	sector_t end_block = 0;
1049
1050	folio = __filemap_get_folio(mapping, index,
1051	FGP_LOCK \| FGP_ACCESSED \| FGP_CREAT, gfp);
1052	if (IS_ERR(folio))
1053	return false;
1054
1055	bh = folio_buffers(folio);
1056	if (bh) {
1057	if (bh->b_size == size) {
1058	end_block = folio_init_buffers(folio, bdev, size);
1059	goto unlock;
1060	}
1061
1062	/*
1063	* Retrying may succeed; for example the folio may finish
1064	* writeback, or buffers may be cleaned. This should not
1065	* happen very often; maybe we have old buffers attached to
1066	* this blockdev's page cache and we're trying to change
1067	* the block size?
1068	*/
1069	if (!try_to_free_buffers(folio)) {
1070	end_block = ~0ULL;
1071	goto unlock;
1072	}
1073	}
1074
1075	bh = folio_alloc_buffers(folio, size, gfp \| __GFP_ACCOUNT);
1076	if (!bh)
1077	goto unlock;
1078
1079	/*
1080	* Link the folio to the buffers and initialise them. Take the
1081	* lock to be atomic wrt __find_get_block(), which does not
1082	* run under the folio lock.
1083	*/
1084	spin_lock(&mapping->i_private_lock);
1085	link_dev_buffers(folio, bh);
1086	end_block = folio_init_buffers(folio, bdev, size);
1087	spin_unlock(&mapping->i_private_lock);
1088	unlock:
1089	folio_unlock(folio);
1090	folio_put(folio);
1091	return block < end_block;
1092	}
1093
1094	/*
1095	* Create buffers for the specified block device block's folio. If
1096	* that folio was dirty, the buffers are set dirty also. Returns false
1097	* if we've hit a permanent error.
1098	*/
1099	static bool grow_buffers(struct block_device *bdev, sector_t block,
1100	unsigned size, gfp_t gfp)
1101	{
1102	loff_t pos;
1103
1104	/*
1105	* Check for a block which lies outside our maximum possible
1106	* pagecache index.
1107	*/
1108	if (check_mul_overflow(block, (sector_t)size, &pos) \|\| pos > MAX_LFS_FILESIZE) {
1109	printk(KERN_ERR "%s: requested out-of-range block %llu for device %pg\n",
1110	__func__, (unsigned long long)block,
1111	bdev);
1112	return false;
1113	}
1114
1115	/* Create a folio with the proper size buffers */
1116	return grow_dev_folio(bdev, block, pos / PAGE_SIZE, size, gfp);
1117	}
1118
1119	static struct buffer_head *
1120	__getblk_slow(struct block_device *bdev, sector_t block,
1121	unsigned size, gfp_t gfp)
1122	{
1123	bool blocking = gfpflags_allow_blocking(gfp);
1124
1125	/* Size must be multiple of hard sectorsize */
1126	if (unlikely(size & (bdev_logical_block_size(bdev)-1) \|\|
1127	(size < 512 \|\| size > PAGE_SIZE))) {
1128	printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1129	size);
1130	printk(KERN_ERR "logical block size: %d\n",
1131	bdev_logical_block_size(bdev));
1132
1133	dump_stack();
1134	return NULL;
1135	}
1136
1137	for (;;) {
1138	struct buffer_head *bh;
1139
1140	if (!grow_buffers(bdev, block, size, gfp))
1141	return NULL;
1142
1143	if (blocking)
1144	bh = __find_get_block_nonatomic(bdev, block, size);
1145	else
1146	bh = __find_get_block(bdev, block, size);
1147	if (bh)
1148	return bh;
1149	}
1150	}
1151
1152	/*
1153	* The relationship between dirty buffers and dirty pages:
1154	*
1155	* Whenever a page has any dirty buffers, the page's dirty bit is set, and
1156	* the page is tagged dirty in the page cache.
1157	*
1158	* At all times, the dirtiness of the buffers represents the dirtiness of
1159	* subsections of the page. If the page has buffers, the page dirty bit is
1160	* merely a hint about the true dirty state.
1161	*
1162	* When a page is set dirty in its entirety, all its buffers are marked dirty
1163	* (if the page has buffers).
1164	*
1165	* When a buffer is marked dirty, its page is dirtied, but the page's other
1166	* buffers are not.
1167	*
1168	* Also. When blockdev buffers are explicitly read with bread(), they
1169	* individually become uptodate. But their backing page remains not
1170	* uptodate - even if all of its buffers are uptodate. A subsequent
1171	* block_read_full_folio() against that folio will discover all the uptodate
1172	* buffers, will set the folio uptodate and will perform no I/O.
1173	*/
1174
1175	/**
1176	* mark_buffer_dirty - mark a buffer_head as needing writeout
1177	* @bh: the buffer_head to mark dirty
1178	*
1179	* mark_buffer_dirty() will set the dirty bit against the buffer, then set
1180	* its backing page dirty, then tag the page as dirty in the page cache
1181	* and then attach the address_space's inode to its superblock's dirty
1182	* inode list.
1183	*
1184	* mark_buffer_dirty() is atomic. It takes bh->b_folio->mapping->i_private_lock,
1185	* i_pages lock and mapping->host->i_lock.
1186	*/
1187	void mark_buffer_dirty(struct buffer_head *bh)
1188	{
1189	WARN_ON_ONCE(!buffer_uptodate(bh));
1190
1191	trace_block_dirty_buffer(bh);
1192
1193	/*
1194	* Very carefully optimize the it-is-already-dirty case.
1195	*
1196	* Don't let the final "is it dirty" escape to before we
1197	* perhaps modified the buffer.
1198	*/
1199	if (buffer_dirty(bh)) {
1200	smp_mb();
1201	if (buffer_dirty(bh))
1202	return;
1203	}
1204
1205	if (!test_set_buffer_dirty(bh)) {
1206	struct folio *folio = bh->b_folio;
1207	struct address_space *mapping = NULL;
1208
1209	if (!folio_test_set_dirty(folio)) {
1210	mapping = folio->mapping;
1211	if (mapping)
1212	__folio_mark_dirty(folio, mapping, 0);
1213	}
1214	if (mapping)
1215	__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
1216	}
1217	}
1218	EXPORT_SYMBOL(mark_buffer_dirty);
1219
1220	void mark_buffer_write_io_error(struct buffer_head *bh)
1221	{
1222	set_buffer_write_io_error(bh);
1223	/* FIXME: do we need to set this in both places? */
1224	if (bh->b_folio && bh->b_folio->mapping)
1225	mapping_set_error(bh->b_folio->mapping, -EIO);
1226	if (bh->b_assoc_map)
1227	mapping_set_error(bh->b_assoc_map, -EIO);
1228	}
1229	EXPORT_SYMBOL(mark_buffer_write_io_error);
1230
1231	/**
1232	* __brelse - Release a buffer.
1233	* @bh: The buffer to release.
1234	*
1235	* This variant of brelse() can be called if @bh is guaranteed to not be NULL.
1236	*/
1237	void __brelse(struct buffer_head *bh)
1238	{
1239	if (atomic_read(&bh->b_count)) {
1240	put_bh(bh);
1241	return;
1242	}
1243	WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1244	}
1245	EXPORT_SYMBOL(__brelse);
1246
1247	/**
1248	* __bforget - Discard any dirty data in a buffer.
1249	* @bh: The buffer to forget.
1250	*
1251	* This variant of bforget() can be called if @bh is guaranteed to not
1252	* be NULL.
1253	*/
1254	void __bforget(struct buffer_head *bh)
1255	{
1256	clear_buffer_dirty(bh);
1257	if (bh->b_assoc_map) {
1258	struct address_space *buffer_mapping = bh->b_folio->mapping;
1259
1260	spin_lock(&buffer_mapping->i_private_lock);
1261	list_del_init(&bh->b_assoc_buffers);
1262	bh->b_assoc_map = NULL;
1263	spin_unlock(&buffer_mapping->i_private_lock);
1264	}
1265	__brelse(bh);
1266	}
1267	EXPORT_SYMBOL(__bforget);
1268
1269	static struct buffer_head __bread_slow(struct buffer_head bh)
1270	{
1271	lock_buffer(bh);
1272	if (buffer_uptodate(bh)) {
1273	unlock_buffer(bh);
1274	return bh;
1275	} else {
1276	get_bh(bh);
1277	bh->b_end_io = end_buffer_read_sync;
1278	submit_bh(REQ_OP_READ, bh);
1279	wait_on_buffer(bh);
1280	if (buffer_uptodate(bh))
1281	return bh;
1282	}
1283	brelse(bh);
1284	return NULL;
1285	}
1286
1287	/*
1288	* Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
1289	* The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
1290	* refcount elevated by one when they're in an LRU. A buffer can only appear
1291	* once in a particular CPU's LRU. A single buffer can be present in multiple
1292	* CPU's LRUs at the same time.
1293	*
1294	* This is a transparent caching front-end to sb_bread(), sb_getblk() and
1295	* sb_find_get_block().
1296	*
1297	* The LRUs themselves only need locking against invalidate_bh_lrus. We use
1298	* a local interrupt disable for that.
1299	*/
1300
1301	#define BH_LRU_SIZE 16
1302
1303	struct bh_lru {
1304	struct buffer_head *bhs[BH_LRU_SIZE];
1305	};
1306
1307	static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1308
1309	#ifdef CONFIG_SMP
1310	#define bh_lru_lock() local_irq_disable()
1311	#define bh_lru_unlock() local_irq_enable()
1312	#else
1313	#define bh_lru_lock() preempt_disable()
1314	#define bh_lru_unlock() preempt_enable()
1315	#endif
1316
1317	static inline void check_irqs_on(void)
1318	{
1319	#ifdef irqs_disabled
1320	BUG_ON(irqs_disabled());
1321	#endif
1322	}
1323
1324	/*
1325	* Install a buffer_head into this cpu's LRU. If not already in the LRU, it is
1326	* inserted at the front, and the buffer_head at the back if any is evicted.
1327	* Or, if already in the LRU it is moved to the front.
1328	*/
1329	static void bh_lru_install(struct buffer_head *bh)
1330	{
1331	struct buffer_head *evictee = bh;
1332	struct bh_lru *b;
1333	int i;
1334
1335	check_irqs_on();
1336	bh_lru_lock();
1337
1338	/*
1339	* the refcount of buffer_head in bh_lru prevents dropping the
1340	* attached page(i.e., try_to_free_buffers) so it could cause
1341	* failing page migration.
1342	* Skip putting upcoming bh into bh_lru until migration is done.
1343	*/
1344	if (lru_cache_disabled() \|\| cpu_is_isolated(smp_processor_id())) {
1345	bh_lru_unlock();
1346	return;
1347	}
1348
1349	b = this_cpu_ptr(&bh_lrus);
1350	for (i = 0; i < BH_LRU_SIZE; i++) {
1351	swap(evictee, b->bhs[i]);
1352	if (evictee == bh) {
1353	bh_lru_unlock();
1354	return;
1355	}
1356	}
1357
1358	get_bh(bh);
1359	bh_lru_unlock();
1360	brelse(evictee);
1361	}
1362
1363	/*
1364	* Look up the bh in this cpu's LRU. If it's there, move it to the head.
1365	*/
1366	static struct buffer_head *
1367	lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
1368	{
1369	struct buffer_head *ret = NULL;
1370	unsigned int i;
1371
1372	check_irqs_on();
1373	bh_lru_lock();
1374	if (cpu_is_isolated(smp_processor_id())) {
1375	bh_lru_unlock();
1376	return NULL;
1377	}
1378	for (i = 0; i < BH_LRU_SIZE; i++) {
1379	struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
1380
1381	if (bh && bh->b_blocknr == block && bh->b_bdev == bdev &&
1382	bh->b_size == size) {
1383	if (i) {
1384	while (i) {
1385	__this_cpu_write(bh_lrus.bhs[i],
1386	__this_cpu_read(bh_lrus.bhs[i - 1]));
1387	i--;
1388	}
1389	__this_cpu_write(bh_lrus.bhs[0], bh);
1390	}
1391	get_bh(bh);
1392	ret = bh;
1393	break;
1394	}
1395	}
1396	bh_lru_unlock();
1397	return ret;
1398	}
1399
1400	/*
1401	* Perform a pagecache lookup for the matching buffer. If it's there, refresh
1402	* it in the LRU and mark it as accessed. If it is not present then return
1403	* NULL. Atomic context callers may also return NULL if the buffer is being
1404	* migrated; similarly the page is not marked accessed either.
1405	*/
1406	static struct buffer_head *
1407	find_get_block_common(struct block_device *bdev, sector_t block,
1408	unsigned size, bool atomic)
1409	{
1410	struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1411
1412	if (bh == NULL) {
1413	/* __find_get_block_slow will mark the page accessed */
1414	bh = __find_get_block_slow(bdev, block, atomic);
1415	if (bh)
1416	bh_lru_install(bh);
1417	} else
1418	touch_buffer(bh);
1419
1420	return bh;
1421	}
1422
1423	struct buffer_head *
1424	__find_get_block(struct block_device *bdev, sector_t block, unsigned size)
1425	{
1426	return find_get_block_common(bdev, block, size, true);
1427	}
1428	EXPORT_SYMBOL(__find_get_block);
1429
1430	/* same as __find_get_block() but allows sleeping contexts */
1431	struct buffer_head *
1432	__find_get_block_nonatomic(struct block_device *bdev, sector_t block,
1433	unsigned size)
1434	{
1435	return find_get_block_common(bdev, block, size, false);
1436	}
1437	EXPORT_SYMBOL(__find_get_block_nonatomic);
1438
1439	/**
1440	* bdev_getblk - Get a buffer_head in a block device's buffer cache.
1441	* @bdev: The block device.
1442	* @block: The block number.
1443	* @size: The size of buffer_heads for this @bdev.
1444	* @gfp: The memory allocation flags to use.
1445	*
1446	* The returned buffer head has its reference count incremented, but is
1447	* not locked. The caller should call brelse() when it has finished
1448	* with the buffer. The buffer may not be uptodate. If needed, the
1449	* caller can bring it uptodate either by reading it or overwriting it.
1450	*
1451	* Return: The buffer head, or NULL if memory could not be allocated.
1452	*/
1453	struct buffer_head bdev_getblk(struct block_device bdev, sector_t block,
1454	unsigned size, gfp_t gfp)
1455	{
1456	struct buffer_head *bh;
1457
1458	if (gfpflags_allow_blocking(gfp))
1459	bh = __find_get_block_nonatomic(bdev, block, size);
1460	else
1461	bh = __find_get_block(bdev, block, size);
1462
1463	might_alloc(gfp);
1464	if (bh)
1465	return bh;
1466
1467	return __getblk_slow(bdev, block, size, gfp);
1468	}
1469	EXPORT_SYMBOL(bdev_getblk);
1470
1471	/*
1472	* Do async read-ahead on a buffer..
1473	*/
1474	void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
1475	{
1476	struct buffer_head *bh = bdev_getblk(bdev, block, size,
1477	GFP_NOWAIT \| __GFP_MOVABLE);
1478
1479	if (likely(bh)) {
1480	bh_readahead(bh, REQ_RAHEAD);
1481	brelse(bh);
1482	}
1483	}
1484	EXPORT_SYMBOL(__breadahead);
1485
1486	/**
1487	* __bread_gfp() - Read a block.
1488	* @bdev: The block device to read from.
1489	* @block: Block number in units of block size.
1490	* @size: The block size of this device in bytes.
1491	* @gfp: Not page allocation flags; see below.
1492	*
1493	* You are not expected to call this function. You should use one of
1494	* sb_bread(), sb_bread_unmovable() or __bread().
1495	*
1496	* Read a specified block, and return the buffer head that refers to it.
1497	* If @gfp is 0, the memory will be allocated using the block device's
1498	* default GFP flags. If @gfp is __GFP_MOVABLE, the memory may be
1499	* allocated from a movable area. Do not pass in a complete set of
1500	* GFP flags.
1501	*
1502	* The returned buffer head has its refcount increased. The caller should
1503	* call brelse() when it has finished with the buffer.
1504	*
1505	* Context: May sleep waiting for I/O.
1506	* Return: NULL if the block was unreadable.
1507	*/
1508	struct buffer_head __bread_gfp(struct block_device bdev, sector_t block,
1509	unsigned size, gfp_t gfp)
1510	{
1511	struct buffer_head *bh;
1512
1513	gfp \|= mapping_gfp_constraint(bdev->bd_mapping, ~__GFP_FS);
1514
1515	/*
1516	* Prefer looping in the allocator rather than here, at least that
1517	* code knows what it's doing.
1518	*/
1519	gfp \|= __GFP_NOFAIL;
1520
1521	bh = bdev_getblk(bdev, block, size, gfp);
1522
1523	if (likely(bh) && !buffer_uptodate(bh))
1524	bh = __bread_slow(bh);
1525	return bh;
1526	}
1527	EXPORT_SYMBOL(__bread_gfp);
1528
1529	static void __invalidate_bh_lrus(struct bh_lru *b)
1530	{
1531	int i;
1532
1533	for (i = 0; i < BH_LRU_SIZE; i++) {
1534	brelse(b->bhs[i]);
1535	b->bhs[i] = NULL;
1536	}
1537	}
1538	/*
1539	* invalidate_bh_lrus() is called rarely - but not only at unmount.
1540	* This doesn't race because it runs in each cpu either in irq
1541	* or with preempt disabled.
1542	*/
1543	static void invalidate_bh_lru(void *arg)
1544	{
1545	struct bh_lru *b = &get_cpu_var(bh_lrus);
1546
1547	__invalidate_bh_lrus(b);
1548	put_cpu_var(bh_lrus);
1549	}
1550
1551	bool has_bh_in_lru(int cpu, void *dummy)
1552	{
1553	struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
1554	int i;
1555
1556	for (i = 0; i < BH_LRU_SIZE; i++) {
1557	if (b->bhs[i])
1558	return true;
1559	}
1560
1561	return false;
1562	}
1563
1564	void invalidate_bh_lrus(void)
1565	{
1566	on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1);
1567	}
1568	EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
1569
1570	/*
1571	* It's called from workqueue context so we need a bh_lru_lock to close
1572	* the race with preemption/irq.
1573	*/
1574	void invalidate_bh_lrus_cpu(void)
1575	{
1576	struct bh_lru *b;
1577
1578	bh_lru_lock();
1579	b = this_cpu_ptr(&bh_lrus);
1580	__invalidate_bh_lrus(b);
1581	bh_lru_unlock();
1582	}
1583
1584	void folio_set_bh(struct buffer_head bh, struct folio folio,
1585	unsigned long offset)
1586	{
1587	bh->b_folio = folio;
1588	BUG_ON(offset >= folio_size(folio));
1589	if (folio_test_highmem(folio))
1590	/*
1591	* This catches illegal uses and preserves the offset:
1592	*/
1593	bh->b_data = (char *)(0 + offset);
1594	else
1595	bh->b_data = folio_address(folio) + offset;
1596	}
1597	EXPORT_SYMBOL(folio_set_bh);
1598
1599	/*
1600	* Called when truncating a buffer on a page completely.
1601	*/
1602
1603	/* Bits that are cleared during an invalidate */
1604	#define BUFFER_FLAGS_DISCARD \
1605	(1 << BH_Mapped \| 1 << BH_New \| 1 << BH_Req \| \
1606	1 << BH_Delay \| 1 << BH_Unwritten)
1607
1608	static void discard_buffer(struct buffer_head * bh)
1609	{
1610	unsigned long b_state;
1611
1612	lock_buffer(bh);
1613	clear_buffer_dirty(bh);
1614	bh->b_bdev = NULL;
1615	b_state = READ_ONCE(bh->b_state);
1616	do {
1617	} while (!try_cmpxchg_relaxed(&bh->b_state, &b_state,
1618	b_state & ~BUFFER_FLAGS_DISCARD));
1619	unlock_buffer(bh);
1620	}
1621
1622	/**
1623	* block_invalidate_folio - Invalidate part or all of a buffer-backed folio.
1624	* @folio: The folio which is affected.
1625	* @offset: start of the range to invalidate
1626	* @length: length of the range to invalidate
1627	*
1628	* block_invalidate_folio() is called when all or part of the folio has been
1629	* invalidated by a truncate operation.
1630	*
1631	* block_invalidate_folio() does not have to release all buffers, but it must
1632	* ensure that no dirty buffer is left outside @offset and that no I/O
1633	* is underway against any of the blocks which are outside the truncation
1634	* point. Because the caller is about to free (and possibly reuse) those
1635	* blocks on-disk.
1636	*/
1637	void block_invalidate_folio(struct folio *folio, size_t offset, size_t length)
1638	{
1639	struct buffer_head head, bh, *next;
1640	size_t curr_off = 0;
1641	size_t stop = length + offset;
1642
1643	BUG_ON(!folio_test_locked(folio));
1644
1645	/*
1646	* Check for overflow
1647	*/
1648	BUG_ON(stop > folio_size(folio) \|\| stop < length);
1649
1650	head = folio_buffers(folio);
1651	if (!head)
1652	return;
1653
1654	bh = head;
1655	do {
1656	size_t next_off = curr_off + bh->b_size;
1657	next = bh->b_this_page;
1658
1659	/*
1660	* Are we still fully in range ?
1661	*/
1662	if (next_off > stop)
1663	goto out;
1664
1665	/*
1666	* is this block fully invalidated?
1667	*/
1668	if (offset <= curr_off)
1669	discard_buffer(bh);
1670	curr_off = next_off;
1671	bh = next;
1672	} while (bh != head);
1673
1674	/*
1675	* We release buffers only if the entire folio is being invalidated.
1676	* The get_block cached value has been unconditionally invalidated,
1677	* so real IO is not possible anymore.
1678	*/
1679	if (length == folio_size(folio))
1680	filemap_release_folio(folio, 0);
1681	out:
1682	folio_clear_mappedtodisk(folio);
1683	}
1684	EXPORT_SYMBOL(block_invalidate_folio);
1685
1686	/*
1687	* We attach and possibly dirty the buffers atomically wrt
1688	* block_dirty_folio() via i_private_lock. try_to_free_buffers
1689	* is already excluded via the folio lock.
1690	*/
1691	struct buffer_head create_empty_buffers(struct folio folio,
1692	unsigned long blocksize, unsigned long b_state)
1693	{
1694	struct buffer_head bh, head, *tail;
1695	gfp_t gfp = GFP_NOFS \| __GFP_ACCOUNT \| __GFP_NOFAIL;
1696
1697	head = folio_alloc_buffers(folio, blocksize, gfp);
1698	bh = head;
1699	do {
1700	bh->b_state \|= b_state;
1701	tail = bh;
1702	bh = bh->b_this_page;
1703	} while (bh);
1704	tail->b_this_page = head;
1705
1706	spin_lock(&folio->mapping->i_private_lock);
1707	if (folio_test_uptodate(folio) \|\| folio_test_dirty(folio)) {
1708	bh = head;
1709	do {
1710	if (folio_test_dirty(folio))
1711	set_buffer_dirty(bh);
1712	if (folio_test_uptodate(folio))
1713	set_buffer_uptodate(bh);
1714	bh = bh->b_this_page;
1715	} while (bh != head);
1716	}
1717	folio_attach_private(folio, head);
1718	spin_unlock(&folio->mapping->i_private_lock);
1719
1720	return head;
1721	}
1722	EXPORT_SYMBOL(create_empty_buffers);
1723
1724	/**
1725	* clean_bdev_aliases: clean a range of buffers in block device
1726	* @bdev: Block device to clean buffers in
1727	* @block: Start of a range of blocks to clean
1728	* @len: Number of blocks to clean
1729	*
1730	* We are taking a range of blocks for data and we don't want writeback of any
1731	* buffer-cache aliases starting from return from this function and until the
1732	* moment when something will explicitly mark the buffer dirty (hopefully that
1733	* will not happen until we will free that block ;-) We don't even need to mark
1734	* it not-uptodate - nobody can expect anything from a newly allocated buffer
1735	* anyway. We used to use unmap_buffer() for such invalidation, but that was
1736	* wrong. We definitely don't want to mark the alias unmapped, for example - it
1737	* would confuse anyone who might pick it with bread() afterwards...
1738	*
1739	* Also.. Note that bforget() doesn't lock the buffer. So there can be
1740	* writeout I/O going on against recently-freed buffers. We don't wait on that
1741	* I/O in bforget() - it's more efficient to wait on the I/O only if we really
1742	* need to. That happens here.
1743	*/
1744	void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
1745	{
1746	struct address_space *bd_mapping = bdev->bd_mapping;
1747	const int blkbits = bd_mapping->host->i_blkbits;
1748	struct folio_batch fbatch;
1749	pgoff_t index = ((loff_t)block << blkbits) / PAGE_SIZE;
1750	pgoff_t end;
1751	int i, count;
1752	struct buffer_head *bh;
1753	struct buffer_head *head;
1754
1755	end = ((loff_t)(block + len - 1) << blkbits) / PAGE_SIZE;
1756	folio_batch_init(&fbatch);
1757	while (filemap_get_folios(bd_mapping, &index, end, &fbatch)) {
1758	count = folio_batch_count(&fbatch);
1759	for (i = 0; i < count; i++) {
1760	struct folio *folio = fbatch.folios[i];
1761
1762	if (!folio_buffers(folio))
1763	continue;
1764	/*
1765	* We use folio lock instead of bd_mapping->i_private_lock
1766	* to pin buffers here since we can afford to sleep and
1767	* it scales better than a global spinlock lock.
1768	*/
1769	folio_lock(folio);
1770	/* Recheck when the folio is locked which pins bhs */
1771	head = folio_buffers(folio);
1772	if (!head)
1773	goto unlock_page;
1774	bh = head;
1775	do {
1776	if (!buffer_mapped(bh) \|\| (bh->b_blocknr < block))
1777	goto next;
1778	if (bh->b_blocknr >= block + len)
1779	break;
1780	clear_buffer_dirty(bh);
1781	wait_on_buffer(bh);
1782	clear_buffer_req(bh);
1783	next:
1784	bh = bh->b_this_page;
1785	} while (bh != head);
1786	unlock_page:
1787	folio_unlock(folio);
1788	}
1789	folio_batch_release(&fbatch);
1790	cond_resched();
1791	/* End of range already reached? */
1792	if (index > end \|\| !index)
1793	break;
1794	}
1795	}
1796	EXPORT_SYMBOL(clean_bdev_aliases);
1797
1798	static struct buffer_head folio_create_buffers(struct folio folio,
1799	struct inode *inode,
1800	unsigned int b_state)
1801	{
1802	struct buffer_head *bh;
1803
1804	BUG_ON(!folio_test_locked(folio));
1805
1806	bh = folio_buffers(folio);
1807	if (!bh)
1808	bh = create_empty_buffers(folio,
1809	1 << READ_ONCE(inode->i_blkbits), b_state);
1810	return bh;
1811	}
1812
1813	/*
1814	* NOTE! All mapped/uptodate combinations are valid:
1815	*
1816	* Mapped Uptodate Meaning
1817	*
1818	* No No "unknown" - must do get_block()
1819	* No Yes "hole" - zero-filled
1820	* Yes No "allocated" - allocated on disk, not read in
1821	* Yes Yes "valid" - allocated and up-to-date in memory.
1822	*
1823	* "Dirty" is valid only with the last case (mapped+uptodate).
1824	*/
1825
1826	/*
1827	* While block_write_full_folio is writing back the dirty buffers under
1828	* the page lock, whoever dirtied the buffers may decide to clean them
1829	* again at any time. We handle that by only looking at the buffer
1830	* state inside lock_buffer().
1831	*
1832	* If block_write_full_folio() is called for regular writeback
1833	* (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1834	* locked buffer. This only can happen if someone has written the buffer
1835	* directly, with submit_bh(). At the address_space level PageWriteback
1836	* prevents this contention from occurring.
1837	*
1838	* If block_write_full_folio() is called with wbc->sync_mode ==
1839	* WB_SYNC_ALL, the writes are posted using REQ_SYNC; this
1840	* causes the writes to be flagged as synchronous writes.
1841	*/
1842	int __block_write_full_folio(struct inode inode, struct folio folio,
1843	get_block_t get_block, struct writeback_control wbc)
1844	{
1845	int err;
1846	sector_t block;
1847	sector_t last_block;
1848	struct buffer_head bh, head;
1849	size_t blocksize;
1850	int nr_underway = 0;
1851	blk_opf_t write_flags = wbc_to_write_flags(wbc);
1852
1853	head = folio_create_buffers(folio, inode,
1854	(1 << BH_Dirty) \| (1 << BH_Uptodate));
1855
1856	/*
1857	* Be very careful. We have no exclusion from block_dirty_folio
1858	* here, and the (potentially unmapped) buffers may become dirty at
1859	* any time. If a buffer becomes dirty here after we've inspected it
1860	* then we just miss that fact, and the folio stays dirty.
1861	*
1862	* Buffers outside i_size may be dirtied by block_dirty_folio;
1863	* handle that here by just cleaning them.
1864	*/
1865
1866	bh = head;
1867	blocksize = bh->b_size;
1868
1869	block = div_u64(folio_pos(folio), blocksize);
1870	last_block = div_u64(i_size_read(inode) - 1, blocksize);
1871
1872	/*
1873	* Get all the dirty buffers mapped to disk addresses and
1874	* handle any aliases from the underlying blockdev's mapping.
1875	*/
1876	do {
1877	if (block > last_block) {
1878	/*
1879	* mapped buffers outside i_size will occur, because
1880	* this folio can be outside i_size when there is a
1881	* truncate in progress.
1882	*/
1883	/*
1884	* The buffer was zeroed by block_write_full_folio()
1885	*/
1886	clear_buffer_dirty(bh);
1887	set_buffer_uptodate(bh);
1888	} else if ((!buffer_mapped(bh) \|\| buffer_delay(bh)) &&
1889	buffer_dirty(bh)) {
1890	WARN_ON(bh->b_size != blocksize);
1891	err = get_block(inode, block, bh, 1);
1892	if (err)
1893	goto recover;
1894	clear_buffer_delay(bh);
1895	if (buffer_new(bh)) {
1896	/* blockdev mappings never come here */
1897	clear_buffer_new(bh);
1898	clean_bdev_bh_alias(bh);
1899	}
1900	}
1901	bh = bh->b_this_page;
1902	block++;
1903	} while (bh != head);
1904
1905	do {
1906	if (!buffer_mapped(bh))
1907	continue;
1908	/*
1909	* If it's a fully non-blocking write attempt and we cannot
1910	* lock the buffer then redirty the folio. Note that this can
1911	* potentially cause a busy-wait loop from writeback threads
1912	* and kswapd activity, but those code paths have their own
1913	* higher-level throttling.
1914	*/
1915	if (wbc->sync_mode != WB_SYNC_NONE) {
1916	lock_buffer(bh);
1917	} else if (!trylock_buffer(bh)) {
1918	folio_redirty_for_writepage(wbc, folio);
1919	continue;
1920	}
1921	if (test_clear_buffer_dirty(bh)) {
1922	mark_buffer_async_write_endio(bh,
1923	end_buffer_async_write);
1924	} else {
1925	unlock_buffer(bh);
1926	}
1927	} while ((bh = bh->b_this_page) != head);
1928
1929	/*
1930	* The folio and its buffers are protected by the writeback flag,
1931	* so we can drop the bh refcounts early.
1932	*/
1933	BUG_ON(folio_test_writeback(folio));
1934	folio_start_writeback(folio);
1935
1936	do {
1937	struct buffer_head *next = bh->b_this_page;
1938	if (buffer_async_write(bh)) {
1939	submit_bh_wbc(REQ_OP_WRITE \| write_flags, bh,
1940	inode->i_write_hint, wbc);
1941	nr_underway++;
1942	}
1943	bh = next;
1944	} while (bh != head);
1945	folio_unlock(folio);
1946
1947	err = 0;
1948	done:
1949	if (nr_underway == 0) {
1950	/*
1951	* The folio was marked dirty, but the buffers were
1952	* clean. Someone wrote them back by hand with
1953	* write_dirty_buffer/submit_bh. A rare case.
1954	*/
1955	folio_end_writeback(folio);
1956
1957	/*
1958	* The folio and buffer_heads can be released at any time from
1959	* here on.
1960	*/
1961	}
1962	return err;
1963
1964	recover:
1965	/*
1966	* ENOSPC, or some other error. We may already have added some
1967	* blocks to the file, so we need to write these out to avoid
1968	* exposing stale data.
1969	* The folio is currently locked and not marked for writeback
1970	*/
1971	bh = head;
1972	/* Recovery: lock and submit the mapped buffers */
1973	do {
1974	if (buffer_mapped(bh) && buffer_dirty(bh) &&
1975	!buffer_delay(bh)) {
1976	lock_buffer(bh);
1977	mark_buffer_async_write_endio(bh,
1978	end_buffer_async_write);
1979	} else {
1980	/*
1981	* The buffer may have been set dirty during
1982	* attachment to a dirty folio.
1983	*/
1984	clear_buffer_dirty(bh);
1985	}
1986	} while ((bh = bh->b_this_page) != head);
1987	BUG_ON(folio_test_writeback(folio));
1988	mapping_set_error(folio->mapping, err);
1989	folio_start_writeback(folio);
1990	do {
1991	struct buffer_head *next = bh->b_this_page;
1992	if (buffer_async_write(bh)) {
1993	clear_buffer_dirty(bh);
1994	submit_bh_wbc(REQ_OP_WRITE \| write_flags, bh,
1995	inode->i_write_hint, wbc);
1996	nr_underway++;
1997	}
1998	bh = next;
1999	} while (bh != head);
2000	folio_unlock(folio);
2001	goto done;
2002	}
2003	EXPORT_SYMBOL(__block_write_full_folio);
2004
2005	/*
2006	* If a folio has any new buffers, zero them out here, and mark them uptodate
2007	* and dirty so they'll be written out (in order to prevent uninitialised
2008	* block data from leaking). And clear the new bit.
2009	*/
2010	void folio_zero_new_buffers(struct folio *folio, size_t from, size_t to)
2011	{
2012	size_t block_start, block_end;
2013	struct buffer_head head, bh;
2014
2015	BUG_ON(!folio_test_locked(folio));
2016	head = folio_buffers(folio);
2017	if (!head)
2018	return;
2019
2020	bh = head;
2021	block_start = 0;
2022	do {
2023	block_end = block_start + bh->b_size;
2024
2025	if (buffer_new(bh)) {
2026	if (block_end > from && block_start < to) {
2027	if (!folio_test_uptodate(folio)) {
2028	size_t start, xend;
2029
2030	start = max(from, block_start);
2031	xend = min(to, block_end);
2032
2033	folio_zero_segment(folio, start, xend);
2034	set_buffer_uptodate(bh);
2035	}
2036
2037	clear_buffer_new(bh);
2038	mark_buffer_dirty(bh);
2039	}
2040	}
2041
2042	block_start = block_end;
2043	bh = bh->b_this_page;
2044	} while (bh != head);
2045	}
2046	EXPORT_SYMBOL(folio_zero_new_buffers);
2047
2048	static int
2049	iomap_to_bh(struct inode inode, sector_t block, struct buffer_head bh,
2050	const struct iomap *iomap)
2051	{
2052	loff_t offset = (loff_t)block << inode->i_blkbits;
2053
2054	bh->b_bdev = iomap->bdev;
2055
2056	/*
2057	* Block points to offset in file we need to map, iomap contains
2058	* the offset at which the map starts. If the map ends before the
2059	* current block, then do not map the buffer and let the caller
2060	* handle it.
2061	*/
2062	if (offset >= iomap->offset + iomap->length)
2063	return -EIO;
2064
2065	switch (iomap->type) {
2066	case IOMAP_HOLE:
2067	/*
2068	* If the buffer is not up to date or beyond the current EOF,
2069	* we need to mark it as new to ensure sub-block zeroing is
2070	* executed if necessary.
2071	*/
2072	if (!buffer_uptodate(bh) \|\|
2073	(offset >= i_size_read(inode)))
2074	set_buffer_new(bh);
2075	return 0;
2076	case IOMAP_DELALLOC:
2077	if (!buffer_uptodate(bh) \|\|
2078	(offset >= i_size_read(inode)))
2079	set_buffer_new(bh);
2080	set_buffer_uptodate(bh);
2081	set_buffer_mapped(bh);
2082	set_buffer_delay(bh);
2083	return 0;
2084	case IOMAP_UNWRITTEN:
2085	/*
2086	* For unwritten regions, we always need to ensure that regions
2087	* in the block we are not writing to are zeroed. Mark the
2088	* buffer as new to ensure this.
2089	*/
2090	set_buffer_new(bh);
2091	set_buffer_unwritten(bh);
2092	fallthrough;
2093	case IOMAP_MAPPED:
2094	if ((iomap->flags & IOMAP_F_NEW) \|\|
2095	offset >= i_size_read(inode)) {
2096	/*
2097	* This can happen if truncating the block device races
2098	* with the check in the caller as i_size updates on
2099	* block devices aren't synchronized by i_rwsem for
2100	* block devices.
2101	*/
2102	if (S_ISBLK(inode->i_mode))
2103	return -EIO;
2104	set_buffer_new(bh);
2105	}
2106	bh->b_blocknr = (iomap->addr + offset - iomap->offset) >>
2107	inode->i_blkbits;
2108	set_buffer_mapped(bh);
2109	return 0;
2110	default:
2111	WARN_ON_ONCE(1);
2112	return -EIO;
2113	}
2114	}
2115
2116	int __block_write_begin_int(struct folio *folio, loff_t pos, unsigned len,
2117	get_block_t get_block, const struct iomap iomap)
2118	{
2119	size_t from = offset_in_folio(folio, pos);
2120	size_t to = from + len;
2121	struct inode *inode = folio->mapping->host;
2122	size_t block_start, block_end;
2123	sector_t block;
2124	int err = 0;
2125	size_t blocksize;
2126	struct buffer_head bh, head, wait[2], *wait_bh=wait;
2127
2128	BUG_ON(!folio_test_locked(folio));
2129	BUG_ON(to > folio_size(folio));
2130	BUG_ON(from > to);
2131
2132	head = folio_create_buffers(folio, inode, 0);
2133	blocksize = head->b_size;
2134	block = div_u64(folio_pos(folio), blocksize);
2135
2136	for (bh = head, block_start = 0; bh != head \|\| !block_start;
2137	block++, block_start=block_end, bh = bh->b_this_page) {
2138	block_end = block_start + blocksize;
2139	if (block_end <= from \|\| block_start >= to) {
2140	if (folio_test_uptodate(folio)) {
2141	if (!buffer_uptodate(bh))
2142	set_buffer_uptodate(bh);
2143	}
2144	continue;
2145	}
2146	if (buffer_new(bh))
2147	clear_buffer_new(bh);
2148	if (!buffer_mapped(bh)) {
2149	WARN_ON(bh->b_size != blocksize);
2150	if (get_block)
2151	err = get_block(inode, block, bh, 1);
2152	else
2153	err = iomap_to_bh(inode, block, bh, iomap);
2154	if (err)
2155	break;
2156
2157	if (buffer_new(bh)) {
2158	clean_bdev_bh_alias(bh);
2159	if (folio_test_uptodate(folio)) {
2160	clear_buffer_new(bh);
2161	set_buffer_uptodate(bh);
2162	mark_buffer_dirty(bh);
2163	continue;
2164	}
2165	if (block_end > to \|\| block_start < from)
2166	folio_zero_segments(folio,
2167	to, block_end,
2168	block_start, from);
2169	continue;
2170	}
2171	}
2172	if (folio_test_uptodate(folio)) {
2173	if (!buffer_uptodate(bh))
2174	set_buffer_uptodate(bh);
2175	continue;
2176	}
2177	if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
2178	!buffer_unwritten(bh) &&
2179	(block_start < from \|\| block_end > to)) {
2180	bh_read_nowait(bh, 0);
2181	*wait_bh++=bh;
2182	}
2183	}
2184	/*
2185	* If we issued read requests - let them complete.
2186	*/
2187	while(wait_bh > wait) {
2188	wait_on_buffer(*--wait_bh);
2189	if (!buffer_uptodate(*wait_bh))
2190	err = -EIO;
2191	}
2192	if (unlikely(err))
2193	folio_zero_new_buffers(folio, from, to);
2194	return err;
2195	}
2196
2197	int __block_write_begin(struct folio *folio, loff_t pos, unsigned len,
2198	get_block_t *get_block)
2199	{
2200	return __block_write_begin_int(folio, pos, len, get_block, NULL);
2201	}
2202	EXPORT_SYMBOL(__block_write_begin);
2203
2204	void block_commit_write(struct folio *folio, size_t from, size_t to)
2205	{
2206	size_t block_start, block_end;
2207	bool partial = false;
2208	unsigned blocksize;
2209	struct buffer_head bh, head;
2210
2211	bh = head = folio_buffers(folio);
2212	if (!bh)
2213	return;
2214	blocksize = bh->b_size;
2215
2216	block_start = 0;
2217	do {
2218	block_end = block_start + blocksize;
2219	if (block_end <= from \|\| block_start >= to) {
2220	if (!buffer_uptodate(bh))
2221	partial = true;
2222	} else {
2223	set_buffer_uptodate(bh);
2224	mark_buffer_dirty(bh);
2225	}
2226	if (buffer_new(bh))
2227	clear_buffer_new(bh);
2228
2229	block_start = block_end;
2230	bh = bh->b_this_page;
2231	} while (bh != head);
2232
2233	/*
2234	* If this is a partial write which happened to make all buffers
2235	* uptodate then we can optimize away a bogus read_folio() for
2236	* the next read(). Here we 'discover' whether the folio went
2237	* uptodate as a result of this (potentially partial) write.
2238	*/
2239	if (!partial)
2240	folio_mark_uptodate(folio);
2241	}
2242	EXPORT_SYMBOL(block_commit_write);
2243
2244	/*
2245	* block_write_begin takes care of the basic task of block allocation and
2246	* bringing partial write blocks uptodate first.
2247	*
2248	* The filesystem needs to handle block truncation upon failure.
2249	*/
2250	int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
2251	struct folio *foliop, get_block_t get_block)
2252	{
2253	pgoff_t index = pos >> PAGE_SHIFT;
2254	struct folio *folio;
2255	int status;
2256
2257	folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN,
2258	mapping_gfp_mask(mapping));
2259	if (IS_ERR(folio))
2260	return PTR_ERR(folio);
2261
2262	status = __block_write_begin_int(folio, pos, len, get_block, NULL);
2263	if (unlikely(status)) {
2264	folio_unlock(folio);
2265	folio_put(folio);
2266	folio = NULL;
2267	}
2268
2269	*foliop = folio;
2270	return status;
2271	}
2272	EXPORT_SYMBOL(block_write_begin);
2273
2274	int block_write_end(struct file file, struct address_space mapping,
2275	loff_t pos, unsigned len, unsigned copied,
2276	struct folio folio, void fsdata)
2277	{
2278	size_t start = pos - folio_pos(folio);
2279
2280	if (unlikely(copied < len)) {
2281	/*
2282	* The buffers that were written will now be uptodate, so
2283	* we don't have to worry about a read_folio reading them
2284	* and overwriting a partial write. However if we have
2285	* encountered a short write and only partially written
2286	* into a buffer, it will not be marked uptodate, so a
2287	* read_folio might come in and destroy our partial write.
2288	*
2289	* Do the simplest thing, and just treat any short write to a
2290	* non uptodate folio as a zero-length write, and force the
2291	* caller to redo the whole thing.
2292	*/
2293	if (!folio_test_uptodate(folio))
2294	copied = 0;
2295
2296	folio_zero_new_buffers(folio, start+copied, start+len);
2297	}
2298	flush_dcache_folio(folio);
2299
2300	/* This could be a short (even 0-length) commit */
2301	block_commit_write(folio, start, start + copied);
2302
2303	return copied;
2304	}
2305	EXPORT_SYMBOL(block_write_end);
2306
2307	int generic_write_end(struct file file, struct address_space mapping,
2308	loff_t pos, unsigned len, unsigned copied,
2309	struct folio folio, void fsdata)
2310	{
2311	struct inode *inode = mapping->host;
2312	loff_t old_size = inode->i_size;
2313	bool i_size_changed = false;
2314
2315	copied = block_write_end(file, mapping, pos, len, copied, folio, fsdata);
2316
2317	/*
2318	* No need to use i_size_read() here, the i_size cannot change under us
2319	* because we hold i_rwsem.
2320	*
2321	* But it's important to update i_size while still holding folio lock:
2322	* page writeout could otherwise come in and zero beyond i_size.
2323	*/
2324	if (pos + copied > inode->i_size) {
2325	i_size_write(inode, pos + copied);
2326	i_size_changed = true;
2327	}
2328
2329	folio_unlock(folio);
2330	folio_put(folio);
2331
2332	if (old_size < pos)
2333	pagecache_isize_extended(inode, old_size, pos);
2334	/*
2335	* Don't mark the inode dirty under page lock. First, it unnecessarily
2336	* makes the holding time of page lock longer. Second, it forces lock
2337	* ordering of page lock and transaction start for journaling
2338	* filesystems.
2339	*/
2340	if (i_size_changed)
2341	mark_inode_dirty(inode);
2342	return copied;
2343	}
2344	EXPORT_SYMBOL(generic_write_end);
2345
2346	/*
2347	* block_is_partially_uptodate checks whether buffers within a folio are
2348	* uptodate or not.
2349	*
2350	* Returns true if all buffers which correspond to the specified part
2351	* of the folio are uptodate.
2352	*/
2353	bool block_is_partially_uptodate(struct folio *folio, size_t from, size_t count)
2354	{
2355	unsigned block_start, block_end, blocksize;
2356	unsigned to;
2357	struct buffer_head bh, head;
2358	bool ret = true;
2359
2360	head = folio_buffers(folio);
2361	if (!head)
2362	return false;
2363	blocksize = head->b_size;
2364	to = min_t(unsigned, folio_size(folio) - from, count);
2365	to = from + to;
2366	if (from < blocksize && to > folio_size(folio) - blocksize)
2367	return false;
2368
2369	bh = head;
2370	block_start = 0;
2371	do {
2372	block_end = block_start + blocksize;
2373	if (block_end > from && block_start < to) {
2374	if (!buffer_uptodate(bh)) {
2375	ret = false;
2376	break;
2377	}
2378	if (block_end >= to)
2379	break;
2380	}
2381	block_start = block_end;
2382	bh = bh->b_this_page;
2383	} while (bh != head);
2384
2385	return ret;
2386	}
2387	EXPORT_SYMBOL(block_is_partially_uptodate);
2388
2389	/*
2390	* Generic "read_folio" function for block devices that have the normal
2391	* get_block functionality. This is most of the block device filesystems.
2392	* Reads the folio asynchronously --- the unlock_buffer() and
2393	* set/clear_buffer_uptodate() functions propagate buffer state into the
2394	* folio once IO has completed.
2395	*/
2396	int block_read_full_folio(struct folio folio, get_block_t get_block)
2397	{
2398	struct inode *inode = folio->mapping->host;
2399	sector_t iblock, lblock;
2400	struct buffer_head bh, head, *prev = NULL;
2401	size_t blocksize;
2402	int fully_mapped = 1;
2403	bool page_error = false;
2404	loff_t limit = i_size_read(inode);
2405
2406	/* This is needed for ext4. */
2407	if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode))
2408	limit = inode->i_sb->s_maxbytes;
2409
2410	head = folio_create_buffers(folio, inode, 0);
2411	blocksize = head->b_size;
2412
2413	iblock = div_u64(folio_pos(folio), blocksize);
2414	lblock = div_u64(limit + blocksize - 1, blocksize);
2415	bh = head;
2416
2417	do {
2418	if (buffer_uptodate(bh))
2419	continue;
2420
2421	if (!buffer_mapped(bh)) {
2422	int err = 0;
2423
2424	fully_mapped = 0;
2425	if (iblock < lblock) {
2426	WARN_ON(bh->b_size != blocksize);
2427	err = get_block(inode, iblock, bh, 0);
2428	if (err)
2429	page_error = true;
2430	}
2431	if (!buffer_mapped(bh)) {
2432	folio_zero_range(folio, bh_offset(bh),
2433	blocksize);
2434	if (!err)
2435	set_buffer_uptodate(bh);
2436	continue;
2437	}
2438	/*
2439	* get_block() might have updated the buffer
2440	* synchronously
2441	*/
2442	if (buffer_uptodate(bh))
2443	continue;
2444	}
2445
2446	lock_buffer(bh);
2447	if (buffer_uptodate(bh)) {
2448	unlock_buffer(bh);
2449	continue;
2450	}
2451
2452	mark_buffer_async_read(bh);
2453	if (prev)
2454	submit_bh(REQ_OP_READ, prev);
2455	prev = bh;
2456	} while (iblock++, (bh = bh->b_this_page) != head);
2457
2458	if (fully_mapped)
2459	folio_set_mappedtodisk(folio);
2460
2461	/*
2462	* All buffers are uptodate or get_block() returned an error
2463	* when trying to map them - we must finish the read because
2464	* end_buffer_async_read() will never be called on any buffer
2465	* in this folio.
2466	*/
2467	if (prev)
2468	submit_bh(REQ_OP_READ, prev);
2469	else
2470	folio_end_read(folio, !page_error);
2471
2472	return 0;
2473	}
2474	EXPORT_SYMBOL(block_read_full_folio);
2475
2476	/* utility function for filesystems that need to do work on expanding
2477	* truncates. Uses filesystem pagecache writes to allow the filesystem to
2478	* deal with the hole.
2479	*/
2480	int generic_cont_expand_simple(struct inode *inode, loff_t size)
2481	{
2482	struct address_space *mapping = inode->i_mapping;
2483	const struct address_space_operations *aops = mapping->a_ops;
2484	struct folio *folio;
2485	void *fsdata = NULL;
2486	int err;
2487
2488	err = inode_newsize_ok(inode, size);
2489	if (err)
2490	goto out;
2491
2492	err = aops->write_begin(NULL, mapping, size, 0, &folio, &fsdata);
2493	if (err)
2494	goto out;
2495
2496	err = aops->write_end(NULL, mapping, size, 0, 0, folio, fsdata);
2497	BUG_ON(err > 0);
2498
2499	out:
2500	return err;
2501	}
2502	EXPORT_SYMBOL(generic_cont_expand_simple);
2503
2504	static int cont_expand_zero(struct file file, struct address_space mapping,
2505	loff_t pos, loff_t *bytes)
2506	{
2507	struct inode *inode = mapping->host;
2508	const struct address_space_operations *aops = mapping->a_ops;
2509	unsigned int blocksize = i_blocksize(inode);
2510	struct folio *folio;
2511	void *fsdata = NULL;
2512	pgoff_t index, curidx;
2513	loff_t curpos;
2514	unsigned zerofrom, offset, len;
2515	int err = 0;
2516
2517	index = pos >> PAGE_SHIFT;
2518	offset = pos & ~PAGE_MASK;
2519
2520	while (index > (curidx = (curpos = *bytes)>>PAGE_SHIFT)) {
2521	zerofrom = curpos & ~PAGE_MASK;
2522	if (zerofrom & (blocksize-1)) {
2523	*bytes \|= (blocksize-1);
2524	(*bytes)++;
2525	}
2526	len = PAGE_SIZE - zerofrom;
2527
2528	err = aops->write_begin(file, mapping, curpos, len,
2529	&folio, &fsdata);
2530	if (err)
2531	goto out;
2532	folio_zero_range(folio, offset_in_folio(folio, curpos), len);
2533	err = aops->write_end(file, mapping, curpos, len, len,
2534	folio, fsdata);
2535	if (err < 0)
2536	goto out;
2537	BUG_ON(err != len);
2538	err = 0;
2539
2540	balance_dirty_pages_ratelimited(mapping);
2541
2542	if (fatal_signal_pending(current)) {
2543	err = -EINTR;
2544	goto out;
2545	}
2546	}
2547
2548	/* page covers the boundary, find the boundary offset */
2549	if (index == curidx) {
2550	zerofrom = curpos & ~PAGE_MASK;
2551	/* if we will expand the thing last block will be filled */
2552	if (offset <= zerofrom) {
2553	goto out;
2554	}
2555	if (zerofrom & (blocksize-1)) {
2556	*bytes \|= (blocksize-1);
2557	(*bytes)++;
2558	}
2559	len = offset - zerofrom;
2560
2561	err = aops->write_begin(file, mapping, curpos, len,
2562	&folio, &fsdata);
2563	if (err)
2564	goto out;
2565	folio_zero_range(folio, offset_in_folio(folio, curpos), len);
2566	err = aops->write_end(file, mapping, curpos, len, len,
2567	folio, fsdata);
2568	if (err < 0)
2569	goto out;
2570	BUG_ON(err != len);
2571	err = 0;
2572	}
2573	out:
2574	return err;
2575	}
2576
2577	/*
2578	* For moronic filesystems that do not allow holes in file.
2579	* We may have to extend the file.
2580	*/
2581	int cont_write_begin(struct file file, struct address_space mapping,
2582	loff_t pos, unsigned len,
2583	struct folio foliop, void fsdata,
2584	get_block_t get_block, loff_t bytes)
2585	{
2586	struct inode *inode = mapping->host;
2587	unsigned int blocksize = i_blocksize(inode);
2588	unsigned int zerofrom;
2589	int err;
2590
2591	err = cont_expand_zero(file, mapping, pos, bytes);
2592	if (err)
2593	return err;
2594
2595	zerofrom = *bytes & ~PAGE_MASK;
2596	if (pos+len > *bytes && zerofrom & (blocksize-1)) {
2597	*bytes \|= (blocksize-1);
2598	(*bytes)++;
2599	}
2600
2601	return block_write_begin(mapping, pos, len, foliop, get_block);
2602	}
2603	EXPORT_SYMBOL(cont_write_begin);
2604
2605	/*
2606	* block_page_mkwrite() is not allowed to change the file size as it gets
2607	* called from a page fault handler when a page is first dirtied. Hence we must
2608	* be careful to check for EOF conditions here. We set the page up correctly
2609	* for a written page which means we get ENOSPC checking when writing into
2610	* holes and correct delalloc and unwritten extent mapping on filesystems that
2611	* support these features.
2612	*
2613	* We are not allowed to take the i_mutex here so we have to play games to
2614	* protect against truncate races as the page could now be beyond EOF. Because
2615	* truncate writes the inode size before removing pages, once we have the
2616	* page lock we can determine safely if the page is beyond EOF. If it is not
2617	* beyond EOF, then the page is guaranteed safe against truncation until we
2618	* unlock the page.
2619	*
2620	* Direct callers of this function should protect against filesystem freezing
2621	* using sb_start_pagefault() - sb_end_pagefault() functions.
2622	*/
2623	int block_page_mkwrite(struct vm_area_struct vma, struct vm_fault vmf,
2624	get_block_t get_block)
2625	{
2626	struct folio *folio = page_folio(vmf->page);
2627	struct inode *inode = file_inode(vma->vm_file);
2628	unsigned long end;
2629	loff_t size;
2630	int ret;
2631
2632	folio_lock(folio);
2633	size = i_size_read(inode);
2634	if ((folio->mapping != inode->i_mapping) \|\|
2635	(folio_pos(folio) >= size)) {
2636	/* We overload EFAULT to mean page got truncated */
2637	ret = -EFAULT;
2638	goto out_unlock;
2639	}
2640
2641	end = folio_size(folio);
2642	/* folio is wholly or partially inside EOF */
2643	if (folio_pos(folio) + end > size)
2644	end = size - folio_pos(folio);
2645
2646	ret = __block_write_begin_int(folio, 0, end, get_block, NULL);
2647	if (unlikely(ret))
2648	goto out_unlock;
2649
2650	block_commit_write(folio, 0, end);
2651
2652	folio_mark_dirty(folio);
2653	folio_wait_stable(folio);
2654	return 0;
2655	out_unlock:
2656	folio_unlock(folio);
2657	return ret;
2658	}
2659	EXPORT_SYMBOL(block_page_mkwrite);
2660
2661	int block_truncate_page(struct address_space *mapping,
2662	loff_t from, get_block_t *get_block)
2663	{
2664	pgoff_t index = from >> PAGE_SHIFT;
2665	unsigned blocksize;
2666	sector_t iblock;
2667	size_t offset, length, pos;
2668	struct inode *inode = mapping->host;
2669	struct folio *folio;
2670	struct buffer_head *bh;
2671	int err = 0;
2672
2673	blocksize = i_blocksize(inode);
2674	length = from & (blocksize - 1);
2675
2676	/* Block boundary? Nothing to do */
2677	if (!length)
2678	return 0;
2679
2680	length = blocksize - length;
2681	iblock = ((loff_t)index * PAGE_SIZE) >> inode->i_blkbits;
2682
2683	folio = filemap_grab_folio(mapping, index);
2684	if (IS_ERR(folio))
2685	return PTR_ERR(folio);
2686
2687	bh = folio_buffers(folio);
2688	if (!bh)
2689	bh = create_empty_buffers(folio, blocksize, 0);
2690
2691	/* Find the buffer that contains "offset" */
2692	offset = offset_in_folio(folio, from);
2693	pos = blocksize;
2694	while (offset >= pos) {
2695	bh = bh->b_this_page;
2696	iblock++;
2697	pos += blocksize;
2698	}
2699
2700	if (!buffer_mapped(bh)) {
2701	WARN_ON(bh->b_size != blocksize);
2702	err = get_block(inode, iblock, bh, 0);
2703	if (err)
2704	goto unlock;
2705	/* unmapped? It's a hole - nothing to do */
2706	if (!buffer_mapped(bh))
2707	goto unlock;
2708	}
2709
2710	/* Ok, it's mapped. Make sure it's up-to-date */
2711	if (folio_test_uptodate(folio))
2712	set_buffer_uptodate(bh);
2713
2714	if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
2715	err = bh_read(bh, 0);
2716	/* Uhhuh. Read error. Complain and punt. */
2717	if (err < 0)
2718	goto unlock;
2719	}
2720
2721	folio_zero_range(folio, offset, length);
2722	mark_buffer_dirty(bh);
2723
2724	unlock:
2725	folio_unlock(folio);
2726	folio_put(folio);
2727
2728	return err;
2729	}
2730	EXPORT_SYMBOL(block_truncate_page);
2731
2732	/*
2733	* The generic write folio function for buffer-backed address_spaces
2734	*/
2735	int block_write_full_folio(struct folio folio, struct writeback_control wbc,
2736	void *get_block)
2737	{
2738	struct inode * const inode = folio->mapping->host;
2739	loff_t i_size = i_size_read(inode);
2740
2741	/* Is the folio fully inside i_size? */
2742	if (folio_pos(folio) + folio_size(folio) <= i_size)
2743	return __block_write_full_folio(inode, folio, get_block, wbc);
2744
2745	/* Is the folio fully outside i_size? (truncate in progress) */
2746	if (folio_pos(folio) >= i_size) {
2747	folio_unlock(folio);
2748	return 0; /* don't care */
2749	}
2750
2751	/*
2752	* The folio straddles i_size. It must be zeroed out on each and every
2753	* writeback invocation because it may be mmapped. "A file is mapped
2754	* in multiples of the page size. For a file that is not a multiple of
2755	* the page size, the remaining memory is zeroed when mapped, and
2756	* writes to that region are not written out to the file."
2757	*/
2758	folio_zero_segment(folio, offset_in_folio(folio, i_size),
2759	folio_size(folio));
2760	return __block_write_full_folio(inode, folio, get_block, wbc);
2761	}
2762
2763	sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2764	get_block_t *get_block)
2765	{
2766	struct inode *inode = mapping->host;
2767	struct buffer_head tmp = {
2768	.b_size = i_blocksize(inode),
2769	};
2770
2771	get_block(inode, block, &tmp, 0);
2772	return tmp.b_blocknr;
2773	}
2774	EXPORT_SYMBOL(generic_block_bmap);
2775
2776	static void end_bio_bh_io_sync(struct bio *bio)
2777	{
2778	struct buffer_head *bh = bio->bi_private;
2779
2780	if (unlikely(bio_flagged(bio, BIO_QUIET)))
2781	set_bit(BH_Quiet, &bh->b_state);
2782
2783	bh->b_end_io(bh, !bio->bi_status);
2784	bio_put(bio);
2785	}
2786
2787	static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh,
2788	enum rw_hint write_hint,
2789	struct writeback_control *wbc)
2790	{
2791	const enum req_op op = opf & REQ_OP_MASK;
2792	struct bio *bio;
2793
2794	BUG_ON(!buffer_locked(bh));
2795	BUG_ON(!buffer_mapped(bh));
2796	BUG_ON(!bh->b_end_io);
2797	BUG_ON(buffer_delay(bh));
2798	BUG_ON(buffer_unwritten(bh));
2799
2800	/*
2801	* Only clear out a write error when rewriting
2802	*/
2803	if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE))
2804	clear_buffer_write_io_error(bh);
2805
2806	if (buffer_meta(bh))
2807	opf \|= REQ_META;
2808	if (buffer_prio(bh))
2809	opf \|= REQ_PRIO;
2810
2811	bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO);
2812
2813	fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO);
2814
2815	bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2816	bio->bi_write_hint = write_hint;
2817
2818	bio_add_folio_nofail(bio, bh->b_folio, bh->b_size, bh_offset(bh));
2819
2820	bio->bi_end_io = end_bio_bh_io_sync;
2821	bio->bi_private = bh;
2822
2823	/* Take care of bh's that straddle the end of the device */
2824	guard_bio_eod(bio);
2825
2826	if (wbc) {
2827	wbc_init_bio(wbc, bio);
2828	wbc_account_cgroup_owner(wbc, bh->b_folio, bh->b_size);
2829	}
2830
2831	submit_bio(bio);
2832	}
2833
2834	void submit_bh(blk_opf_t opf, struct buffer_head *bh)
2835	{
2836	submit_bh_wbc(opf, bh, WRITE_LIFE_NOT_SET, NULL);
2837	}
2838	EXPORT_SYMBOL(submit_bh);
2839
2840	void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags)
2841	{
2842	lock_buffer(bh);
2843	if (!test_clear_buffer_dirty(bh)) {
2844	unlock_buffer(bh);
2845	return;
2846	}
2847	bh->b_end_io = end_buffer_write_sync;
2848	get_bh(bh);
2849	submit_bh(REQ_OP_WRITE \| op_flags, bh);
2850	}
2851	EXPORT_SYMBOL(write_dirty_buffer);
2852
2853	/*
2854	* For a data-integrity writeout, we need to wait upon any in-progress I/O
2855	* and then start new I/O and then wait upon it. The caller must have a ref on
2856	* the buffer_head.
2857	*/
2858	int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags)
2859	{
2860	WARN_ON(atomic_read(&bh->b_count) < 1);
2861	lock_buffer(bh);
2862	if (test_clear_buffer_dirty(bh)) {
2863	/*
2864	* The bh should be mapped, but it might not be if the
2865	* device was hot-removed. Not much we can do but fail the I/O.
2866	*/
2867	if (!buffer_mapped(bh)) {
2868	unlock_buffer(bh);
2869	return -EIO;
2870	}
2871
2872	get_bh(bh);
2873	bh->b_end_io = end_buffer_write_sync;
2874	submit_bh(REQ_OP_WRITE \| op_flags, bh);
2875	wait_on_buffer(bh);
2876	if (!buffer_uptodate(bh))
2877	return -EIO;
2878	} else {
2879	unlock_buffer(bh);
2880	}
2881	return 0;
2882	}
2883	EXPORT_SYMBOL(__sync_dirty_buffer);
2884
2885	int sync_dirty_buffer(struct buffer_head *bh)
2886	{
2887	return __sync_dirty_buffer(bh, REQ_SYNC);
2888	}
2889	EXPORT_SYMBOL(sync_dirty_buffer);
2890
2891	static inline int buffer_busy(struct buffer_head *bh)
2892	{
2893	return atomic_read(&bh->b_count) \|
2894	(bh->b_state & ((1 << BH_Dirty) \| (1 << BH_Lock)));
2895	}
2896
2897	static bool
2898	drop_buffers(struct folio folio, struct buffer_head *buffers_to_free)
2899	{
2900	struct buffer_head *head = folio_buffers(folio);
2901	struct buffer_head *bh;
2902
2903	bh = head;
2904	do {
2905	if (buffer_busy(bh))
2906	goto failed;
2907	bh = bh->b_this_page;
2908	} while (bh != head);
2909
2910	do {
2911	struct buffer_head *next = bh->b_this_page;
2912
2913	if (bh->b_assoc_map)
2914	__remove_assoc_queue(bh);
2915	bh = next;
2916	} while (bh != head);
2917	*buffers_to_free = head;
2918	folio_detach_private(folio);
2919	return true;
2920	failed:
2921	return false;
2922	}
2923
2924	/**
2925	* try_to_free_buffers - Release buffers attached to this folio.
2926	* @folio: The folio.
2927	*
2928	* If any buffers are in use (dirty, under writeback, elevated refcount),
2929	* no buffers will be freed.
2930	*
2931	* If the folio is dirty but all the buffers are clean then we need to
2932	* be sure to mark the folio clean as well. This is because the folio
2933	* may be against a block device, and a later reattachment of buffers
2934	* to a dirty folio will set all buffers dirty. Which would corrupt
2935	* filesystem data on the same device.
2936	*
2937	* The same applies to regular filesystem folios: if all the buffers are
2938	* clean then we set the folio clean and proceed. To do that, we require
2939	* total exclusion from block_dirty_folio(). That is obtained with
2940	* i_private_lock.
2941	*
2942	* Exclusion against try_to_free_buffers may be obtained by either
2943	* locking the folio or by holding its mapping's i_private_lock.
2944	*
2945	* Context: Process context. @folio must be locked. Will not sleep.
2946	* Return: true if all buffers attached to this folio were freed.
2947	*/
2948	bool try_to_free_buffers(struct folio *folio)
2949	{
2950	struct address_space * const mapping = folio->mapping;
2951	struct buffer_head *buffers_to_free = NULL;
2952	bool ret = 0;
2953
2954	BUG_ON(!folio_test_locked(folio));
2955	if (folio_test_writeback(folio))
2956	return false;
2957
2958	if (mapping == NULL) { /* can this still happen? */
2959	ret = drop_buffers(folio, &buffers_to_free);
2960	goto out;
2961	}
2962
2963	spin_lock(&mapping->i_private_lock);
2964	ret = drop_buffers(folio, &buffers_to_free);
2965
2966	/*
2967	* If the filesystem writes its buffers by hand (eg ext3)
2968	* then we can have clean buffers against a dirty folio. We
2969	* clean the folio here; otherwise the VM will never notice
2970	* that the filesystem did any IO at all.
2971	*
2972	* Also, during truncate, discard_buffer will have marked all
2973	* the folio's buffers clean. We discover that here and clean
2974	* the folio also.
2975	*
2976	* i_private_lock must be held over this entire operation in order
2977	* to synchronise against block_dirty_folio and prevent the
2978	* dirty bit from being lost.
2979	*/
2980	if (ret)
2981	folio_cancel_dirty(folio);
2982	spin_unlock(&mapping->i_private_lock);
2983	out:
2984	if (buffers_to_free) {
2985	struct buffer_head *bh = buffers_to_free;
2986
2987	do {
2988	struct buffer_head *next = bh->b_this_page;
2989	free_buffer_head(bh);
2990	bh = next;
2991	} while (bh != buffers_to_free);
2992	}
2993	return ret;
2994	}
2995	EXPORT_SYMBOL(try_to_free_buffers);
2996
2997	/*
2998	* Buffer-head allocation
2999	*/
3000	static struct kmem_cache *bh_cachep __ro_after_init;
3001
3002	/*
3003	* Once the number of bh's in the machine exceeds this level, we start
3004	* stripping them in writeback.
3005	*/
3006	static unsigned long max_buffer_heads __ro_after_init;
3007
3008	int buffer_heads_over_limit;
3009
3010	struct bh_accounting {
3011	int nr; /* Number of live bh's */
3012	int ratelimit; /* Limit cacheline bouncing */
3013	};
3014
3015	static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3016
3017	static void recalc_bh_state(void)
3018	{
3019	int i;
3020	int tot = 0;
3021
3022	if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
3023	return;
3024	__this_cpu_write(bh_accounting.ratelimit, 0);
3025	for_each_online_cpu(i)
3026	tot += per_cpu(bh_accounting, i).nr;
3027	buffer_heads_over_limit = (tot > max_buffer_heads);
3028	}
3029
3030	struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
3031	{
3032	struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
3033	if (ret) {
3034	INIT_LIST_HEAD(&ret->b_assoc_buffers);
3035	spin_lock_init(&ret->b_uptodate_lock);
3036	preempt_disable();
3037	__this_cpu_inc(bh_accounting.nr);
3038	recalc_bh_state();
3039	preempt_enable();
3040	}
3041	return ret;
3042	}
3043	EXPORT_SYMBOL(alloc_buffer_head);
3044
3045	void free_buffer_head(struct buffer_head *bh)
3046	{
3047	BUG_ON(!list_empty(&bh->b_assoc_buffers));
3048	kmem_cache_free(bh_cachep, bh);
3049	preempt_disable();
3050	__this_cpu_dec(bh_accounting.nr);
3051	recalc_bh_state();
3052	preempt_enable();
3053	}
3054	EXPORT_SYMBOL(free_buffer_head);
3055
3056	static int buffer_exit_cpu_dead(unsigned int cpu)
3057	{
3058	int i;
3059	struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3060
3061	for (i = 0; i < BH_LRU_SIZE; i++) {
3062	brelse(b->bhs[i]);
3063	b->bhs[i] = NULL;
3064	}
3065	this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
3066	per_cpu(bh_accounting, cpu).nr = 0;
3067	return 0;
3068	}
3069
3070	/**
3071	* bh_uptodate_or_lock - Test whether the buffer is uptodate
3072	* @bh: struct buffer_head
3073	*
3074	* Return true if the buffer is up-to-date and false,
3075	* with the buffer locked, if not.
3076	*/
3077	int bh_uptodate_or_lock(struct buffer_head *bh)
3078	{
3079	if (!buffer_uptodate(bh)) {
3080	lock_buffer(bh);
3081	if (!buffer_uptodate(bh))
3082	return 0;
3083	unlock_buffer(bh);
3084	}
3085	return 1;
3086	}
3087	EXPORT_SYMBOL(bh_uptodate_or_lock);
3088
3089	/**
3090	* __bh_read - Submit read for a locked buffer
3091	* @bh: struct buffer_head
3092	* @op_flags: appending REQ_OP_* flags besides REQ_OP_READ
3093	* @wait: wait until reading finish
3094	*
3095	* Returns zero on success or don't wait, and -EIO on error.
3096	*/
3097	int __bh_read(struct buffer_head *bh, blk_opf_t op_flags, bool wait)
3098	{
3099	int ret = 0;
3100
3101	BUG_ON(!buffer_locked(bh));
3102
3103	get_bh(bh);
3104	bh->b_end_io = end_buffer_read_sync;
3105	submit_bh(REQ_OP_READ \| op_flags, bh);
3106	if (wait) {
3107	wait_on_buffer(bh);
3108	if (!buffer_uptodate(bh))
3109	ret = -EIO;
3110	}
3111	return ret;
3112	}
3113	EXPORT_SYMBOL(__bh_read);
3114
3115	/**
3116	* __bh_read_batch - Submit read for a batch of unlocked buffers
3117	* @nr: entry number of the buffer batch
3118	* @bhs: a batch of struct buffer_head
3119	* @op_flags: appending REQ_OP_* flags besides REQ_OP_READ
3120	* @force_lock: force to get a lock on the buffer if set, otherwise drops any
3121	* buffer that cannot lock.
3122	*
3123	* Returns zero on success or don't wait, and -EIO on error.
3124	*/
3125	void __bh_read_batch(int nr, struct buffer_head *bhs[],
3126	blk_opf_t op_flags, bool force_lock)
3127	{
3128	int i;
3129
3130	for (i = 0; i < nr; i++) {
3131	struct buffer_head *bh = bhs[i];
3132
3133	if (buffer_uptodate(bh))
3134	continue;
3135
3136	if (force_lock)
3137	lock_buffer(bh);
3138	else
3139	if (!trylock_buffer(bh))
3140	continue;
3141
3142	if (buffer_uptodate(bh)) {
3143	unlock_buffer(bh);
3144	continue;
3145	}
3146
3147	bh->b_end_io = end_buffer_read_sync;
3148	get_bh(bh);
3149	submit_bh(REQ_OP_READ \| op_flags, bh);
3150	}
3151	}
3152	EXPORT_SYMBOL(__bh_read_batch);
3153
3154	void __init buffer_init(void)
3155	{
3156	unsigned long nrpages;
3157	int ret;
3158
3159	bh_cachep = KMEM_CACHE(buffer_head,
3160	SLAB_RECLAIM_ACCOUNT\|SLAB_PANIC);
3161	/*
3162	* Limit the bh occupancy to 10% of ZONE_NORMAL
3163	*/
3164	nrpages = (nr_free_buffer_pages() * 10) / 100;
3165	max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3166	ret = cpuhp_setup_state_nocalls(CPUHP_FS_BUFF_DEAD, "fs/buffer:dead",
3167	NULL, buffer_exit_cpu_dead);
3168	WARN_ON(ret < 0);
3169	}