linux - fs/buffer.c

3458 lines · 89.6 KiB

1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* linux/fs/buffer.c
4	*
5	* Copyright (C) 1991, 1992, 2002 Linus Torvalds
6	*/
7
8	/*
9	* Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
10	*
11	* Removed a lot of unnecessary code and simplified things now that
12	* the buffer cache isn't our primary cache - Andrew Tridgell 12/96
13	*
14	* Speed up hash, lru, and free list operations. Use gfp() for allocating
15	* hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
16	*
17	* Added 32k buffer block sizes - these are required older ARM systems. - RMK
18	*
19	* async buffer flushing, 1999 Andrea Arcangeli <[email protected]>
20	*/
21
22	#include <linux/kernel.h>
23	#include <linux/sched/signal.h>
24	#include <linux/syscalls.h>
25	#include <linux/fs.h>
26	#include <linux/iomap.h>
27	#include <linux/mm.h>
28	#include <linux/percpu.h>
29	#include <linux/slab.h>
30	#include <linux/capability.h>
31	#include <linux/blkdev.h>
32	#include <linux/file.h>
33	#include <linux/quotaops.h>
34	#include <linux/highmem.h>
35	#include <linux/export.h>
36	#include <linux/backing-dev.h>
37	#include <linux/writeback.h>
38	#include <linux/hash.h>
39	#include <linux/suspend.h>
40	#include <linux/buffer_head.h>
41	#include <linux/task_io_accounting_ops.h>
42	#include <linux/bio.h>
43	#include <linux/cpu.h>
44	#include <linux/bitops.h>
45	#include <linux/mpage.h>
46	#include <linux/bit_spinlock.h>
47	#include <linux/pagevec.h>
48	#include <linux/sched/mm.h>
49	#include <trace/events/block.h>
50	#include <linux/fscrypt.h>
51
52	#include "internal.h"
53
54	static int fsync_buffers_list(spinlock_t lock, struct list_head list);
55	static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
56	enum rw_hint hint, struct writeback_control *wbc);
57
58	#define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
59
60	inline void touch_buffer(struct buffer_head *bh)
61	{
62	trace_block_touch_buffer(bh);
63	mark_page_accessed(bh->b_page);
64	}
65	EXPORT_SYMBOL(touch_buffer);
66
67	void __lock_buffer(struct buffer_head *bh)
68	{
69	wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
70	}
71	EXPORT_SYMBOL(__lock_buffer);
72
73	void unlock_buffer(struct buffer_head *bh)
74	{
75	clear_bit_unlock(BH_Lock, &bh->b_state);
76	smp_mb__after_atomic();
77	wake_up_bit(&bh->b_state, BH_Lock);
78	}
79	EXPORT_SYMBOL(unlock_buffer);
80
81	/*
82	* Returns if the page has dirty or writeback buffers. If all the buffers
83	* are unlocked and clean then the PageDirty information is stale. If
84	* any of the pages are locked, it is assumed they are locked for IO.
85	*/
86	void buffer_check_dirty_writeback(struct page *page,
87	bool dirty, bool writeback)
88	{
89	struct buffer_head head, bh;
90	*dirty = false;
91	*writeback = false;
92
93	BUG_ON(!PageLocked(page));
94
95	if (!page_has_buffers(page))
96	return;
97
98	if (PageWriteback(page))
99	*writeback = true;
100
101	head = page_buffers(page);
102	bh = head;
103	do {
104	if (buffer_locked(bh))
105	*writeback = true;
106
107	if (buffer_dirty(bh))
108	*dirty = true;
109
110	bh = bh->b_this_page;
111	} while (bh != head);
112	}
113	EXPORT_SYMBOL(buffer_check_dirty_writeback);
114
115	/*
116	* Block until a buffer comes unlocked. This doesn't stop it
117	* from becoming locked again - you have to lock it yourself
118	* if you want to preserve its state.
119	*/
120	void __wait_on_buffer(struct buffer_head * bh)
121	{
122	wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
123	}
124	EXPORT_SYMBOL(__wait_on_buffer);
125
126	static void buffer_io_error(struct buffer_head bh, char msg)
127	{
128	if (!test_bit(BH_Quiet, &bh->b_state))
129	printk_ratelimited(KERN_ERR
130	"Buffer I/O error on dev %pg, logical block %llu%s\n",
131	bh->b_bdev, (unsigned long long)bh->b_blocknr, msg);
132	}
133
134	/*
135	* End-of-IO handler helper function which does not touch the bh after
136	* unlocking it.
137	* Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
138	* a race there is benign: unlock_buffer() only use the bh's address for
139	* hashing after unlocking the buffer, so it doesn't actually touch the bh
140	* itself.
141	*/
142	static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
143	{
144	if (uptodate) {
145	set_buffer_uptodate(bh);
146	} else {
147	/* This happens, due to failed read-ahead attempts. */
148	clear_buffer_uptodate(bh);
149	}
150	unlock_buffer(bh);
151	}
152
153	/*
154	* Default synchronous end-of-IO handler.. Just mark it up-to-date and
155	* unlock the buffer. This is what ll_rw_block uses too.
156	*/
157	void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
158	{
159	__end_buffer_read_notouch(bh, uptodate);
160	put_bh(bh);
161	}
162	EXPORT_SYMBOL(end_buffer_read_sync);
163
164	void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
165	{
166	if (uptodate) {
167	set_buffer_uptodate(bh);
168	} else {
169	buffer_io_error(bh, ", lost sync page write");
170	mark_buffer_write_io_error(bh);
171	clear_buffer_uptodate(bh);
172	}
173	unlock_buffer(bh);
174	put_bh(bh);
175	}
176	EXPORT_SYMBOL(end_buffer_write_sync);
177
178	/*
179	* Various filesystems appear to want __find_get_block to be non-blocking.
180	* But it's the page lock which protects the buffers. To get around this,
181	* we get exclusion from try_to_free_buffers with the blockdev mapping's
182	* private_lock.
183	*
184	* Hack idea: for the blockdev mapping, private_lock contention
185	* may be quite high. This code could TryLock the page, and if that
186	* succeeds, there is no need to take private_lock.
187	*/
188	static struct buffer_head *
189	__find_get_block_slow(struct block_device *bdev, sector_t block)
190	{
191	struct inode *bd_inode = bdev->bd_inode;
192	struct address_space *bd_mapping = bd_inode->i_mapping;
193	struct buffer_head *ret = NULL;
194	pgoff_t index;
195	struct buffer_head *bh;
196	struct buffer_head *head;
197	struct page *page;
198	int all_mapped = 1;
199	static DEFINE_RATELIMIT_STATE(last_warned, HZ, 1);
200
201	index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
202	page = find_get_page_flags(bd_mapping, index, FGP_ACCESSED);
203	if (!page)
204	goto out;
205
206	spin_lock(&bd_mapping->private_lock);
207	if (!page_has_buffers(page))
208	goto out_unlock;
209	head = page_buffers(page);
210	bh = head;
211	do {
212	if (!buffer_mapped(bh))
213	all_mapped = 0;
214	else if (bh->b_blocknr == block) {
215	ret = bh;
216	get_bh(bh);
217	goto out_unlock;
218	}
219	bh = bh->b_this_page;
220	} while (bh != head);
221
222	/* we might be here because some of the buffers on this page are
223	* not mapped. This is due to various races between
224	* file io on the block device and getblk. It gets dealt with
225	* elsewhere, don't buffer_error if we had some unmapped buffers
226	*/
227	ratelimit_set_flags(&last_warned, RATELIMIT_MSG_ON_RELEASE);
228	if (all_mapped && __ratelimit(&last_warned)) {
229	printk("__find_get_block_slow() failed. block=%llu, "
230	"b_blocknr=%llu, b_state=0x%08lx, b_size=%zu, "
231	"device %pg blocksize: %d\n",
232	(unsigned long long)block,
233	(unsigned long long)bh->b_blocknr,
234	bh->b_state, bh->b_size, bdev,
235	1 << bd_inode->i_blkbits);
236	}
237	out_unlock:
238	spin_unlock(&bd_mapping->private_lock);
239	put_page(page);
240	out:
241	return ret;
242	}
243
244	static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
245	{
246	unsigned long flags;
247	struct buffer_head *first;
248	struct buffer_head *tmp;
249	struct page *page;
250	int page_uptodate = 1;
251
252	BUG_ON(!buffer_async_read(bh));
253
254	page = bh->b_page;
255	if (uptodate) {
256	set_buffer_uptodate(bh);
257	} else {
258	clear_buffer_uptodate(bh);
259	buffer_io_error(bh, ", async page read");
260	SetPageError(page);
261	}
262
263	/*
264	* Be _very_ careful from here on. Bad things can happen if
265	* two buffer heads end IO at almost the same time and both
266	* decide that the page is now completely done.
267	*/
268	first = page_buffers(page);
269	spin_lock_irqsave(&first->b_uptodate_lock, flags);
270	clear_buffer_async_read(bh);
271	unlock_buffer(bh);
272	tmp = bh;
273	do {
274	if (!buffer_uptodate(tmp))
275	page_uptodate = 0;
276	if (buffer_async_read(tmp)) {
277	BUG_ON(!buffer_locked(tmp));
278	goto still_busy;
279	}
280	tmp = tmp->b_this_page;
281	} while (tmp != bh);
282	spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
283
284	/*
285	* If none of the buffers had errors and they are all
286	* uptodate then we can set the page uptodate.
287	*/
288	if (page_uptodate && !PageError(page))
289	SetPageUptodate(page);
290	unlock_page(page);
291	return;
292
293	still_busy:
294	spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
295	return;
296	}
297
298	struct decrypt_bh_ctx {
299	struct work_struct work;
300	struct buffer_head *bh;
301	};
302
303	static void decrypt_bh(struct work_struct *work)
304	{
305	struct decrypt_bh_ctx *ctx =
306	container_of(work, struct decrypt_bh_ctx, work);
307	struct buffer_head *bh = ctx->bh;
308	int err;
309
310	err = fscrypt_decrypt_pagecache_blocks(bh->b_page, bh->b_size,
311	bh_offset(bh));
312	end_buffer_async_read(bh, err == 0);
313	kfree(ctx);
314	}
315
316	/*
317	* I/O completion handler for block_read_full_page() - pages
318	* which come unlocked at the end of I/O.
319	*/
320	static void end_buffer_async_read_io(struct buffer_head *bh, int uptodate)
321	{
322	/* Decrypt if needed */
323	if (uptodate &&
324	fscrypt_inode_uses_fs_layer_crypto(bh->b_page->mapping->host)) {
325	struct decrypt_bh_ctx ctx = kmalloc(sizeof(ctx), GFP_ATOMIC);
326
327	if (ctx) {
328	INIT_WORK(&ctx->work, decrypt_bh);
329	ctx->bh = bh;
330	fscrypt_enqueue_decrypt_work(&ctx->work);
331	return;
332	}
333	uptodate = 0;
334	}
335	end_buffer_async_read(bh, uptodate);
336	}
337
338	/*
339	* Completion handler for block_write_full_page() - pages which are unlocked
340	* during I/O, and which have PageWriteback cleared upon I/O completion.
341	*/
342	void end_buffer_async_write(struct buffer_head *bh, int uptodate)
343	{
344	unsigned long flags;
345	struct buffer_head *first;
346	struct buffer_head *tmp;
347	struct page *page;
348
349	BUG_ON(!buffer_async_write(bh));
350
351	page = bh->b_page;
352	if (uptodate) {
353	set_buffer_uptodate(bh);
354	} else {
355	buffer_io_error(bh, ", lost async page write");
356	mark_buffer_write_io_error(bh);
357	clear_buffer_uptodate(bh);
358	SetPageError(page);
359	}
360
361	first = page_buffers(page);
362	spin_lock_irqsave(&first->b_uptodate_lock, flags);
363
364	clear_buffer_async_write(bh);
365	unlock_buffer(bh);
366	tmp = bh->b_this_page;
367	while (tmp != bh) {
368	if (buffer_async_write(tmp)) {
369	BUG_ON(!buffer_locked(tmp));
370	goto still_busy;
371	}
372	tmp = tmp->b_this_page;
373	}
374	spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
375	end_page_writeback(page);
376	return;
377
378	still_busy:
379	spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
380	return;
381	}
382	EXPORT_SYMBOL(end_buffer_async_write);
383
384	/*
385	* If a page's buffers are under async readin (end_buffer_async_read
386	* completion) then there is a possibility that another thread of
387	* control could lock one of the buffers after it has completed
388	* but while some of the other buffers have not completed. This
389	* locked buffer would confuse end_buffer_async_read() into not unlocking
390	* the page. So the absence of BH_Async_Read tells end_buffer_async_read()
391	* that this buffer is not under async I/O.
392	*
393	* The page comes unlocked when it has no locked buffer_async buffers
394	* left.
395	*
396	* PageLocked prevents anyone starting new async I/O reads any of
397	* the buffers.
398	*
399	* PageWriteback is used to prevent simultaneous writeout of the same
400	* page.
401	*
402	* PageLocked prevents anyone from starting writeback of a page which is
403	* under read I/O (PageWriteback is only ever set against a locked page).
404	*/
405	static void mark_buffer_async_read(struct buffer_head *bh)
406	{
407	bh->b_end_io = end_buffer_async_read_io;
408	set_buffer_async_read(bh);
409	}
410
411	static void mark_buffer_async_write_endio(struct buffer_head *bh,
412	bh_end_io_t *handler)
413	{
414	bh->b_end_io = handler;
415	set_buffer_async_write(bh);
416	}
417
418	void mark_buffer_async_write(struct buffer_head *bh)
419	{
420	mark_buffer_async_write_endio(bh, end_buffer_async_write);
421	}
422	EXPORT_SYMBOL(mark_buffer_async_write);
423
424
425	/*
426	* fs/buffer.c contains helper functions for buffer-backed address space's
427	* fsync functions. A common requirement for buffer-based filesystems is
428	* that certain data from the backing blockdev needs to be written out for
429	* a successful fsync(). For example, ext2 indirect blocks need to be
430	* written back and waited upon before fsync() returns.
431	*
432	* The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
433	* inode_has_buffers() and invalidate_inode_buffers() are provided for the
434	* management of a list of dependent buffers at ->i_mapping->private_list.
435	*
436	* Locking is a little subtle: try_to_free_buffers() will remove buffers
437	* from their controlling inode's queue when they are being freed. But
438	* try_to_free_buffers() will be operating against the blockdev mapping
439	* at the time, not against the S_ISREG file which depends on those buffers.
440	* So the locking for private_list is via the private_lock in the address_space
441	* which backs the buffers. Which is different from the address_space
442	* against which the buffers are listed. So for a particular address_space,
443	* mapping->private_lock does not protect mapping->private_list! In fact,
444	* mapping->private_list will always be protected by the backing blockdev's
445	* ->private_lock.
446	*
447	* Which introduces a requirement: all buffers on an address_space's
448	* ->private_list must be from the same address_space: the blockdev's.
449	*
450	* address_spaces which do not place buffers at ->private_list via these
451	* utility functions are free to use private_lock and private_list for
452	* whatever they want. The only requirement is that list_empty(private_list)
453	* be true at clear_inode() time.
454	*
455	* FIXME: clear_inode should not call invalidate_inode_buffers(). The
456	* filesystems should do that. invalidate_inode_buffers() should just go
457	* BUG_ON(!list_empty).
458	*
459	* FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
460	* take an address_space, not an inode. And it should be called
461	* mark_buffer_dirty_fsync() to clearly define why those buffers are being
462	* queued up.
463	*
464	* FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
465	* list if it is already on a list. Because if the buffer is on a list,
466	* it must already be on the right one. If not, the filesystem is being
467	* silly. This will save a ton of locking. But first we have to ensure
468	* that buffers are taken off the old inode's list when they are freed
469	* (presumably in truncate). That requires careful auditing of all
470	* filesystems (do it inside bforget()). It could also be done by bringing
471	* b_inode back.
472	*/
473
474	/*
475	* The buffer's backing address_space's private_lock must be held
476	*/
477	static void __remove_assoc_queue(struct buffer_head *bh)
478	{
479	list_del_init(&bh->b_assoc_buffers);
480	WARN_ON(!bh->b_assoc_map);
481	bh->b_assoc_map = NULL;
482	}
483
484	int inode_has_buffers(struct inode *inode)
485	{
486	return !list_empty(&inode->i_data.private_list);
487	}
488
489	/*
490	* osync is designed to support O_SYNC io. It waits synchronously for
491	* all already-submitted IO to complete, but does not queue any new
492	* writes to the disk.
493	*
494	* To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
495	* you dirty the buffers, and then use osync_inode_buffers to wait for
496	* completion. Any other dirty buffers which are not yet queued for
497	* write will not be flushed to disk by the osync.
498	*/
499	static int osync_buffers_list(spinlock_t lock, struct list_head list)
500	{
501	struct buffer_head *bh;
502	struct list_head *p;
503	int err = 0;
504
505	spin_lock(lock);
506	repeat:
507	list_for_each_prev(p, list) {
508	bh = BH_ENTRY(p);
509	if (buffer_locked(bh)) {
510	get_bh(bh);
511	spin_unlock(lock);
512	wait_on_buffer(bh);
513	if (!buffer_uptodate(bh))
514	err = -EIO;
515	brelse(bh);
516	spin_lock(lock);
517	goto repeat;
518	}
519	}
520	spin_unlock(lock);
521	return err;
522	}
523
524	void emergency_thaw_bdev(struct super_block *sb)
525	{
526	while (sb->s_bdev && !thaw_bdev(sb->s_bdev))
527	printk(KERN_WARNING "Emergency Thaw on %pg\n", sb->s_bdev);
528	}
529
530	/**
531	* sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
532	* @mapping: the mapping which wants those buffers written
533	*
534	* Starts I/O against the buffers at mapping->private_list, and waits upon
535	* that I/O.
536	*
537	* Basically, this is a convenience function for fsync().
538	* @mapping is a file or directory which needs those buffers to be written for
539	* a successful fsync().
540	*/
541	int sync_mapping_buffers(struct address_space *mapping)
542	{
543	struct address_space *buffer_mapping = mapping->private_data;
544
545	if (buffer_mapping == NULL \|\| list_empty(&mapping->private_list))
546	return 0;
547
548	return fsync_buffers_list(&buffer_mapping->private_lock,
549	&mapping->private_list);
550	}
551	EXPORT_SYMBOL(sync_mapping_buffers);
552
553	/*
554	* Called when we've recently written block `bblock', and it is known that
555	* `bblock' was for a buffer_boundary() buffer. This means that the block at
556	* `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
557	* dirty, schedule it for IO. So that indirects merge nicely with their data.
558	*/
559	void write_boundary_block(struct block_device *bdev,
560	sector_t bblock, unsigned blocksize)
561	{
562	struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
563	if (bh) {
564	if (buffer_dirty(bh))
565	ll_rw_block(REQ_OP_WRITE, 0, 1, &bh);
566	put_bh(bh);
567	}
568	}
569
570	void mark_buffer_dirty_inode(struct buffer_head bh, struct inode inode)
571	{
572	struct address_space *mapping = inode->i_mapping;
573	struct address_space *buffer_mapping = bh->b_page->mapping;
574
575	mark_buffer_dirty(bh);
576	if (!mapping->private_data) {
577	mapping->private_data = buffer_mapping;
578	} else {
579	BUG_ON(mapping->private_data != buffer_mapping);
580	}
581	if (!bh->b_assoc_map) {
582	spin_lock(&buffer_mapping->private_lock);
583	list_move_tail(&bh->b_assoc_buffers,
584	&mapping->private_list);
585	bh->b_assoc_map = mapping;
586	spin_unlock(&buffer_mapping->private_lock);
587	}
588	}
589	EXPORT_SYMBOL(mark_buffer_dirty_inode);
590
591	/*
592	* Mark the page dirty, and set it dirty in the page cache, and mark the inode
593	* dirty.
594	*
595	* If warn is true, then emit a warning if the page is not uptodate and has
596	* not been truncated.
597	*
598	* The caller must hold lock_page_memcg().
599	*/
600	void __set_page_dirty(struct page page, struct address_space mapping,
601	int warn)
602	{
603	unsigned long flags;
604
605	xa_lock_irqsave(&mapping->i_pages, flags);
606	if (page->mapping) { /* Race with truncate? */
607	WARN_ON_ONCE(warn && !PageUptodate(page));
608	account_page_dirtied(page, mapping);
609	__xa_set_mark(&mapping->i_pages, page_index(page),
610	PAGECACHE_TAG_DIRTY);
611	}
612	xa_unlock_irqrestore(&mapping->i_pages, flags);
613	}
614	EXPORT_SYMBOL_GPL(__set_page_dirty);
615
616	/*
617	* Add a page to the dirty page list.
618	*
619	* It is a sad fact of life that this function is called from several places
620	* deeply under spinlocking. It may not sleep.
621	*
622	* If the page has buffers, the uptodate buffers are set dirty, to preserve
623	* dirty-state coherency between the page and the buffers. It the page does
624	* not have buffers then when they are later attached they will all be set
625	* dirty.
626	*
627	* The buffers are dirtied before the page is dirtied. There's a small race
628	* window in which a writepage caller may see the page cleanness but not the
629	* buffer dirtiness. That's fine. If this code were to set the page dirty
630	* before the buffers, a concurrent writepage caller could clear the page dirty
631	* bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
632	* page on the dirty page list.
633	*
634	* We use private_lock to lock against try_to_free_buffers while using the
635	* page's buffer list. Also use this to protect against clean buffers being
636	* added to the page after it was set dirty.
637	*
638	* FIXME: may need to call ->reservepage here as well. That's rather up to the
639	* address_space though.
640	*/
641	int __set_page_dirty_buffers(struct page *page)
642	{
643	int newly_dirty;
644	struct address_space *mapping = page_mapping(page);
645
646	if (unlikely(!mapping))
647	return !TestSetPageDirty(page);
648
649	spin_lock(&mapping->private_lock);
650	if (page_has_buffers(page)) {
651	struct buffer_head *head = page_buffers(page);
652	struct buffer_head *bh = head;
653
654	do {
655	set_buffer_dirty(bh);
656	bh = bh->b_this_page;
657	} while (bh != head);
658	}
659	/*
660	* Lock out page's memcg migration to keep PageDirty
661	* synchronized with per-memcg dirty page counters.
662	*/
663	lock_page_memcg(page);
664	newly_dirty = !TestSetPageDirty(page);
665	spin_unlock(&mapping->private_lock);
666
667	if (newly_dirty)
668	__set_page_dirty(page, mapping, 1);
669
670	unlock_page_memcg(page);
671
672	if (newly_dirty)
673	__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
674
675	return newly_dirty;
676	}
677	EXPORT_SYMBOL(__set_page_dirty_buffers);
678
679	/*
680	* Write out and wait upon a list of buffers.
681	*
682	* We have conflicting pressures: we want to make sure that all
683	* initially dirty buffers get waited on, but that any subsequently
684	* dirtied buffers don't. After all, we don't want fsync to last
685	* forever if somebody is actively writing to the file.
686	*
687	* Do this in two main stages: first we copy dirty buffers to a
688	* temporary inode list, queueing the writes as we go. Then we clean
689	* up, waiting for those writes to complete.
690	*
691	* During this second stage, any subsequent updates to the file may end
692	* up refiling the buffer on the original inode's dirty list again, so
693	* there is a chance we will end up with a buffer queued for write but
694	* not yet completed on that list. So, as a final cleanup we go through
695	* the osync code to catch these locked, dirty buffers without requeuing
696	* any newly dirty buffers for write.
697	*/
698	static int fsync_buffers_list(spinlock_t lock, struct list_head list)
699	{
700	struct buffer_head *bh;
701	struct list_head tmp;
702	struct address_space *mapping;
703	int err = 0, err2;
704	struct blk_plug plug;
705
706	INIT_LIST_HEAD(&tmp);
707	blk_start_plug(&plug);
708
709	spin_lock(lock);
710	while (!list_empty(list)) {
711	bh = BH_ENTRY(list->next);
712	mapping = bh->b_assoc_map;
713	__remove_assoc_queue(bh);
714	/* Avoid race with mark_buffer_dirty_inode() which does
715	* a lockless check and we rely on seeing the dirty bit */
716	smp_mb();
717	if (buffer_dirty(bh) \|\| buffer_locked(bh)) {
718	list_add(&bh->b_assoc_buffers, &tmp);
719	bh->b_assoc_map = mapping;
720	if (buffer_dirty(bh)) {
721	get_bh(bh);
722	spin_unlock(lock);
723	/*
724	* Ensure any pending I/O completes so that
725	* write_dirty_buffer() actually writes the
726	* current contents - it is a noop if I/O is
727	* still in flight on potentially older
728	* contents.
729	*/
730	write_dirty_buffer(bh, REQ_SYNC);
731
732	/*
733	* Kick off IO for the previous mapping. Note
734	* that we will not run the very last mapping,
735	* wait_on_buffer() will do that for us
736	* through sync_buffer().
737	*/
738	brelse(bh);
739	spin_lock(lock);
740	}
741	}
742	}
743
744	spin_unlock(lock);
745	blk_finish_plug(&plug);
746	spin_lock(lock);
747
748	while (!list_empty(&tmp)) {
749	bh = BH_ENTRY(tmp.prev);
750	get_bh(bh);
751	mapping = bh->b_assoc_map;
752	__remove_assoc_queue(bh);
753	/* Avoid race with mark_buffer_dirty_inode() which does
754	* a lockless check and we rely on seeing the dirty bit */
755	smp_mb();
756	if (buffer_dirty(bh)) {
757	list_add(&bh->b_assoc_buffers,
758	&mapping->private_list);
759	bh->b_assoc_map = mapping;
760	}
761	spin_unlock(lock);
762	wait_on_buffer(bh);
763	if (!buffer_uptodate(bh))
764	err = -EIO;
765	brelse(bh);
766	spin_lock(lock);
767	}
768
769	spin_unlock(lock);
770	err2 = osync_buffers_list(lock, list);
771	if (err)
772	return err;
773	else
774	return err2;
775	}
776
777	/*
778	* Invalidate any and all dirty buffers on a given inode. We are
779	* probably unmounting the fs, but that doesn't mean we have already
780	* done a sync(). Just drop the buffers from the inode list.
781	*
782	* NOTE: we take the inode's blockdev's mapping's private_lock. Which
783	* assumes that all the buffers are against the blockdev. Not true
784	* for reiserfs.
785	*/
786	void invalidate_inode_buffers(struct inode *inode)
787	{
788	if (inode_has_buffers(inode)) {
789	struct address_space *mapping = &inode->i_data;
790	struct list_head *list = &mapping->private_list;
791	struct address_space *buffer_mapping = mapping->private_data;
792
793	spin_lock(&buffer_mapping->private_lock);
794	while (!list_empty(list))
795	__remove_assoc_queue(BH_ENTRY(list->next));
796	spin_unlock(&buffer_mapping->private_lock);
797	}
798	}
799	EXPORT_SYMBOL(invalidate_inode_buffers);
800
801	/*
802	* Remove any clean buffers from the inode's buffer list. This is called
803	* when we're trying to free the inode itself. Those buffers can pin it.
804	*
805	* Returns true if all buffers were removed.
806	*/
807	int remove_inode_buffers(struct inode *inode)
808	{
809	int ret = 1;
810
811	if (inode_has_buffers(inode)) {
812	struct address_space *mapping = &inode->i_data;
813	struct list_head *list = &mapping->private_list;
814	struct address_space *buffer_mapping = mapping->private_data;
815
816	spin_lock(&buffer_mapping->private_lock);
817	while (!list_empty(list)) {
818	struct buffer_head *bh = BH_ENTRY(list->next);
819	if (buffer_dirty(bh)) {
820	ret = 0;
821	break;
822	}
823	__remove_assoc_queue(bh);
824	}
825	spin_unlock(&buffer_mapping->private_lock);
826	}
827	return ret;
828	}
829
830	/*
831	* Create the appropriate buffers when given a page for data area and
832	* the size of each buffer.. Use the bh->b_this_page linked list to
833	* follow the buffers created. Return NULL if unable to create more
834	* buffers.
835	*
836	* The retry flag is used to differentiate async IO (paging, swapping)
837	* which may not fail from ordinary buffer allocations.
838	*/
839	struct buffer_head alloc_page_buffers(struct page page, unsigned long size,
840	bool retry)
841	{
842	struct buffer_head bh, head;
843	gfp_t gfp = GFP_NOFS \| __GFP_ACCOUNT;
844	long offset;
845	struct mem_cgroup memcg, old_memcg;
846
847	if (retry)
848	gfp \|= __GFP_NOFAIL;
849
850	/* The page lock pins the memcg */
851	memcg = page_memcg(page);
852	old_memcg = set_active_memcg(memcg);
853
854	head = NULL;
855	offset = PAGE_SIZE;
856	while ((offset -= size) >= 0) {
857	bh = alloc_buffer_head(gfp);
858	if (!bh)
859	goto no_grow;
860
861	bh->b_this_page = head;
862	bh->b_blocknr = -1;
863	head = bh;
864
865	bh->b_size = size;
866
867	/* Link the buffer to its page */
868	set_bh_page(bh, page, offset);
869	}
870	out:
871	set_active_memcg(old_memcg);
872	return head;
873	/*
874	* In case anything failed, we just free everything we got.
875	*/
876	no_grow:
877	if (head) {
878	do {
879	bh = head;
880	head = head->b_this_page;
881	free_buffer_head(bh);
882	} while (head);
883	}
884
885	goto out;
886	}
887	EXPORT_SYMBOL_GPL(alloc_page_buffers);
888
889	static inline void
890	link_dev_buffers(struct page page, struct buffer_head head)
891	{
892	struct buffer_head bh, tail;
893
894	bh = head;
895	do {
896	tail = bh;
897	bh = bh->b_this_page;
898	} while (bh);
899	tail->b_this_page = head;
900	attach_page_private(page, head);
901	}
902
903	static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
904	{
905	sector_t retval = ~((sector_t)0);
906	loff_t sz = i_size_read(bdev->bd_inode);
907
908	if (sz) {
909	unsigned int sizebits = blksize_bits(size);
910	retval = (sz >> sizebits);
911	}
912	return retval;
913	}
914
915	/*
916	* Initialise the state of a blockdev page's buffers.
917	*/
918	static sector_t
919	init_page_buffers(struct page page, struct block_device bdev,
920	sector_t block, int size)
921	{
922	struct buffer_head *head = page_buffers(page);
923	struct buffer_head *bh = head;
924	int uptodate = PageUptodate(page);
925	sector_t end_block = blkdev_max_block(I_BDEV(bdev->bd_inode), size);
926
927	do {
928	if (!buffer_mapped(bh)) {
929	bh->b_end_io = NULL;
930	bh->b_private = NULL;
931	bh->b_bdev = bdev;
932	bh->b_blocknr = block;
933	if (uptodate)
934	set_buffer_uptodate(bh);
935	if (block < end_block)
936	set_buffer_mapped(bh);
937	}
938	block++;
939	bh = bh->b_this_page;
940	} while (bh != head);
941
942	/*
943	* Caller needs to validate requested block against end of device.
944	*/
945	return end_block;
946	}
947
948	/*
949	* Create the page-cache page that contains the requested block.
950	*
951	* This is used purely for blockdev mappings.
952	*/
953	static int
954	grow_dev_page(struct block_device *bdev, sector_t block,
955	pgoff_t index, int size, int sizebits, gfp_t gfp)
956	{
957	struct inode *inode = bdev->bd_inode;
958	struct page *page;
959	struct buffer_head *bh;
960	sector_t end_block;
961	int ret = 0;
962	gfp_t gfp_mask;
963
964	gfp_mask = mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS) \| gfp;
965
966	/*
967	* XXX: __getblk_slow() can not really deal with failure and
968	* will endlessly loop on improvised global reclaim. Prefer
969	* looping in the allocator rather than here, at least that
970	* code knows what it's doing.
971	*/
972	gfp_mask \|= __GFP_NOFAIL;
973
974	page = find_or_create_page(inode->i_mapping, index, gfp_mask);
975
976	BUG_ON(!PageLocked(page));
977
978	if (page_has_buffers(page)) {
979	bh = page_buffers(page);
980	if (bh->b_size == size) {
981	end_block = init_page_buffers(page, bdev,
982	(sector_t)index << sizebits,
983	size);
984	goto done;
985	}
986	if (!try_to_free_buffers(page))
987	goto failed;
988	}
989
990	/*
991	* Allocate some buffers for this page
992	*/
993	bh = alloc_page_buffers(page, size, true);
994
995	/*
996	* Link the page to the buffers and initialise them. Take the
997	* lock to be atomic wrt __find_get_block(), which does not
998	* run under the page lock.
999	*/
1000	spin_lock(&inode->i_mapping->private_lock);
1001	link_dev_buffers(page, bh);
1002	end_block = init_page_buffers(page, bdev, (sector_t)index << sizebits,
1003	size);
1004	spin_unlock(&inode->i_mapping->private_lock);
1005	done:
1006	ret = (block < end_block) ? 1 : -ENXIO;
1007	failed:
1008	unlock_page(page);
1009	put_page(page);
1010	return ret;
1011	}
1012
1013	/*
1014	* Create buffers for the specified block device block's page. If
1015	* that page was dirty, the buffers are set dirty also.
1016	*/
1017	static int
1018	grow_buffers(struct block_device *bdev, sector_t block, int size, gfp_t gfp)
1019	{
1020	pgoff_t index;
1021	int sizebits;
1022
1023	sizebits = PAGE_SHIFT - __ffs(size);
1024	index = block >> sizebits;
1025
1026	/*
1027	* Check for a block which wants to lie outside our maximum possible
1028	* pagecache index. (this comparison is done using sector_t types).
1029	*/
1030	if (unlikely(index != block >> sizebits)) {
1031	printk(KERN_ERR "%s: requested out-of-range block %llu for "
1032	"device %pg\n",
1033	__func__, (unsigned long long)block,
1034	bdev);
1035	return -EIO;
1036	}
1037
1038	/* Create a page with the proper size buffers.. */
1039	return grow_dev_page(bdev, block, index, size, sizebits, gfp);
1040	}
1041
1042	static struct buffer_head *
1043	__getblk_slow(struct block_device *bdev, sector_t block,
1044	unsigned size, gfp_t gfp)
1045	{
1046	/* Size must be multiple of hard sectorsize */
1047	if (unlikely(size & (bdev_logical_block_size(bdev)-1) \|\|
1048	(size < 512 \|\| size > PAGE_SIZE))) {
1049	printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1050	size);
1051	printk(KERN_ERR "logical block size: %d\n",
1052	bdev_logical_block_size(bdev));
1053
1054	dump_stack();
1055	return NULL;
1056	}
1057
1058	for (;;) {
1059	struct buffer_head *bh;
1060	int ret;
1061
1062	bh = __find_get_block(bdev, block, size);
1063	if (bh)
1064	return bh;
1065
1066	ret = grow_buffers(bdev, block, size, gfp);
1067	if (ret < 0)
1068	return NULL;
1069	}
1070	}
1071
1072	/*
1073	* The relationship between dirty buffers and dirty pages:
1074	*
1075	* Whenever a page has any dirty buffers, the page's dirty bit is set, and
1076	* the page is tagged dirty in the page cache.
1077	*
1078	* At all times, the dirtiness of the buffers represents the dirtiness of
1079	* subsections of the page. If the page has buffers, the page dirty bit is
1080	* merely a hint about the true dirty state.
1081	*
1082	* When a page is set dirty in its entirety, all its buffers are marked dirty
1083	* (if the page has buffers).
1084	*
1085	* When a buffer is marked dirty, its page is dirtied, but the page's other
1086	* buffers are not.
1087	*
1088	* Also. When blockdev buffers are explicitly read with bread(), they
1089	* individually become uptodate. But their backing page remains not
1090	* uptodate - even if all of its buffers are uptodate. A subsequent
1091	* block_read_full_page() against that page will discover all the uptodate
1092	* buffers, will set the page uptodate and will perform no I/O.
1093	*/
1094
1095	/**
1096	* mark_buffer_dirty - mark a buffer_head as needing writeout
1097	* @bh: the buffer_head to mark dirty
1098	*
1099	* mark_buffer_dirty() will set the dirty bit against the buffer, then set
1100	* its backing page dirty, then tag the page as dirty in the page cache
1101	* and then attach the address_space's inode to its superblock's dirty
1102	* inode list.
1103	*
1104	* mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
1105	* i_pages lock and mapping->host->i_lock.
1106	*/
1107	void mark_buffer_dirty(struct buffer_head *bh)
1108	{
1109	WARN_ON_ONCE(!buffer_uptodate(bh));
1110
1111	trace_block_dirty_buffer(bh);
1112
1113	/*
1114	* Very carefully optimize the it-is-already-dirty case.
1115	*
1116	* Don't let the final "is it dirty" escape to before we
1117	* perhaps modified the buffer.
1118	*/
1119	if (buffer_dirty(bh)) {
1120	smp_mb();
1121	if (buffer_dirty(bh))
1122	return;
1123	}
1124
1125	if (!test_set_buffer_dirty(bh)) {
1126	struct page *page = bh->b_page;
1127	struct address_space *mapping = NULL;
1128
1129	lock_page_memcg(page);
1130	if (!TestSetPageDirty(page)) {
1131	mapping = page_mapping(page);
1132	if (mapping)
1133	__set_page_dirty(page, mapping, 0);
1134	}
1135	unlock_page_memcg(page);
1136	if (mapping)
1137	__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
1138	}
1139	}
1140	EXPORT_SYMBOL(mark_buffer_dirty);
1141
1142	void mark_buffer_write_io_error(struct buffer_head *bh)
1143	{
1144	struct super_block *sb;
1145
1146	set_buffer_write_io_error(bh);
1147	/* FIXME: do we need to set this in both places? */
1148	if (bh->b_page && bh->b_page->mapping)
1149	mapping_set_error(bh->b_page->mapping, -EIO);
1150	if (bh->b_assoc_map)
1151	mapping_set_error(bh->b_assoc_map, -EIO);
1152	rcu_read_lock();
1153	sb = READ_ONCE(bh->b_bdev->bd_super);
1154	if (sb)
1155	errseq_set(&sb->s_wb_err, -EIO);
1156	rcu_read_unlock();
1157	}
1158	EXPORT_SYMBOL(mark_buffer_write_io_error);
1159
1160	/*
1161	* Decrement a buffer_head's reference count. If all buffers against a page
1162	* have zero reference count, are clean and unlocked, and if the page is clean
1163	* and unlocked then try_to_free_buffers() may strip the buffers from the page
1164	* in preparation for freeing it (sometimes, rarely, buffers are removed from
1165	* a page but it ends up not being freed, and buffers may later be reattached).
1166	*/
1167	void __brelse(struct buffer_head * buf)
1168	{
1169	if (atomic_read(&buf->b_count)) {
1170	put_bh(buf);
1171	return;
1172	}
1173	WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1174	}
1175	EXPORT_SYMBOL(__brelse);
1176
1177	/*
1178	* bforget() is like brelse(), except it discards any
1179	* potentially dirty data.
1180	*/
1181	void __bforget(struct buffer_head *bh)
1182	{
1183	clear_buffer_dirty(bh);
1184	if (bh->b_assoc_map) {
1185	struct address_space *buffer_mapping = bh->b_page->mapping;
1186
1187	spin_lock(&buffer_mapping->private_lock);
1188	list_del_init(&bh->b_assoc_buffers);
1189	bh->b_assoc_map = NULL;
1190	spin_unlock(&buffer_mapping->private_lock);
1191	}
1192	__brelse(bh);
1193	}
1194	EXPORT_SYMBOL(__bforget);
1195
1196	static struct buffer_head __bread_slow(struct buffer_head bh)
1197	{
1198	lock_buffer(bh);
1199	if (buffer_uptodate(bh)) {
1200	unlock_buffer(bh);
1201	return bh;
1202	} else {
1203	get_bh(bh);
1204	bh->b_end_io = end_buffer_read_sync;
1205	submit_bh(REQ_OP_READ, 0, bh);
1206	wait_on_buffer(bh);
1207	if (buffer_uptodate(bh))
1208	return bh;
1209	}
1210	brelse(bh);
1211	return NULL;
1212	}
1213
1214	/*
1215	* Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
1216	* The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
1217	* refcount elevated by one when they're in an LRU. A buffer can only appear
1218	* once in a particular CPU's LRU. A single buffer can be present in multiple
1219	* CPU's LRUs at the same time.
1220	*
1221	* This is a transparent caching front-end to sb_bread(), sb_getblk() and
1222	* sb_find_get_block().
1223	*
1224	* The LRUs themselves only need locking against invalidate_bh_lrus. We use
1225	* a local interrupt disable for that.
1226	*/
1227
1228	#define BH_LRU_SIZE 16
1229
1230	struct bh_lru {
1231	struct buffer_head *bhs[BH_LRU_SIZE];
1232	};
1233
1234	static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1235
1236	#ifdef CONFIG_SMP
1237	#define bh_lru_lock() local_irq_disable()
1238	#define bh_lru_unlock() local_irq_enable()
1239	#else
1240	#define bh_lru_lock() preempt_disable()
1241	#define bh_lru_unlock() preempt_enable()
1242	#endif
1243
1244	static inline void check_irqs_on(void)
1245	{
1246	#ifdef irqs_disabled
1247	BUG_ON(irqs_disabled());
1248	#endif
1249	}
1250
1251	/*
1252	* Install a buffer_head into this cpu's LRU. If not already in the LRU, it is
1253	* inserted at the front, and the buffer_head at the back if any is evicted.
1254	* Or, if already in the LRU it is moved to the front.
1255	*/
1256	static void bh_lru_install(struct buffer_head *bh)
1257	{
1258	struct buffer_head *evictee = bh;
1259	struct bh_lru *b;
1260	int i;
1261
1262	check_irqs_on();
1263	/*
1264	* the refcount of buffer_head in bh_lru prevents dropping the
1265	* attached page(i.e., try_to_free_buffers) so it could cause
1266	* failing page migration.
1267	* Skip putting upcoming bh into bh_lru until migration is done.
1268	*/
1269	if (lru_cache_disabled())
1270	return;
1271
1272	bh_lru_lock();
1273
1274	b = this_cpu_ptr(&bh_lrus);
1275	for (i = 0; i < BH_LRU_SIZE; i++) {
1276	swap(evictee, b->bhs[i]);
1277	if (evictee == bh) {
1278	bh_lru_unlock();
1279	return;
1280	}
1281	}
1282
1283	get_bh(bh);
1284	bh_lru_unlock();
1285	brelse(evictee);
1286	}
1287
1288	/*
1289	* Look up the bh in this cpu's LRU. If it's there, move it to the head.
1290	*/
1291	static struct buffer_head *
1292	lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
1293	{
1294	struct buffer_head *ret = NULL;
1295	unsigned int i;
1296
1297	check_irqs_on();
1298	bh_lru_lock();
1299	for (i = 0; i < BH_LRU_SIZE; i++) {
1300	struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
1301
1302	if (bh && bh->b_blocknr == block && bh->b_bdev == bdev &&
1303	bh->b_size == size) {
1304	if (i) {
1305	while (i) {
1306	__this_cpu_write(bh_lrus.bhs[i],
1307	__this_cpu_read(bh_lrus.bhs[i - 1]));
1308	i--;
1309	}
1310	__this_cpu_write(bh_lrus.bhs[0], bh);
1311	}
1312	get_bh(bh);
1313	ret = bh;
1314	break;
1315	}
1316	}
1317	bh_lru_unlock();
1318	return ret;
1319	}
1320
1321	/*
1322	* Perform a pagecache lookup for the matching buffer. If it's there, refresh
1323	* it in the LRU and mark it as accessed. If it is not present then return
1324	* NULL
1325	*/
1326	struct buffer_head *
1327	__find_get_block(struct block_device *bdev, sector_t block, unsigned size)
1328	{
1329	struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1330
1331	if (bh == NULL) {
1332	/* __find_get_block_slow will mark the page accessed */
1333	bh = __find_get_block_slow(bdev, block);
1334	if (bh)
1335	bh_lru_install(bh);
1336	} else
1337	touch_buffer(bh);
1338
1339	return bh;
1340	}
1341	EXPORT_SYMBOL(__find_get_block);
1342
1343	/*
1344	* __getblk_gfp() will locate (and, if necessary, create) the buffer_head
1345	* which corresponds to the passed block_device, block and size. The
1346	* returned buffer has its reference count incremented.
1347	*
1348	* __getblk_gfp() will lock up the machine if grow_dev_page's
1349	* try_to_free_buffers() attempt is failing. FIXME, perhaps?
1350	*/
1351	struct buffer_head *
1352	__getblk_gfp(struct block_device *bdev, sector_t block,
1353	unsigned size, gfp_t gfp)
1354	{
1355	struct buffer_head *bh = __find_get_block(bdev, block, size);
1356
1357	might_sleep();
1358	if (bh == NULL)
1359	bh = __getblk_slow(bdev, block, size, gfp);
1360	return bh;
1361	}
1362	EXPORT_SYMBOL(__getblk_gfp);
1363
1364	/*
1365	* Do async read-ahead on a buffer..
1366	*/
1367	void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
1368	{
1369	struct buffer_head *bh = __getblk(bdev, block, size);
1370	if (likely(bh)) {
1371	ll_rw_block(REQ_OP_READ, REQ_RAHEAD, 1, &bh);
1372	brelse(bh);
1373	}
1374	}
1375	EXPORT_SYMBOL(__breadahead);
1376
1377	void __breadahead_gfp(struct block_device *bdev, sector_t block, unsigned size,
1378	gfp_t gfp)
1379	{
1380	struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp);
1381	if (likely(bh)) {
1382	ll_rw_block(REQ_OP_READ, REQ_RAHEAD, 1, &bh);
1383	brelse(bh);
1384	}
1385	}
1386	EXPORT_SYMBOL(__breadahead_gfp);
1387
1388	/**
1389	* __bread_gfp() - reads a specified block and returns the bh
1390	* @bdev: the block_device to read from
1391	* @block: number of block
1392	* @size: size (in bytes) to read
1393	* @gfp: page allocation flag
1394	*
1395	* Reads a specified block, and returns buffer head that contains it.
1396	* The page cache can be allocated from non-movable area
1397	* not to prevent page migration if you set gfp to zero.
1398	* It returns NULL if the block was unreadable.
1399	*/
1400	struct buffer_head *
1401	__bread_gfp(struct block_device *bdev, sector_t block,
1402	unsigned size, gfp_t gfp)
1403	{
1404	struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp);
1405
1406	if (likely(bh) && !buffer_uptodate(bh))
1407	bh = __bread_slow(bh);
1408	return bh;
1409	}
1410	EXPORT_SYMBOL(__bread_gfp);
1411
1412	static void __invalidate_bh_lrus(struct bh_lru *b)
1413	{
1414	int i;
1415
1416	for (i = 0; i < BH_LRU_SIZE; i++) {
1417	brelse(b->bhs[i]);
1418	b->bhs[i] = NULL;
1419	}
1420	}
1421	/*
1422	* invalidate_bh_lrus() is called rarely - but not only at unmount.
1423	* This doesn't race because it runs in each cpu either in irq
1424	* or with preempt disabled.
1425	*/
1426	static void invalidate_bh_lru(void *arg)
1427	{
1428	struct bh_lru *b = &get_cpu_var(bh_lrus);
1429
1430	__invalidate_bh_lrus(b);
1431	put_cpu_var(bh_lrus);
1432	}
1433
1434	bool has_bh_in_lru(int cpu, void *dummy)
1435	{
1436	struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
1437	int i;
1438
1439	for (i = 0; i < BH_LRU_SIZE; i++) {
1440	if (b->bhs[i])
1441	return true;
1442	}
1443
1444	return false;
1445	}
1446
1447	void invalidate_bh_lrus(void)
1448	{
1449	on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1);
1450	}
1451	EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
1452
1453	void invalidate_bh_lrus_cpu(int cpu)
1454	{
1455	struct bh_lru *b;
1456
1457	bh_lru_lock();
1458	b = per_cpu_ptr(&bh_lrus, cpu);
1459	__invalidate_bh_lrus(b);
1460	bh_lru_unlock();
1461	}
1462
1463	void set_bh_page(struct buffer_head *bh,
1464	struct page *page, unsigned long offset)
1465	{
1466	bh->b_page = page;
1467	BUG_ON(offset >= PAGE_SIZE);
1468	if (PageHighMem(page))
1469	/*
1470	* This catches illegal uses and preserves the offset:
1471	*/
1472	bh->b_data = (char *)(0 + offset);
1473	else
1474	bh->b_data = page_address(page) + offset;
1475	}
1476	EXPORT_SYMBOL(set_bh_page);
1477
1478	/*
1479	* Called when truncating a buffer on a page completely.
1480	*/
1481
1482	/* Bits that are cleared during an invalidate */
1483	#define BUFFER_FLAGS_DISCARD \
1484	(1 << BH_Mapped \| 1 << BH_New \| 1 << BH_Req \| \
1485	1 << BH_Delay \| 1 << BH_Unwritten)
1486
1487	static void discard_buffer(struct buffer_head * bh)
1488	{
1489	unsigned long b_state, b_state_old;
1490
1491	lock_buffer(bh);
1492	clear_buffer_dirty(bh);
1493	bh->b_bdev = NULL;
1494	b_state = bh->b_state;
1495	for (;;) {
1496	b_state_old = cmpxchg(&bh->b_state, b_state,
1497	(b_state & ~BUFFER_FLAGS_DISCARD));
1498	if (b_state_old == b_state)
1499	break;
1500	b_state = b_state_old;
1501	}
1502	unlock_buffer(bh);
1503	}
1504
1505	/**
1506	* block_invalidatepage - invalidate part or all of a buffer-backed page
1507	*
1508	* @page: the page which is affected
1509	* @offset: start of the range to invalidate
1510	* @length: length of the range to invalidate
1511	*
1512	* block_invalidatepage() is called when all or part of the page has become
1513	* invalidated by a truncate operation.
1514	*
1515	* block_invalidatepage() does not have to release all buffers, but it must
1516	* ensure that no dirty buffer is left outside @offset and that no I/O
1517	* is underway against any of the blocks which are outside the truncation
1518	* point. Because the caller is about to free (and possibly reuse) those
1519	* blocks on-disk.
1520	*/
1521	void block_invalidatepage(struct page *page, unsigned int offset,
1522	unsigned int length)
1523	{
1524	struct buffer_head head, bh, *next;
1525	unsigned int curr_off = 0;
1526	unsigned int stop = length + offset;
1527
1528	BUG_ON(!PageLocked(page));
1529	if (!page_has_buffers(page))
1530	goto out;
1531
1532	/*
1533	* Check for overflow
1534	*/
1535	BUG_ON(stop > PAGE_SIZE \|\| stop < length);
1536
1537	head = page_buffers(page);
1538	bh = head;
1539	do {
1540	unsigned int next_off = curr_off + bh->b_size;
1541	next = bh->b_this_page;
1542
1543	/*
1544	* Are we still fully in range ?
1545	*/
1546	if (next_off > stop)
1547	goto out;
1548
1549	/*
1550	* is this block fully invalidated?
1551	*/
1552	if (offset <= curr_off)
1553	discard_buffer(bh);
1554	curr_off = next_off;
1555	bh = next;
1556	} while (bh != head);
1557
1558	/*
1559	* We release buffers only if the entire page is being invalidated.
1560	* The get_block cached value has been unconditionally invalidated,
1561	* so real IO is not possible anymore.
1562	*/
1563	if (length == PAGE_SIZE)
1564	try_to_release_page(page, 0);
1565	out:
1566	return;
1567	}
1568	EXPORT_SYMBOL(block_invalidatepage);
1569
1570
1571	/*
1572	* We attach and possibly dirty the buffers atomically wrt
1573	* __set_page_dirty_buffers() via private_lock. try_to_free_buffers
1574	* is already excluded via the page lock.
1575	*/
1576	void create_empty_buffers(struct page *page,
1577	unsigned long blocksize, unsigned long b_state)
1578	{
1579	struct buffer_head bh, head, *tail;
1580
1581	head = alloc_page_buffers(page, blocksize, true);
1582	bh = head;
1583	do {
1584	bh->b_state \|= b_state;
1585	tail = bh;
1586	bh = bh->b_this_page;
1587	} while (bh);
1588	tail->b_this_page = head;
1589
1590	spin_lock(&page->mapping->private_lock);
1591	if (PageUptodate(page) \|\| PageDirty(page)) {
1592	bh = head;
1593	do {
1594	if (PageDirty(page))
1595	set_buffer_dirty(bh);
1596	if (PageUptodate(page))
1597	set_buffer_uptodate(bh);
1598	bh = bh->b_this_page;
1599	} while (bh != head);
1600	}
1601	attach_page_private(page, head);
1602	spin_unlock(&page->mapping->private_lock);
1603	}
1604	EXPORT_SYMBOL(create_empty_buffers);
1605
1606	/**
1607	* clean_bdev_aliases: clean a range of buffers in block device
1608	* @bdev: Block device to clean buffers in
1609	* @block: Start of a range of blocks to clean
1610	* @len: Number of blocks to clean
1611	*
1612	* We are taking a range of blocks for data and we don't want writeback of any
1613	* buffer-cache aliases starting from return from this function and until the
1614	* moment when something will explicitly mark the buffer dirty (hopefully that
1615	* will not happen until we will free that block ;-) We don't even need to mark
1616	* it not-uptodate - nobody can expect anything from a newly allocated buffer
1617	* anyway. We used to use unmap_buffer() for such invalidation, but that was
1618	* wrong. We definitely don't want to mark the alias unmapped, for example - it
1619	* would confuse anyone who might pick it with bread() afterwards...
1620	*
1621	* Also.. Note that bforget() doesn't lock the buffer. So there can be
1622	* writeout I/O going on against recently-freed buffers. We don't wait on that
1623	* I/O in bforget() - it's more efficient to wait on the I/O only if we really
1624	* need to. That happens here.
1625	*/
1626	void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
1627	{
1628	struct inode *bd_inode = bdev->bd_inode;
1629	struct address_space *bd_mapping = bd_inode->i_mapping;
1630	struct pagevec pvec;
1631	pgoff_t index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
1632	pgoff_t end;
1633	int i, count;
1634	struct buffer_head *bh;
1635	struct buffer_head *head;
1636
1637	end = (block + len - 1) >> (PAGE_SHIFT - bd_inode->i_blkbits);
1638	pagevec_init(&pvec);
1639	while (pagevec_lookup_range(&pvec, bd_mapping, &index, end)) {
1640	count = pagevec_count(&pvec);
1641	for (i = 0; i < count; i++) {
1642	struct page *page = pvec.pages[i];
1643
1644	if (!page_has_buffers(page))
1645	continue;
1646	/*
1647	* We use page lock instead of bd_mapping->private_lock
1648	* to pin buffers here since we can afford to sleep and
1649	* it scales better than a global spinlock lock.
1650	*/
1651	lock_page(page);
1652	/* Recheck when the page is locked which pins bhs */
1653	if (!page_has_buffers(page))
1654	goto unlock_page;
1655	head = page_buffers(page);
1656	bh = head;
1657	do {
1658	if (!buffer_mapped(bh) \|\| (bh->b_blocknr < block))
1659	goto next;
1660	if (bh->b_blocknr >= block + len)
1661	break;
1662	clear_buffer_dirty(bh);
1663	wait_on_buffer(bh);
1664	clear_buffer_req(bh);
1665	next:
1666	bh = bh->b_this_page;
1667	} while (bh != head);
1668	unlock_page:
1669	unlock_page(page);
1670	}
1671	pagevec_release(&pvec);
1672	cond_resched();
1673	/* End of range already reached? */
1674	if (index > end \|\| !index)
1675	break;
1676	}
1677	}
1678	EXPORT_SYMBOL(clean_bdev_aliases);
1679
1680	/*
1681	* Size is a power-of-two in the range 512..PAGE_SIZE,
1682	* and the case we care about most is PAGE_SIZE.
1683	*
1684	* So this could possibly be written with those
1685	* constraints in mind (relevant mostly if some
1686	* architecture has a slow bit-scan instruction)
1687	*/
1688	static inline int block_size_bits(unsigned int blocksize)
1689	{
1690	return ilog2(blocksize);
1691	}
1692
1693	static struct buffer_head create_page_buffers(struct page page, struct inode *inode, unsigned int b_state)
1694	{
1695	BUG_ON(!PageLocked(page));
1696
1697	if (!page_has_buffers(page))
1698	create_empty_buffers(page, 1 << READ_ONCE(inode->i_blkbits),
1699	b_state);
1700	return page_buffers(page);
1701	}
1702
1703	/*
1704	* NOTE! All mapped/uptodate combinations are valid:
1705	*
1706	* Mapped Uptodate Meaning
1707	*
1708	* No No "unknown" - must do get_block()
1709	* No Yes "hole" - zero-filled
1710	* Yes No "allocated" - allocated on disk, not read in
1711	* Yes Yes "valid" - allocated and up-to-date in memory.
1712	*
1713	* "Dirty" is valid only with the last case (mapped+uptodate).
1714	*/
1715
1716	/*
1717	* While block_write_full_page is writing back the dirty buffers under
1718	* the page lock, whoever dirtied the buffers may decide to clean them
1719	* again at any time. We handle that by only looking at the buffer
1720	* state inside lock_buffer().
1721	*
1722	* If block_write_full_page() is called for regular writeback
1723	* (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1724	* locked buffer. This only can happen if someone has written the buffer
1725	* directly, with submit_bh(). At the address_space level PageWriteback
1726	* prevents this contention from occurring.
1727	*
1728	* If block_write_full_page() is called with wbc->sync_mode ==
1729	* WB_SYNC_ALL, the writes are posted using REQ_SYNC; this
1730	* causes the writes to be flagged as synchronous writes.
1731	*/
1732	int __block_write_full_page(struct inode inode, struct page page,
1733	get_block_t get_block, struct writeback_control wbc,
1734	bh_end_io_t *handler)
1735	{
1736	int err;
1737	sector_t block;
1738	sector_t last_block;
1739	struct buffer_head bh, head;
1740	unsigned int blocksize, bbits;
1741	int nr_underway = 0;
1742	int write_flags = wbc_to_write_flags(wbc);
1743
1744	head = create_page_buffers(page, inode,
1745	(1 << BH_Dirty)\|(1 << BH_Uptodate));
1746
1747	/*
1748	* Be very careful. We have no exclusion from __set_page_dirty_buffers
1749	* here, and the (potentially unmapped) buffers may become dirty at
1750	* any time. If a buffer becomes dirty here after we've inspected it
1751	* then we just miss that fact, and the page stays dirty.
1752	*
1753	* Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1754	* handle that here by just cleaning them.
1755	*/
1756
1757	bh = head;
1758	blocksize = bh->b_size;
1759	bbits = block_size_bits(blocksize);
1760
1761	block = (sector_t)page->index << (PAGE_SHIFT - bbits);
1762	last_block = (i_size_read(inode) - 1) >> bbits;
1763
1764	/*
1765	* Get all the dirty buffers mapped to disk addresses and
1766	* handle any aliases from the underlying blockdev's mapping.
1767	*/
1768	do {
1769	if (block > last_block) {
1770	/*
1771	* mapped buffers outside i_size will occur, because
1772	* this page can be outside i_size when there is a
1773	* truncate in progress.
1774	*/
1775	/*
1776	* The buffer was zeroed by block_write_full_page()
1777	*/
1778	clear_buffer_dirty(bh);
1779	set_buffer_uptodate(bh);
1780	} else if ((!buffer_mapped(bh) \|\| buffer_delay(bh)) &&
1781	buffer_dirty(bh)) {
1782	WARN_ON(bh->b_size != blocksize);
1783	err = get_block(inode, block, bh, 1);
1784	if (err)
1785	goto recover;
1786	clear_buffer_delay(bh);
1787	if (buffer_new(bh)) {
1788	/* blockdev mappings never come here */
1789	clear_buffer_new(bh);
1790	clean_bdev_bh_alias(bh);
1791	}
1792	}
1793	bh = bh->b_this_page;
1794	block++;
1795	} while (bh != head);
1796
1797	do {
1798	if (!buffer_mapped(bh))
1799	continue;
1800	/*
1801	* If it's a fully non-blocking write attempt and we cannot
1802	* lock the buffer then redirty the page. Note that this can
1803	* potentially cause a busy-wait loop from writeback threads
1804	* and kswapd activity, but those code paths have their own
1805	* higher-level throttling.
1806	*/
1807	if (wbc->sync_mode != WB_SYNC_NONE) {
1808	lock_buffer(bh);
1809	} else if (!trylock_buffer(bh)) {
1810	redirty_page_for_writepage(wbc, page);
1811	continue;
1812	}
1813	if (test_clear_buffer_dirty(bh)) {
1814	mark_buffer_async_write_endio(bh, handler);
1815	} else {
1816	unlock_buffer(bh);
1817	}
1818	} while ((bh = bh->b_this_page) != head);
1819
1820	/*
1821	* The page and its buffers are protected by PageWriteback(), so we can
1822	* drop the bh refcounts early.
1823	*/
1824	BUG_ON(PageWriteback(page));
1825	set_page_writeback(page);
1826
1827	do {
1828	struct buffer_head *next = bh->b_this_page;
1829	if (buffer_async_write(bh)) {
1830	submit_bh_wbc(REQ_OP_WRITE, write_flags, bh,
1831	inode->i_write_hint, wbc);
1832	nr_underway++;
1833	}
1834	bh = next;
1835	} while (bh != head);
1836	unlock_page(page);
1837
1838	err = 0;
1839	done:
1840	if (nr_underway == 0) {
1841	/*
1842	* The page was marked dirty, but the buffers were
1843	* clean. Someone wrote them back by hand with
1844	* ll_rw_block/submit_bh. A rare case.
1845	*/
1846	end_page_writeback(page);
1847
1848	/*
1849	* The page and buffer_heads can be released at any time from
1850	* here on.
1851	*/
1852	}
1853	return err;
1854
1855	recover:
1856	/*
1857	* ENOSPC, or some other error. We may already have added some
1858	* blocks to the file, so we need to write these out to avoid
1859	* exposing stale data.
1860	* The page is currently locked and not marked for writeback
1861	*/
1862	bh = head;
1863	/* Recovery: lock and submit the mapped buffers */
1864	do {
1865	if (buffer_mapped(bh) && buffer_dirty(bh) &&
1866	!buffer_delay(bh)) {
1867	lock_buffer(bh);
1868	mark_buffer_async_write_endio(bh, handler);
1869	} else {
1870	/*
1871	* The buffer may have been set dirty during
1872	* attachment to a dirty page.
1873	*/
1874	clear_buffer_dirty(bh);
1875	}
1876	} while ((bh = bh->b_this_page) != head);
1877	SetPageError(page);
1878	BUG_ON(PageWriteback(page));
1879	mapping_set_error(page->mapping, err);
1880	set_page_writeback(page);
1881	do {
1882	struct buffer_head *next = bh->b_this_page;
1883	if (buffer_async_write(bh)) {
1884	clear_buffer_dirty(bh);
1885	submit_bh_wbc(REQ_OP_WRITE, write_flags, bh,
1886	inode->i_write_hint, wbc);
1887	nr_underway++;
1888	}
1889	bh = next;
1890	} while (bh != head);
1891	unlock_page(page);
1892	goto done;
1893	}
1894	EXPORT_SYMBOL(__block_write_full_page);
1895
1896	/*
1897	* If a page has any new buffers, zero them out here, and mark them uptodate
1898	* and dirty so they'll be written out (in order to prevent uninitialised
1899	* block data from leaking). And clear the new bit.
1900	*/
1901	void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
1902	{
1903	unsigned int block_start, block_end;
1904	struct buffer_head head, bh;
1905
1906	BUG_ON(!PageLocked(page));
1907	if (!page_has_buffers(page))
1908	return;
1909
1910	bh = head = page_buffers(page);
1911	block_start = 0;
1912	do {
1913	block_end = block_start + bh->b_size;
1914
1915	if (buffer_new(bh)) {
1916	if (block_end > from && block_start < to) {
1917	if (!PageUptodate(page)) {
1918	unsigned start, size;
1919
1920	start = max(from, block_start);
1921	size = min(to, block_end) - start;
1922
1923	zero_user(page, start, size);
1924	set_buffer_uptodate(bh);
1925	}
1926
1927	clear_buffer_new(bh);
1928	mark_buffer_dirty(bh);
1929	}
1930	}
1931
1932	block_start = block_end;
1933	bh = bh->b_this_page;
1934	} while (bh != head);
1935	}
1936	EXPORT_SYMBOL(page_zero_new_buffers);
1937
1938	static void
1939	iomap_to_bh(struct inode inode, sector_t block, struct buffer_head bh,
1940	struct iomap *iomap)
1941	{
1942	loff_t offset = block << inode->i_blkbits;
1943
1944	bh->b_bdev = iomap->bdev;
1945
1946	/*
1947	* Block points to offset in file we need to map, iomap contains
1948	* the offset at which the map starts. If the map ends before the
1949	* current block, then do not map the buffer and let the caller
1950	* handle it.
1951	*/
1952	BUG_ON(offset >= iomap->offset + iomap->length);
1953
1954	switch (iomap->type) {
1955	case IOMAP_HOLE:
1956	/*
1957	* If the buffer is not up to date or beyond the current EOF,
1958	* we need to mark it as new to ensure sub-block zeroing is
1959	* executed if necessary.
1960	*/
1961	if (!buffer_uptodate(bh) \|\|
1962	(offset >= i_size_read(inode)))
1963	set_buffer_new(bh);
1964	break;
1965	case IOMAP_DELALLOC:
1966	if (!buffer_uptodate(bh) \|\|
1967	(offset >= i_size_read(inode)))
1968	set_buffer_new(bh);
1969	set_buffer_uptodate(bh);
1970	set_buffer_mapped(bh);
1971	set_buffer_delay(bh);
1972	break;
1973	case IOMAP_UNWRITTEN:
1974	/*
1975	* For unwritten regions, we always need to ensure that regions
1976	* in the block we are not writing to are zeroed. Mark the
1977	* buffer as new to ensure this.
1978	*/
1979	set_buffer_new(bh);
1980	set_buffer_unwritten(bh);
1981	fallthrough;
1982	case IOMAP_MAPPED:
1983	if ((iomap->flags & IOMAP_F_NEW) \|\|
1984	offset >= i_size_read(inode))
1985	set_buffer_new(bh);
1986	bh->b_blocknr = (iomap->addr + offset - iomap->offset) >>
1987	inode->i_blkbits;
1988	set_buffer_mapped(bh);
1989	break;
1990	}
1991	}
1992
1993	int __block_write_begin_int(struct page *page, loff_t pos, unsigned len,
1994	get_block_t get_block, struct iomap iomap)
1995	{
1996	unsigned from = pos & (PAGE_SIZE - 1);
1997	unsigned to = from + len;
1998	struct inode *inode = page->mapping->host;
1999	unsigned block_start, block_end;
2000	sector_t block;
2001	int err = 0;
2002	unsigned blocksize, bbits;
2003	struct buffer_head bh, head, wait[2], *wait_bh=wait;
2004
2005	BUG_ON(!PageLocked(page));
2006	BUG_ON(from > PAGE_SIZE);
2007	BUG_ON(to > PAGE_SIZE);
2008	BUG_ON(from > to);
2009
2010	head = create_page_buffers(page, inode, 0);
2011	blocksize = head->b_size;
2012	bbits = block_size_bits(blocksize);
2013
2014	block = (sector_t)page->index << (PAGE_SHIFT - bbits);
2015
2016	for(bh = head, block_start = 0; bh != head \|\| !block_start;
2017	block++, block_start=block_end, bh = bh->b_this_page) {
2018	block_end = block_start + blocksize;
2019	if (block_end <= from \|\| block_start >= to) {
2020	if (PageUptodate(page)) {
2021	if (!buffer_uptodate(bh))
2022	set_buffer_uptodate(bh);
2023	}
2024	continue;
2025	}
2026	if (buffer_new(bh))
2027	clear_buffer_new(bh);
2028	if (!buffer_mapped(bh)) {
2029	WARN_ON(bh->b_size != blocksize);
2030	if (get_block) {
2031	err = get_block(inode, block, bh, 1);
2032	if (err)
2033	break;
2034	} else {
2035	iomap_to_bh(inode, block, bh, iomap);
2036	}
2037
2038	if (buffer_new(bh)) {
2039	clean_bdev_bh_alias(bh);
2040	if (PageUptodate(page)) {
2041	clear_buffer_new(bh);
2042	set_buffer_uptodate(bh);
2043	mark_buffer_dirty(bh);
2044	continue;
2045	}
2046	if (block_end > to \|\| block_start < from)
2047	zero_user_segments(page,
2048	to, block_end,
2049	block_start, from);
2050	continue;
2051	}
2052	}
2053	if (PageUptodate(page)) {
2054	if (!buffer_uptodate(bh))
2055	set_buffer_uptodate(bh);
2056	continue;
2057	}
2058	if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
2059	!buffer_unwritten(bh) &&
2060	(block_start < from \|\| block_end > to)) {
2061	ll_rw_block(REQ_OP_READ, 0, 1, &bh);
2062	*wait_bh++=bh;
2063	}
2064	}
2065	/*
2066	* If we issued read requests - let them complete.
2067	*/
2068	while(wait_bh > wait) {
2069	wait_on_buffer(*--wait_bh);
2070	if (!buffer_uptodate(*wait_bh))
2071	err = -EIO;
2072	}
2073	if (unlikely(err))
2074	page_zero_new_buffers(page, from, to);
2075	return err;
2076	}
2077
2078	int __block_write_begin(struct page *page, loff_t pos, unsigned len,
2079	get_block_t *get_block)
2080	{
2081	return __block_write_begin_int(page, pos, len, get_block, NULL);
2082	}
2083	EXPORT_SYMBOL(__block_write_begin);
2084
2085	static int __block_commit_write(struct inode inode, struct page page,
2086	unsigned from, unsigned to)
2087	{
2088	unsigned block_start, block_end;
2089	int partial = 0;
2090	unsigned blocksize;
2091	struct buffer_head bh, head;
2092
2093	bh = head = page_buffers(page);
2094	blocksize = bh->b_size;
2095
2096	block_start = 0;
2097	do {
2098	block_end = block_start + blocksize;
2099	if (block_end <= from \|\| block_start >= to) {
2100	if (!buffer_uptodate(bh))
2101	partial = 1;
2102	} else {
2103	set_buffer_uptodate(bh);
2104	mark_buffer_dirty(bh);
2105	}
2106	if (buffer_new(bh))
2107	clear_buffer_new(bh);
2108
2109	block_start = block_end;
2110	bh = bh->b_this_page;
2111	} while (bh != head);
2112
2113	/*
2114	* If this is a partial write which happened to make all buffers
2115	* uptodate then we can optimize away a bogus readpage() for
2116	* the next read(). Here we 'discover' whether the page went
2117	* uptodate as a result of this (potentially partial) write.
2118	*/
2119	if (!partial)
2120	SetPageUptodate(page);
2121	return 0;
2122	}
2123
2124	/*
2125	* block_write_begin takes care of the basic task of block allocation and
2126	* bringing partial write blocks uptodate first.
2127	*
2128	* The filesystem needs to handle block truncation upon failure.
2129	*/
2130	int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
2131	unsigned flags, struct page *pagep, get_block_t get_block)
2132	{
2133	pgoff_t index = pos >> PAGE_SHIFT;
2134	struct page *page;
2135	int status;
2136
2137	page = grab_cache_page_write_begin(mapping, index, flags);
2138	if (!page)
2139	return -ENOMEM;
2140
2141	status = __block_write_begin(page, pos, len, get_block);
2142	if (unlikely(status)) {
2143	unlock_page(page);
2144	put_page(page);
2145	page = NULL;
2146	}
2147
2148	*pagep = page;
2149	return status;
2150	}
2151	EXPORT_SYMBOL(block_write_begin);
2152
2153	int block_write_end(struct file file, struct address_space mapping,
2154	loff_t pos, unsigned len, unsigned copied,
2155	struct page page, void fsdata)
2156	{
2157	struct inode *inode = mapping->host;
2158	unsigned start;
2159
2160	start = pos & (PAGE_SIZE - 1);
2161
2162	if (unlikely(copied < len)) {
2163	/*
2164	* The buffers that were written will now be uptodate, so we
2165	* don't have to worry about a readpage reading them and
2166	* overwriting a partial write. However if we have encountered
2167	* a short write and only partially written into a buffer, it
2168	* will not be marked uptodate, so a readpage might come in and
2169	* destroy our partial write.
2170	*
2171	* Do the simplest thing, and just treat any short write to a
2172	* non uptodate page as a zero-length write, and force the
2173	* caller to redo the whole thing.
2174	*/
2175	if (!PageUptodate(page))
2176	copied = 0;
2177
2178	page_zero_new_buffers(page, start+copied, start+len);
2179	}
2180	flush_dcache_page(page);
2181
2182	/* This could be a short (even 0-length) commit */
2183	__block_commit_write(inode, page, start, start+copied);
2184
2185	return copied;
2186	}
2187	EXPORT_SYMBOL(block_write_end);
2188
2189	int generic_write_end(struct file file, struct address_space mapping,
2190	loff_t pos, unsigned len, unsigned copied,
2191	struct page page, void fsdata)
2192	{
2193	struct inode *inode = mapping->host;
2194	loff_t old_size = inode->i_size;
2195	bool i_size_changed = false;
2196
2197	copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
2198
2199	/*
2200	* No need to use i_size_read() here, the i_size cannot change under us
2201	* because we hold i_rwsem.
2202	*
2203	* But it's important to update i_size while still holding page lock:
2204	* page writeout could otherwise come in and zero beyond i_size.
2205	*/
2206	if (pos + copied > inode->i_size) {
2207	i_size_write(inode, pos + copied);
2208	i_size_changed = true;
2209	}
2210
2211	unlock_page(page);
2212	put_page(page);
2213
2214	if (old_size < pos)
2215	pagecache_isize_extended(inode, old_size, pos);
2216	/*
2217	* Don't mark the inode dirty under page lock. First, it unnecessarily
2218	* makes the holding time of page lock longer. Second, it forces lock
2219	* ordering of page lock and transaction start for journaling
2220	* filesystems.
2221	*/
2222	if (i_size_changed)
2223	mark_inode_dirty(inode);
2224	return copied;
2225	}
2226	EXPORT_SYMBOL(generic_write_end);
2227
2228	/*
2229	* block_is_partially_uptodate checks whether buffers within a page are
2230	* uptodate or not.
2231	*
2232	* Returns true if all buffers which correspond to a file portion
2233	* we want to read are uptodate.
2234	*/
2235	int block_is_partially_uptodate(struct page *page, unsigned long from,
2236	unsigned long count)
2237	{
2238	unsigned block_start, block_end, blocksize;
2239	unsigned to;
2240	struct buffer_head bh, head;
2241	int ret = 1;
2242
2243	if (!page_has_buffers(page))
2244	return 0;
2245
2246	head = page_buffers(page);
2247	blocksize = head->b_size;
2248	to = min_t(unsigned, PAGE_SIZE - from, count);
2249	to = from + to;
2250	if (from < blocksize && to > PAGE_SIZE - blocksize)
2251	return 0;
2252
2253	bh = head;
2254	block_start = 0;
2255	do {
2256	block_end = block_start + blocksize;
2257	if (block_end > from && block_start < to) {
2258	if (!buffer_uptodate(bh)) {
2259	ret = 0;
2260	break;
2261	}
2262	if (block_end >= to)
2263	break;
2264	}
2265	block_start = block_end;
2266	bh = bh->b_this_page;
2267	} while (bh != head);
2268
2269	return ret;
2270	}
2271	EXPORT_SYMBOL(block_is_partially_uptodate);
2272
2273	/*
2274	* Generic "read page" function for block devices that have the normal
2275	* get_block functionality. This is most of the block device filesystems.
2276	* Reads the page asynchronously --- the unlock_buffer() and
2277	* set/clear_buffer_uptodate() functions propagate buffer state into the
2278	* page struct once IO has completed.
2279	*/
2280	int block_read_full_page(struct page page, get_block_t get_block)
2281	{
2282	struct inode *inode = page->mapping->host;
2283	sector_t iblock, lblock;
2284	struct buffer_head bh, head, *arr[MAX_BUF_PER_PAGE];
2285	unsigned int blocksize, bbits;
2286	int nr, i;
2287	int fully_mapped = 1;
2288
2289	head = create_page_buffers(page, inode, 0);
2290	blocksize = head->b_size;
2291	bbits = block_size_bits(blocksize);
2292
2293	iblock = (sector_t)page->index << (PAGE_SHIFT - bbits);
2294	lblock = (i_size_read(inode)+blocksize-1) >> bbits;
2295	bh = head;
2296	nr = 0;
2297	i = 0;
2298
2299	do {
2300	if (buffer_uptodate(bh))
2301	continue;
2302
2303	if (!buffer_mapped(bh)) {
2304	int err = 0;
2305
2306	fully_mapped = 0;
2307	if (iblock < lblock) {
2308	WARN_ON(bh->b_size != blocksize);
2309	err = get_block(inode, iblock, bh, 0);
2310	if (err)
2311	SetPageError(page);
2312	}
2313	if (!buffer_mapped(bh)) {
2314	zero_user(page, i * blocksize, blocksize);
2315	if (!err)
2316	set_buffer_uptodate(bh);
2317	continue;
2318	}
2319	/*
2320	* get_block() might have updated the buffer
2321	* synchronously
2322	*/
2323	if (buffer_uptodate(bh))
2324	continue;
2325	}
2326	arr[nr++] = bh;
2327	} while (i++, iblock++, (bh = bh->b_this_page) != head);
2328
2329	if (fully_mapped)
2330	SetPageMappedToDisk(page);
2331
2332	if (!nr) {
2333	/*
2334	* All buffers are uptodate - we can set the page uptodate
2335	* as well. But not if get_block() returned an error.
2336	*/
2337	if (!PageError(page))
2338	SetPageUptodate(page);
2339	unlock_page(page);
2340	return 0;
2341	}
2342
2343	/* Stage two: lock the buffers */
2344	for (i = 0; i < nr; i++) {
2345	bh = arr[i];
2346	lock_buffer(bh);
2347	mark_buffer_async_read(bh);
2348	}
2349
2350	/*
2351	* Stage 3: start the IO. Check for uptodateness
2352	* inside the buffer lock in case another process reading
2353	* the underlying blockdev brought it uptodate (the sct fix).
2354	*/
2355	for (i = 0; i < nr; i++) {
2356	bh = arr[i];
2357	if (buffer_uptodate(bh))
2358	end_buffer_async_read(bh, 1);
2359	else
2360	submit_bh(REQ_OP_READ, 0, bh);
2361	}
2362	return 0;
2363	}
2364	EXPORT_SYMBOL(block_read_full_page);
2365
2366	/* utility function for filesystems that need to do work on expanding
2367	* truncates. Uses filesystem pagecache writes to allow the filesystem to
2368	* deal with the hole.
2369	*/
2370	int generic_cont_expand_simple(struct inode *inode, loff_t size)
2371	{
2372	struct address_space *mapping = inode->i_mapping;
2373	struct page *page;
2374	void *fsdata;
2375	int err;
2376
2377	err = inode_newsize_ok(inode, size);
2378	if (err)
2379	goto out;
2380
2381	err = pagecache_write_begin(NULL, mapping, size, 0,
2382	AOP_FLAG_CONT_EXPAND, &page, &fsdata);
2383	if (err)
2384	goto out;
2385
2386	err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata);
2387	BUG_ON(err > 0);
2388
2389	out:
2390	return err;
2391	}
2392	EXPORT_SYMBOL(generic_cont_expand_simple);
2393
2394	static int cont_expand_zero(struct file file, struct address_space mapping,
2395	loff_t pos, loff_t *bytes)
2396	{
2397	struct inode *inode = mapping->host;
2398	unsigned int blocksize = i_blocksize(inode);
2399	struct page *page;
2400	void *fsdata;
2401	pgoff_t index, curidx;
2402	loff_t curpos;
2403	unsigned zerofrom, offset, len;
2404	int err = 0;
2405
2406	index = pos >> PAGE_SHIFT;
2407	offset = pos & ~PAGE_MASK;
2408
2409	while (index > (curidx = (curpos = *bytes)>>PAGE_SHIFT)) {
2410	zerofrom = curpos & ~PAGE_MASK;
2411	if (zerofrom & (blocksize-1)) {
2412	*bytes \|= (blocksize-1);
2413	(*bytes)++;
2414	}
2415	len = PAGE_SIZE - zerofrom;
2416
2417	err = pagecache_write_begin(file, mapping, curpos, len, 0,
2418	&page, &fsdata);
2419	if (err)
2420	goto out;
2421	zero_user(page, zerofrom, len);
2422	err = pagecache_write_end(file, mapping, curpos, len, len,
2423	page, fsdata);
2424	if (err < 0)
2425	goto out;
2426	BUG_ON(err != len);
2427	err = 0;
2428
2429	balance_dirty_pages_ratelimited(mapping);
2430
2431	if (fatal_signal_pending(current)) {
2432	err = -EINTR;
2433	goto out;
2434	}
2435	}
2436
2437	/* page covers the boundary, find the boundary offset */
2438	if (index == curidx) {
2439	zerofrom = curpos & ~PAGE_MASK;
2440	/* if we will expand the thing last block will be filled */
2441	if (offset <= zerofrom) {
2442	goto out;
2443	}
2444	if (zerofrom & (blocksize-1)) {
2445	*bytes \|= (blocksize-1);
2446	(*bytes)++;
2447	}
2448	len = offset - zerofrom;
2449
2450	err = pagecache_write_begin(file, mapping, curpos, len, 0,
2451	&page, &fsdata);
2452	if (err)
2453	goto out;
2454	zero_user(page, zerofrom, len);
2455	err = pagecache_write_end(file, mapping, curpos, len, len,
2456	page, fsdata);
2457	if (err < 0)
2458	goto out;
2459	BUG_ON(err != len);
2460	err = 0;
2461	}
2462	out:
2463	return err;
2464	}
2465
2466	/*
2467	* For moronic filesystems that do not allow holes in file.
2468	* We may have to extend the file.
2469	*/
2470	int cont_write_begin(struct file file, struct address_space mapping,
2471	loff_t pos, unsigned len, unsigned flags,
2472	struct page pagep, void fsdata,
2473	get_block_t get_block, loff_t bytes)
2474	{
2475	struct inode *inode = mapping->host;
2476	unsigned int blocksize = i_blocksize(inode);
2477	unsigned int zerofrom;
2478	int err;
2479
2480	err = cont_expand_zero(file, mapping, pos, bytes);
2481	if (err)
2482	return err;
2483
2484	zerofrom = *bytes & ~PAGE_MASK;
2485	if (pos+len > *bytes && zerofrom & (blocksize-1)) {
2486	*bytes \|= (blocksize-1);
2487	(*bytes)++;
2488	}
2489
2490	return block_write_begin(mapping, pos, len, flags, pagep, get_block);
2491	}
2492	EXPORT_SYMBOL(cont_write_begin);
2493
2494	int block_commit_write(struct page *page, unsigned from, unsigned to)
2495	{
2496	struct inode *inode = page->mapping->host;
2497	__block_commit_write(inode,page,from,to);
2498	return 0;
2499	}
2500	EXPORT_SYMBOL(block_commit_write);
2501
2502	/*
2503	* block_page_mkwrite() is not allowed to change the file size as it gets
2504	* called from a page fault handler when a page is first dirtied. Hence we must
2505	* be careful to check for EOF conditions here. We set the page up correctly
2506	* for a written page which means we get ENOSPC checking when writing into
2507	* holes and correct delalloc and unwritten extent mapping on filesystems that
2508	* support these features.
2509	*
2510	* We are not allowed to take the i_mutex here so we have to play games to
2511	* protect against truncate races as the page could now be beyond EOF. Because
2512	* truncate writes the inode size before removing pages, once we have the
2513	* page lock we can determine safely if the page is beyond EOF. If it is not
2514	* beyond EOF, then the page is guaranteed safe against truncation until we
2515	* unlock the page.
2516	*
2517	* Direct callers of this function should protect against filesystem freezing
2518	* using sb_start_pagefault() - sb_end_pagefault() functions.
2519	*/
2520	int block_page_mkwrite(struct vm_area_struct vma, struct vm_fault vmf,
2521	get_block_t get_block)
2522	{
2523	struct page *page = vmf->page;
2524	struct inode *inode = file_inode(vma->vm_file);
2525	unsigned long end;
2526	loff_t size;
2527	int ret;
2528
2529	lock_page(page);
2530	size = i_size_read(inode);
2531	if ((page->mapping != inode->i_mapping) \|\|
2532	(page_offset(page) > size)) {
2533	/* We overload EFAULT to mean page got truncated */
2534	ret = -EFAULT;
2535	goto out_unlock;
2536	}
2537
2538	/* page is wholly or partially inside EOF */
2539	if (((page->index + 1) << PAGE_SHIFT) > size)
2540	end = size & ~PAGE_MASK;
2541	else
2542	end = PAGE_SIZE;
2543
2544	ret = __block_write_begin(page, 0, end, get_block);
2545	if (!ret)
2546	ret = block_commit_write(page, 0, end);
2547
2548	if (unlikely(ret < 0))
2549	goto out_unlock;
2550	set_page_dirty(page);
2551	wait_for_stable_page(page);
2552	return 0;
2553	out_unlock:
2554	unlock_page(page);
2555	return ret;
2556	}
2557	EXPORT_SYMBOL(block_page_mkwrite);
2558
2559	/*
2560	* nobh_write_begin()'s prereads are special: the buffer_heads are freed
2561	* immediately, while under the page lock. So it needs a special end_io
2562	* handler which does not touch the bh after unlocking it.
2563	*/
2564	static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2565	{
2566	__end_buffer_read_notouch(bh, uptodate);
2567	}
2568
2569	/*
2570	* Attach the singly-linked list of buffers created by nobh_write_begin, to
2571	* the page (converting it to circular linked list and taking care of page
2572	* dirty races).
2573	*/
2574	static void attach_nobh_buffers(struct page page, struct buffer_head head)
2575	{
2576	struct buffer_head *bh;
2577
2578	BUG_ON(!PageLocked(page));
2579
2580	spin_lock(&page->mapping->private_lock);
2581	bh = head;
2582	do {
2583	if (PageDirty(page))
2584	set_buffer_dirty(bh);
2585	if (!bh->b_this_page)
2586	bh->b_this_page = head;
2587	bh = bh->b_this_page;
2588	} while (bh != head);
2589	attach_page_private(page, head);
2590	spin_unlock(&page->mapping->private_lock);
2591	}
2592
2593	/*
2594	* On entry, the page is fully not uptodate.
2595	* On exit the page is fully uptodate in the areas outside (from,to)
2596	* The filesystem needs to handle block truncation upon failure.
2597	*/
2598	int nobh_write_begin(struct address_space *mapping,
2599	loff_t pos, unsigned len, unsigned flags,
2600	struct page pagep, void fsdata,
2601	get_block_t *get_block)
2602	{
2603	struct inode *inode = mapping->host;
2604	const unsigned blkbits = inode->i_blkbits;
2605	const unsigned blocksize = 1 << blkbits;
2606	struct buffer_head head, bh;
2607	struct page *page;
2608	pgoff_t index;
2609	unsigned from, to;
2610	unsigned block_in_page;
2611	unsigned block_start, block_end;
2612	sector_t block_in_file;
2613	int nr_reads = 0;
2614	int ret = 0;
2615	int is_mapped_to_disk = 1;
2616
2617	index = pos >> PAGE_SHIFT;
2618	from = pos & (PAGE_SIZE - 1);
2619	to = from + len;
2620
2621	page = grab_cache_page_write_begin(mapping, index, flags);
2622	if (!page)
2623	return -ENOMEM;
2624	*pagep = page;
2625	*fsdata = NULL;
2626
2627	if (page_has_buffers(page)) {
2628	ret = __block_write_begin(page, pos, len, get_block);
2629	if (unlikely(ret))
2630	goto out_release;
2631	return ret;
2632	}
2633
2634	if (PageMappedToDisk(page))
2635	return 0;
2636
2637	/*
2638	* Allocate buffers so that we can keep track of state, and potentially
2639	* attach them to the page if an error occurs. In the common case of
2640	* no error, they will just be freed again without ever being attached
2641	* to the page (which is all OK, because we're under the page lock).
2642	*
2643	* Be careful: the buffer linked list is a NULL terminated one, rather
2644	* than the circular one we're used to.
2645	*/
2646	head = alloc_page_buffers(page, blocksize, false);
2647	if (!head) {
2648	ret = -ENOMEM;
2649	goto out_release;
2650	}
2651
2652	block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
2653
2654	/*
2655	* We loop across all blocks in the page, whether or not they are
2656	* part of the affected region. This is so we can discover if the
2657	* page is fully mapped-to-disk.
2658	*/
2659	for (block_start = 0, block_in_page = 0, bh = head;
2660	block_start < PAGE_SIZE;
2661	block_in_page++, block_start += blocksize, bh = bh->b_this_page) {
2662	int create;
2663
2664	block_end = block_start + blocksize;
2665	bh->b_state = 0;
2666	create = 1;
2667	if (block_start >= to)
2668	create = 0;
2669	ret = get_block(inode, block_in_file + block_in_page,
2670	bh, create);
2671	if (ret)
2672	goto failed;
2673	if (!buffer_mapped(bh))
2674	is_mapped_to_disk = 0;
2675	if (buffer_new(bh))
2676	clean_bdev_bh_alias(bh);
2677	if (PageUptodate(page)) {
2678	set_buffer_uptodate(bh);
2679	continue;
2680	}
2681	if (buffer_new(bh) \|\| !buffer_mapped(bh)) {
2682	zero_user_segments(page, block_start, from,
2683	to, block_end);
2684	continue;
2685	}
2686	if (buffer_uptodate(bh))
2687	continue; /* reiserfs does this */
2688	if (block_start < from \|\| block_end > to) {
2689	lock_buffer(bh);
2690	bh->b_end_io = end_buffer_read_nobh;
2691	submit_bh(REQ_OP_READ, 0, bh);
2692	nr_reads++;
2693	}
2694	}
2695
2696	if (nr_reads) {
2697	/*
2698	* The page is locked, so these buffers are protected from
2699	* any VM or truncate activity. Hence we don't need to care
2700	* for the buffer_head refcounts.
2701	*/
2702	for (bh = head; bh; bh = bh->b_this_page) {
2703	wait_on_buffer(bh);
2704	if (!buffer_uptodate(bh))
2705	ret = -EIO;
2706	}
2707	if (ret)
2708	goto failed;
2709	}
2710
2711	if (is_mapped_to_disk)
2712	SetPageMappedToDisk(page);
2713
2714	fsdata = head; / to be released by nobh_write_end */
2715
2716	return 0;
2717
2718	failed:
2719	BUG_ON(!ret);
2720	/*
2721	* Error recovery is a bit difficult. We need to zero out blocks that
2722	* were newly allocated, and dirty them to ensure they get written out.
2723	* Buffers need to be attached to the page at this point, otherwise
2724	* the handling of potential IO errors during writeout would be hard
2725	* (could try doing synchronous writeout, but what if that fails too?)
2726	*/
2727	attach_nobh_buffers(page, head);
2728	page_zero_new_buffers(page, from, to);
2729
2730	out_release:
2731	unlock_page(page);
2732	put_page(page);
2733	*pagep = NULL;
2734
2735	return ret;
2736	}
2737	EXPORT_SYMBOL(nobh_write_begin);
2738
2739	int nobh_write_end(struct file file, struct address_space mapping,
2740	loff_t pos, unsigned len, unsigned copied,
2741	struct page page, void fsdata)
2742	{
2743	struct inode *inode = page->mapping->host;
2744	struct buffer_head *head = fsdata;
2745	struct buffer_head *bh;
2746	BUG_ON(fsdata != NULL && page_has_buffers(page));
2747
2748	if (unlikely(copied < len) && head)
2749	attach_nobh_buffers(page, head);
2750	if (page_has_buffers(page))
2751	return generic_write_end(file, mapping, pos, len,
2752	copied, page, fsdata);
2753
2754	SetPageUptodate(page);
2755	set_page_dirty(page);
2756	if (pos+copied > inode->i_size) {
2757	i_size_write(inode, pos+copied);
2758	mark_inode_dirty(inode);
2759	}
2760
2761	unlock_page(page);
2762	put_page(page);
2763
2764	while (head) {
2765	bh = head;
2766	head = head->b_this_page;
2767	free_buffer_head(bh);
2768	}
2769
2770	return copied;
2771	}
2772	EXPORT_SYMBOL(nobh_write_end);
2773
2774	/*
2775	* nobh_writepage() - based on block_full_write_page() except
2776	* that it tries to operate without attaching bufferheads to
2777	* the page.
2778	*/
2779	int nobh_writepage(struct page page, get_block_t get_block,
2780	struct writeback_control *wbc)
2781	{
2782	struct inode * const inode = page->mapping->host;
2783	loff_t i_size = i_size_read(inode);
2784	const pgoff_t end_index = i_size >> PAGE_SHIFT;
2785	unsigned offset;
2786	int ret;
2787
2788	/* Is the page fully inside i_size? */
2789	if (page->index < end_index)
2790	goto out;
2791
2792	/* Is the page fully outside i_size? (truncate in progress) */
2793	offset = i_size & (PAGE_SIZE-1);
2794	if (page->index >= end_index+1 \|\| !offset) {
2795	unlock_page(page);
2796	return 0; /* don't care */
2797	}
2798
2799	/*
2800	* The page straddles i_size. It must be zeroed out on each and every
2801	* writepage invocation because it may be mmapped. "A file is mapped
2802	* in multiples of the page size. For a file that is not a multiple of
2803	* the page size, the remaining memory is zeroed when mapped, and
2804	* writes to that region are not written out to the file."
2805	*/
2806	zero_user_segment(page, offset, PAGE_SIZE);
2807	out:
2808	ret = mpage_writepage(page, get_block, wbc);
2809	if (ret == -EAGAIN)
2810	ret = __block_write_full_page(inode, page, get_block, wbc,
2811	end_buffer_async_write);
2812	return ret;
2813	}
2814	EXPORT_SYMBOL(nobh_writepage);
2815
2816	int nobh_truncate_page(struct address_space *mapping,
2817	loff_t from, get_block_t *get_block)
2818	{
2819	pgoff_t index = from >> PAGE_SHIFT;
2820	unsigned offset = from & (PAGE_SIZE-1);
2821	unsigned blocksize;
2822	sector_t iblock;
2823	unsigned length, pos;
2824	struct inode *inode = mapping->host;
2825	struct page *page;
2826	struct buffer_head map_bh;
2827	int err;
2828
2829	blocksize = i_blocksize(inode);
2830	length = offset & (blocksize - 1);
2831
2832	/* Block boundary? Nothing to do */
2833	if (!length)
2834	return 0;
2835
2836	length = blocksize - length;
2837	iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
2838
2839	page = grab_cache_page(mapping, index);
2840	err = -ENOMEM;
2841	if (!page)
2842	goto out;
2843
2844	if (page_has_buffers(page)) {
2845	has_buffers:
2846	unlock_page(page);
2847	put_page(page);
2848	return block_truncate_page(mapping, from, get_block);
2849	}
2850
2851	/* Find the buffer that contains "offset" */
2852	pos = blocksize;
2853	while (offset >= pos) {
2854	iblock++;
2855	pos += blocksize;
2856	}
2857
2858	map_bh.b_size = blocksize;
2859	map_bh.b_state = 0;
2860	err = get_block(inode, iblock, &map_bh, 0);
2861	if (err)
2862	goto unlock;
2863	/* unmapped? It's a hole - nothing to do */
2864	if (!buffer_mapped(&map_bh))
2865	goto unlock;
2866
2867	/* Ok, it's mapped. Make sure it's up-to-date */
2868	if (!PageUptodate(page)) {
2869	err = mapping->a_ops->readpage(NULL, page);
2870	if (err) {
2871	put_page(page);
2872	goto out;
2873	}
2874	lock_page(page);
2875	if (!PageUptodate(page)) {
2876	err = -EIO;
2877	goto unlock;
2878	}
2879	if (page_has_buffers(page))
2880	goto has_buffers;
2881	}
2882	zero_user(page, offset, length);
2883	set_page_dirty(page);
2884	err = 0;
2885
2886	unlock:
2887	unlock_page(page);
2888	put_page(page);
2889	out:
2890	return err;
2891	}
2892	EXPORT_SYMBOL(nobh_truncate_page);
2893
2894	int block_truncate_page(struct address_space *mapping,
2895	loff_t from, get_block_t *get_block)
2896	{
2897	pgoff_t index = from >> PAGE_SHIFT;
2898	unsigned offset = from & (PAGE_SIZE-1);
2899	unsigned blocksize;
2900	sector_t iblock;
2901	unsigned length, pos;
2902	struct inode *inode = mapping->host;
2903	struct page *page;
2904	struct buffer_head *bh;
2905	int err;
2906
2907	blocksize = i_blocksize(inode);
2908	length = offset & (blocksize - 1);
2909
2910	/* Block boundary? Nothing to do */
2911	if (!length)
2912	return 0;
2913
2914	length = blocksize - length;
2915	iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
2916
2917	page = grab_cache_page(mapping, index);
2918	err = -ENOMEM;
2919	if (!page)
2920	goto out;
2921
2922	if (!page_has_buffers(page))
2923	create_empty_buffers(page, blocksize, 0);
2924
2925	/* Find the buffer that contains "offset" */
2926	bh = page_buffers(page);
2927	pos = blocksize;
2928	while (offset >= pos) {
2929	bh = bh->b_this_page;
2930	iblock++;
2931	pos += blocksize;
2932	}
2933
2934	err = 0;
2935	if (!buffer_mapped(bh)) {
2936	WARN_ON(bh->b_size != blocksize);
2937	err = get_block(inode, iblock, bh, 0);
2938	if (err)
2939	goto unlock;
2940	/* unmapped? It's a hole - nothing to do */
2941	if (!buffer_mapped(bh))
2942	goto unlock;
2943	}
2944
2945	/* Ok, it's mapped. Make sure it's up-to-date */
2946	if (PageUptodate(page))
2947	set_buffer_uptodate(bh);
2948
2949	if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
2950	err = -EIO;
2951	ll_rw_block(REQ_OP_READ, 0, 1, &bh);
2952	wait_on_buffer(bh);
2953	/* Uhhuh. Read error. Complain and punt. */
2954	if (!buffer_uptodate(bh))
2955	goto unlock;
2956	}
2957
2958	zero_user(page, offset, length);
2959	mark_buffer_dirty(bh);
2960	err = 0;
2961
2962	unlock:
2963	unlock_page(page);
2964	put_page(page);
2965	out:
2966	return err;
2967	}
2968	EXPORT_SYMBOL(block_truncate_page);
2969
2970	/*
2971	* The generic ->writepage function for buffer-backed address_spaces
2972	*/
2973	int block_write_full_page(struct page page, get_block_t get_block,
2974	struct writeback_control *wbc)
2975	{
2976	struct inode * const inode = page->mapping->host;
2977	loff_t i_size = i_size_read(inode);
2978	const pgoff_t end_index = i_size >> PAGE_SHIFT;
2979	unsigned offset;
2980
2981	/* Is the page fully inside i_size? */
2982	if (page->index < end_index)
2983	return __block_write_full_page(inode, page, get_block, wbc,
2984	end_buffer_async_write);
2985
2986	/* Is the page fully outside i_size? (truncate in progress) */
2987	offset = i_size & (PAGE_SIZE-1);
2988	if (page->index >= end_index+1 \|\| !offset) {
2989	unlock_page(page);
2990	return 0; /* don't care */
2991	}
2992
2993	/*
2994	* The page straddles i_size. It must be zeroed out on each and every
2995	* writepage invocation because it may be mmapped. "A file is mapped
2996	* in multiples of the page size. For a file that is not a multiple of
2997	* the page size, the remaining memory is zeroed when mapped, and
2998	* writes to that region are not written out to the file."
2999	*/
3000	zero_user_segment(page, offset, PAGE_SIZE);
3001	return __block_write_full_page(inode, page, get_block, wbc,
3002	end_buffer_async_write);
3003	}
3004	EXPORT_SYMBOL(block_write_full_page);
3005
3006	sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
3007	get_block_t *get_block)
3008	{
3009	struct inode *inode = mapping->host;
3010	struct buffer_head tmp = {
3011	.b_size = i_blocksize(inode),
3012	};
3013
3014	get_block(inode, block, &tmp, 0);
3015	return tmp.b_blocknr;
3016	}
3017	EXPORT_SYMBOL(generic_block_bmap);
3018
3019	static void end_bio_bh_io_sync(struct bio *bio)
3020	{
3021	struct buffer_head *bh = bio->bi_private;
3022
3023	if (unlikely(bio_flagged(bio, BIO_QUIET)))
3024	set_bit(BH_Quiet, &bh->b_state);
3025
3026	bh->b_end_io(bh, !bio->bi_status);
3027	bio_put(bio);
3028	}
3029
3030	static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
3031	enum rw_hint write_hint, struct writeback_control *wbc)
3032	{
3033	struct bio *bio;
3034
3035	BUG_ON(!buffer_locked(bh));
3036	BUG_ON(!buffer_mapped(bh));
3037	BUG_ON(!bh->b_end_io);
3038	BUG_ON(buffer_delay(bh));
3039	BUG_ON(buffer_unwritten(bh));
3040
3041	/*
3042	* Only clear out a write error when rewriting
3043	*/
3044	if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE))
3045	clear_buffer_write_io_error(bh);
3046
3047	bio = bio_alloc(GFP_NOIO, 1);
3048
3049	fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO);
3050
3051	bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
3052	bio_set_dev(bio, bh->b_bdev);
3053	bio->bi_write_hint = write_hint;
3054
3055	bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
3056	BUG_ON(bio->bi_iter.bi_size != bh->b_size);
3057
3058	bio->bi_end_io = end_bio_bh_io_sync;
3059	bio->bi_private = bh;
3060
3061	if (buffer_meta(bh))
3062	op_flags \|= REQ_META;
3063	if (buffer_prio(bh))
3064	op_flags \|= REQ_PRIO;
3065	bio_set_op_attrs(bio, op, op_flags);
3066
3067	/* Take care of bh's that straddle the end of the device */
3068	guard_bio_eod(bio);
3069
3070	if (wbc) {
3071	wbc_init_bio(wbc, bio);
3072	wbc_account_cgroup_owner(wbc, bh->b_page, bh->b_size);
3073	}
3074
3075	submit_bio(bio);
3076	return 0;
3077	}
3078
3079	int submit_bh(int op, int op_flags, struct buffer_head *bh)
3080	{
3081	return submit_bh_wbc(op, op_flags, bh, 0, NULL);
3082	}
3083	EXPORT_SYMBOL(submit_bh);
3084
3085	/**
3086	* ll_rw_block: low-level access to block devices (DEPRECATED)
3087	* @op: whether to %READ or %WRITE
3088	* @op_flags: req_flag_bits
3089	* @nr: number of &struct buffer_heads in the array
3090	* @bhs: array of pointers to &struct buffer_head
3091	*
3092	* ll_rw_block() takes an array of pointers to &struct buffer_heads, and
3093	* requests an I/O operation on them, either a %REQ_OP_READ or a %REQ_OP_WRITE.
3094	* @op_flags contains flags modifying the detailed I/O behavior, most notably
3095	* %REQ_RAHEAD.
3096	*
3097	* This function drops any buffer that it cannot get a lock on (with the
3098	* BH_Lock state bit), any buffer that appears to be clean when doing a write
3099	* request, and any buffer that appears to be up-to-date when doing read
3100	* request. Further it marks as clean buffers that are processed for
3101	* writing (the buffer cache won't assume that they are actually clean
3102	* until the buffer gets unlocked).
3103	*
3104	* ll_rw_block sets b_end_io to simple completion handler that marks
3105	* the buffer up-to-date (if appropriate), unlocks the buffer and wakes
3106	* any waiters.
3107	*
3108	* All of the buffers must be for the same device, and must also be a
3109	* multiple of the current approved size for the device.
3110	*/
3111	void ll_rw_block(int op, int op_flags, int nr, struct buffer_head *bhs[])
3112	{
3113	int i;
3114
3115	for (i = 0; i < nr; i++) {
3116	struct buffer_head *bh = bhs[i];
3117
3118	if (!trylock_buffer(bh))
3119	continue;
3120	if (op == WRITE) {
3121	if (test_clear_buffer_dirty(bh)) {
3122	bh->b_end_io = end_buffer_write_sync;
3123	get_bh(bh);
3124	submit_bh(op, op_flags, bh);
3125	continue;
3126	}
3127	} else {
3128	if (!buffer_uptodate(bh)) {
3129	bh->b_end_io = end_buffer_read_sync;
3130	get_bh(bh);
3131	submit_bh(op, op_flags, bh);
3132	continue;
3133	}
3134	}
3135	unlock_buffer(bh);
3136	}
3137	}
3138	EXPORT_SYMBOL(ll_rw_block);
3139
3140	void write_dirty_buffer(struct buffer_head *bh, int op_flags)
3141	{
3142	lock_buffer(bh);
3143	if (!test_clear_buffer_dirty(bh)) {
3144	unlock_buffer(bh);
3145	return;
3146	}
3147	bh->b_end_io = end_buffer_write_sync;
3148	get_bh(bh);
3149	submit_bh(REQ_OP_WRITE, op_flags, bh);
3150	}
3151	EXPORT_SYMBOL(write_dirty_buffer);
3152
3153	/*
3154	* For a data-integrity writeout, we need to wait upon any in-progress I/O
3155	* and then start new I/O and then wait upon it. The caller must have a ref on
3156	* the buffer_head.
3157	*/
3158	int __sync_dirty_buffer(struct buffer_head *bh, int op_flags)
3159	{
3160	int ret = 0;
3161
3162	WARN_ON(atomic_read(&bh->b_count) < 1);
3163	lock_buffer(bh);
3164	if (test_clear_buffer_dirty(bh)) {
3165	/*
3166	* The bh should be mapped, but it might not be if the
3167	* device was hot-removed. Not much we can do but fail the I/O.
3168	*/
3169	if (!buffer_mapped(bh)) {
3170	unlock_buffer(bh);
3171	return -EIO;
3172	}
3173
3174	get_bh(bh);
3175	bh->b_end_io = end_buffer_write_sync;
3176	ret = submit_bh(REQ_OP_WRITE, op_flags, bh);
3177	wait_on_buffer(bh);
3178	if (!ret && !buffer_uptodate(bh))
3179	ret = -EIO;
3180	} else {
3181	unlock_buffer(bh);
3182	}
3183	return ret;
3184	}
3185	EXPORT_SYMBOL(__sync_dirty_buffer);
3186
3187	int sync_dirty_buffer(struct buffer_head *bh)
3188	{
3189	return __sync_dirty_buffer(bh, REQ_SYNC);
3190	}
3191	EXPORT_SYMBOL(sync_dirty_buffer);
3192
3193	/*
3194	* try_to_free_buffers() checks if all the buffers on this particular page
3195	* are unused, and releases them if so.
3196	*
3197	* Exclusion against try_to_free_buffers may be obtained by either
3198	* locking the page or by holding its mapping's private_lock.
3199	*
3200	* If the page is dirty but all the buffers are clean then we need to
3201	* be sure to mark the page clean as well. This is because the page
3202	* may be against a block device, and a later reattachment of buffers
3203	* to a dirty page will set all buffers dirty. Which would corrupt
3204	* filesystem data on the same device.
3205	*
3206	* The same applies to regular filesystem pages: if all the buffers are
3207	* clean then we set the page clean and proceed. To do that, we require
3208	* total exclusion from __set_page_dirty_buffers(). That is obtained with
3209	* private_lock.
3210	*
3211	* try_to_free_buffers() is non-blocking.
3212	*/
3213	static inline int buffer_busy(struct buffer_head *bh)
3214	{
3215	return atomic_read(&bh->b_count) \|
3216	(bh->b_state & ((1 << BH_Dirty) \| (1 << BH_Lock)));
3217	}
3218
3219	static int
3220	drop_buffers(struct page page, struct buffer_head *buffers_to_free)
3221	{
3222	struct buffer_head *head = page_buffers(page);
3223	struct buffer_head *bh;
3224
3225	bh = head;
3226	do {
3227	if (buffer_busy(bh))
3228	goto failed;
3229	bh = bh->b_this_page;
3230	} while (bh != head);
3231
3232	do {
3233	struct buffer_head *next = bh->b_this_page;
3234
3235	if (bh->b_assoc_map)
3236	__remove_assoc_queue(bh);
3237	bh = next;
3238	} while (bh != head);
3239	*buffers_to_free = head;
3240	detach_page_private(page);
3241	return 1;
3242	failed:
3243	return 0;
3244	}
3245
3246	int try_to_free_buffers(struct page *page)
3247	{
3248	struct address_space * const mapping = page->mapping;
3249	struct buffer_head *buffers_to_free = NULL;
3250	int ret = 0;
3251
3252	BUG_ON(!PageLocked(page));
3253	if (PageWriteback(page))
3254	return 0;
3255
3256	if (mapping == NULL) { /* can this still happen? */
3257	ret = drop_buffers(page, &buffers_to_free);
3258	goto out;
3259	}
3260
3261	spin_lock(&mapping->private_lock);
3262	ret = drop_buffers(page, &buffers_to_free);
3263
3264	/*
3265	* If the filesystem writes its buffers by hand (eg ext3)
3266	* then we can have clean buffers against a dirty page. We
3267	* clean the page here; otherwise the VM will never notice
3268	* that the filesystem did any IO at all.
3269	*
3270	* Also, during truncate, discard_buffer will have marked all
3271	* the page's buffers clean. We discover that here and clean
3272	* the page also.
3273	*
3274	* private_lock must be held over this entire operation in order
3275	* to synchronise against __set_page_dirty_buffers and prevent the
3276	* dirty bit from being lost.
3277	*/
3278	if (ret)
3279	cancel_dirty_page(page);
3280	spin_unlock(&mapping->private_lock);
3281	out:
3282	if (buffers_to_free) {
3283	struct buffer_head *bh = buffers_to_free;
3284
3285	do {
3286	struct buffer_head *next = bh->b_this_page;
3287	free_buffer_head(bh);
3288	bh = next;
3289	} while (bh != buffers_to_free);
3290	}
3291	return ret;
3292	}
3293	EXPORT_SYMBOL(try_to_free_buffers);
3294
3295	/*
3296	* There are no bdflush tunables left. But distributions are
3297	* still running obsolete flush daemons, so we terminate them here.
3298	*
3299	* Use of bdflush() is deprecated and will be removed in a future kernel.
3300	* The `flush-X' kernel threads fully replace bdflush daemons and this call.
3301	*/
3302	SYSCALL_DEFINE2(bdflush, int, func, long, data)
3303	{
3304	static int msg_count;
3305
3306	if (!capable(CAP_SYS_ADMIN))
3307	return -EPERM;
3308
3309	if (msg_count < 5) {
3310	msg_count++;
3311	printk(KERN_INFO
3312	"warning: process `%s' used the obsolete bdflush"
3313	" system call\n", current->comm);
3314	printk(KERN_INFO "Fix your initscripts?\n");
3315	}
3316
3317	if (func == 1)
3318	do_exit(0);
3319	return 0;
3320	}
3321
3322	/*
3323	* Buffer-head allocation
3324	*/
3325	static struct kmem_cache *bh_cachep __read_mostly;
3326
3327	/*
3328	* Once the number of bh's in the machine exceeds this level, we start
3329	* stripping them in writeback.
3330	*/
3331	static unsigned long max_buffer_heads;
3332
3333	int buffer_heads_over_limit;
3334
3335	struct bh_accounting {
3336	int nr; /* Number of live bh's */
3337	int ratelimit; /* Limit cacheline bouncing */
3338	};
3339
3340	static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3341
3342	static void recalc_bh_state(void)
3343	{
3344	int i;
3345	int tot = 0;
3346
3347	if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
3348	return;
3349	__this_cpu_write(bh_accounting.ratelimit, 0);
3350	for_each_online_cpu(i)
3351	tot += per_cpu(bh_accounting, i).nr;
3352	buffer_heads_over_limit = (tot > max_buffer_heads);
3353	}
3354
3355	struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
3356	{
3357	struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
3358	if (ret) {
3359	INIT_LIST_HEAD(&ret->b_assoc_buffers);
3360	spin_lock_init(&ret->b_uptodate_lock);
3361	preempt_disable();
3362	__this_cpu_inc(bh_accounting.nr);
3363	recalc_bh_state();
3364	preempt_enable();
3365	}
3366	return ret;
3367	}
3368	EXPORT_SYMBOL(alloc_buffer_head);
3369
3370	void free_buffer_head(struct buffer_head *bh)
3371	{
3372	BUG_ON(!list_empty(&bh->b_assoc_buffers));
3373	kmem_cache_free(bh_cachep, bh);
3374	preempt_disable();
3375	__this_cpu_dec(bh_accounting.nr);
3376	recalc_bh_state();
3377	preempt_enable();
3378	}
3379	EXPORT_SYMBOL(free_buffer_head);
3380
3381	static int buffer_exit_cpu_dead(unsigned int cpu)
3382	{
3383	int i;
3384	struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3385
3386	for (i = 0; i < BH_LRU_SIZE; i++) {
3387	brelse(b->bhs[i]);
3388	b->bhs[i] = NULL;
3389	}
3390	this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
3391	per_cpu(bh_accounting, cpu).nr = 0;
3392	return 0;
3393	}
3394
3395	/**
3396	* bh_uptodate_or_lock - Test whether the buffer is uptodate
3397	* @bh: struct buffer_head
3398	*
3399	* Return true if the buffer is up-to-date and false,
3400	* with the buffer locked, if not.
3401	*/
3402	int bh_uptodate_or_lock(struct buffer_head *bh)
3403	{
3404	if (!buffer_uptodate(bh)) {
3405	lock_buffer(bh);
3406	if (!buffer_uptodate(bh))
3407	return 0;
3408	unlock_buffer(bh);
3409	}
3410	return 1;
3411	}
3412	EXPORT_SYMBOL(bh_uptodate_or_lock);
3413
3414	/**
3415	* bh_submit_read - Submit a locked buffer for reading
3416	* @bh: struct buffer_head
3417	*
3418	* Returns zero on success and -EIO on error.
3419	*/
3420	int bh_submit_read(struct buffer_head *bh)
3421	{
3422	BUG_ON(!buffer_locked(bh));
3423
3424	if (buffer_uptodate(bh)) {
3425	unlock_buffer(bh);
3426	return 0;
3427	}
3428
3429	get_bh(bh);
3430	bh->b_end_io = end_buffer_read_sync;
3431	submit_bh(REQ_OP_READ, 0, bh);
3432	wait_on_buffer(bh);
3433	if (buffer_uptodate(bh))
3434	return 0;
3435	return -EIO;
3436	}
3437	EXPORT_SYMBOL(bh_submit_read);
3438
3439	void __init buffer_init(void)
3440	{
3441	unsigned long nrpages;
3442	int ret;
3443
3444	bh_cachep = kmem_cache_create("buffer_head",
3445	sizeof(struct buffer_head), 0,
3446	(SLAB_RECLAIM_ACCOUNT\|SLAB_PANIC\|
3447	SLAB_MEM_SPREAD),
3448	NULL);
3449
3450	/*
3451	* Limit the bh occupancy to 10% of ZONE_NORMAL
3452	*/
3453	nrpages = (nr_free_buffer_pages() * 10) / 100;
3454	max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3455	ret = cpuhp_setup_state_nocalls(CPUHP_FS_BUFF_DEAD, "fs/buffer:dead",
3456	NULL, buffer_exit_cpu_dead);
3457	WARN_ON(ret < 0);
3458	}