linux - fs/btrfs/inode.c

10885 lines · 314.2 KiB

1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Copyright (C) 2007 Oracle. All rights reserved.
4	*/
5
6	#include <linux/kernel.h>
7	#include <linux/bio.h>
8	#include <linux/blk-cgroup.h>
9	#include <linux/file.h>
10	#include <linux/filelock.h>
11	#include <linux/fs.h>
12	#include <linux/fs_struct.h>
13	#include <linux/pagemap.h>
14	#include <linux/highmem.h>
15	#include <linux/time.h>
16	#include <linux/init.h>
17	#include <linux/string.h>
18	#include <linux/backing-dev.h>
19	#include <linux/writeback.h>
20	#include <linux/compat.h>
21	#include <linux/xattr.h>
22	#include <linux/posix_acl.h>
23	#include <linux/falloc.h>
24	#include <linux/slab.h>
25	#include <linux/ratelimit.h>
26	#include <linux/btrfs.h>
27	#include <linux/blkdev.h>
28	#include <linux/posix_acl_xattr.h>
29	#include <linux/uio.h>
30	#include <linux/magic.h>
31	#include <linux/iversion.h>
32	#include <linux/swap.h>
33	#include <linux/migrate.h>
34	#include <linux/sched/mm.h>
35	#include <linux/iomap.h>
36	#include <linux/unaligned.h>
37	#include "misc.h"
38	#include "ctree.h"
39	#include "disk-io.h"
40	#include "transaction.h"
41	#include "btrfs_inode.h"
42	#include "ordered-data.h"
43	#include "xattr.h"
44	#include "tree-log.h"
45	#include "bio.h"
46	#include "compression.h"
47	#include "locking.h"
48	#include "props.h"
49	#include "qgroup.h"
50	#include "delalloc-space.h"
51	#include "block-group.h"
52	#include "space-info.h"
53	#include "zoned.h"
54	#include "subpage.h"
55	#include "inode-item.h"
56	#include "fs.h"
57	#include "accessors.h"
58	#include "extent-tree.h"
59	#include "root-tree.h"
60	#include "defrag.h"
61	#include "dir-item.h"
62	#include "file-item.h"
63	#include "uuid-tree.h"
64	#include "ioctl.h"
65	#include "file.h"
66	#include "acl.h"
67	#include "relocation.h"
68	#include "verity.h"
69	#include "super.h"
70	#include "orphan.h"
71	#include "backref.h"
72	#include "raid-stripe-tree.h"
73	#include "fiemap.h"
74	#include "delayed-inode.h"
75
76	#define COW_FILE_RANGE_KEEP_LOCKED (1UL << 0)
77
78	struct btrfs_iget_args {
79	u64 ino;
80	struct btrfs_root *root;
81	};
82
83	struct btrfs_rename_ctx {
84	/* Output field. Stores the index number of the old directory entry. */
85	u64 index;
86	};
87
88	/*
89	* Used by data_reloc_print_warning_inode() to pass needed info for filename
90	* resolution and output of error message.
91	*/
92	struct data_reloc_warn {
93	struct btrfs_path path;
94	struct btrfs_fs_info *fs_info;
95	u64 extent_item_size;
96	u64 logical;
97	int mirror_num;
98	};
99
100	/*
101	* For the file_extent_tree, we want to hold the inode lock when we lookup and
102	* update the disk_i_size, but lockdep will complain because our io_tree we hold
103	* the tree lock and get the inode lock when setting delalloc. These two things
104	* are unrelated, so make a class for the file_extent_tree so we don't get the
105	* two locking patterns mixed up.
106	*/
107	static struct lock_class_key file_extent_tree_class;
108
109	static const struct inode_operations btrfs_dir_inode_operations;
110	static const struct inode_operations btrfs_symlink_inode_operations;
111	static const struct inode_operations btrfs_special_inode_operations;
112	static const struct inode_operations btrfs_file_inode_operations;
113	static const struct address_space_operations btrfs_aops;
114	static const struct file_operations btrfs_dir_file_operations;
115
116	static struct kmem_cache *btrfs_inode_cachep;
117
118	static int btrfs_setsize(struct inode inode, struct iattr attr);
119	static int btrfs_truncate(struct btrfs_inode *inode, bool skip_writeback);
120
121	static noinline int run_delalloc_cow(struct btrfs_inode *inode,
122	struct folio *locked_folio, u64 start,
123	u64 end, struct writeback_control *wbc,
124	bool pages_dirty);
125
126	static int data_reloc_print_warning_inode(u64 inum, u64 offset, u64 num_bytes,
127	u64 root, void *warn_ctx)
128	{
129	struct data_reloc_warn *warn = warn_ctx;
130	struct btrfs_fs_info *fs_info = warn->fs_info;
131	struct extent_buffer *eb;
132	struct btrfs_inode_item *inode_item;
133	struct inode_fs_paths *ipath __free(inode_fs_paths) = NULL;
134	struct btrfs_root *local_root;
135	struct btrfs_key key;
136	unsigned int nofs_flag;
137	u32 nlink;
138	int ret;
139
140	local_root = btrfs_get_fs_root(fs_info, root, true);
141	if (IS_ERR(local_root)) {
142	ret = PTR_ERR(local_root);
143	goto err;
144	}
145
146	/* This makes the path point to (inum INODE_ITEM ioff). */
147	key.objectid = inum;
148	key.type = BTRFS_INODE_ITEM_KEY;
149	key.offset = 0;
150
151	ret = btrfs_search_slot(NULL, local_root, &key, &warn->path, 0, 0);
152	if (ret) {
153	btrfs_put_root(local_root);
154	btrfs_release_path(&warn->path);
155	goto err;
156	}
157
158	eb = warn->path.nodes[0];
159	inode_item = btrfs_item_ptr(eb, warn->path.slots[0], struct btrfs_inode_item);
160	nlink = btrfs_inode_nlink(eb, inode_item);
161	btrfs_release_path(&warn->path);
162
163	nofs_flag = memalloc_nofs_save();
164	ipath = init_ipath(4096, local_root, &warn->path);
165	memalloc_nofs_restore(nofs_flag);
166	if (IS_ERR(ipath)) {
167	btrfs_put_root(local_root);
168	ret = PTR_ERR(ipath);
169	ipath = NULL;
170	/*
171	* -ENOMEM, not a critical error, just output an generic error
172	* without filename.
173	*/
174	btrfs_warn(fs_info,
175	"checksum error at logical %llu mirror %u root %llu, inode %llu offset %llu",
176	warn->logical, warn->mirror_num, root, inum, offset);
177	return ret;
178	}
179	ret = paths_from_inode(inum, ipath);
180	if (ret < 0) {
181	btrfs_put_root(local_root);
182	goto err;
183	}
184
185	/*
186	* We deliberately ignore the bit ipath might have been too small to
187	* hold all of the paths here
188	*/
189	for (int i = 0; i < ipath->fspath->elem_cnt; i++) {
190	btrfs_warn(fs_info,
191	"checksum error at logical %llu mirror %u root %llu inode %llu offset %llu length %u links %u (path: %s)",
192	warn->logical, warn->mirror_num, root, inum, offset,
193	fs_info->sectorsize, nlink,
194	(char *)(unsigned long)ipath->fspath->val[i]);
195	}
196
197	btrfs_put_root(local_root);
198	return 0;
199
200	err:
201	btrfs_warn(fs_info,
202	"checksum error at logical %llu mirror %u root %llu inode %llu offset %llu, path resolving failed with ret=%d",
203	warn->logical, warn->mirror_num, root, inum, offset, ret);
204
205	return ret;
206	}
207
208	/*
209	* Do extra user-friendly error output (e.g. lookup all the affected files).
210	*
211	* Return true if we succeeded doing the backref lookup.
212	* Return false if such lookup failed, and has to fallback to the old error message.
213	*/
214	static void print_data_reloc_error(const struct btrfs_inode *inode, u64 file_off,
215	const u8 csum, const u8 csum_expected,
216	int mirror_num)
217	{
218	struct btrfs_fs_info *fs_info = inode->root->fs_info;
219	BTRFS_PATH_AUTO_RELEASE(path);
220	struct btrfs_key found_key = { 0 };
221	struct extent_buffer *eb;
222	struct btrfs_extent_item *ei;
223	const u32 csum_size = fs_info->csum_size;
224	u64 logical;
225	u64 flags;
226	u32 item_size;
227	int ret;
228
229	logical = btrfs_get_reloc_bg_bytenr(fs_info);
230
231	if (logical == U64_MAX) {
232	btrfs_warn_rl(fs_info, "has data reloc tree but no running relocation");
233	btrfs_warn_rl(fs_info,
234	"csum failed root %lld ino %llu off %llu csum " BTRFS_CSUM_FMT " expected csum " BTRFS_CSUM_FMT " mirror %d",
235	btrfs_root_id(inode->root), btrfs_ino(inode), file_off,
236	BTRFS_CSUM_FMT_VALUE(csum_size, csum),
237	BTRFS_CSUM_FMT_VALUE(csum_size, csum_expected),
238	mirror_num);
239	return;
240	}
241
242	logical += file_off;
243	btrfs_warn_rl(fs_info,
244	"csum failed root %lld ino %llu off %llu logical %llu csum " BTRFS_CSUM_FMT " expected csum " BTRFS_CSUM_FMT " mirror %d",
245	btrfs_root_id(inode->root),
246	btrfs_ino(inode), file_off, logical,
247	BTRFS_CSUM_FMT_VALUE(csum_size, csum),
248	BTRFS_CSUM_FMT_VALUE(csum_size, csum_expected),
249	mirror_num);
250
251	ret = extent_from_logical(fs_info, logical, &path, &found_key, &flags);
252	if (ret < 0) {
253	btrfs_err_rl(fs_info, "failed to lookup extent item for logical %llu: %pe",
254	logical, ERR_PTR(ret));
255	return;
256	}
257	eb = path.nodes[0];
258	ei = btrfs_item_ptr(eb, path.slots[0], struct btrfs_extent_item);
259	item_size = btrfs_item_size(eb, path.slots[0]);
260	if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
261	unsigned long ptr = 0;
262	u64 ref_root;
263	u8 ref_level;
264
265	while (true) {
266	ret = tree_backref_for_extent(&ptr, eb, &found_key, ei,
267	item_size, &ref_root,
268	&ref_level);
269	if (ret < 0) {
270	btrfs_warn_rl(fs_info,
271	"failed to resolve tree backref for logical %llu: %d",
272	logical, ret);
273	break;
274	}
275	if (ret > 0)
276	break;
277
278	btrfs_warn_rl(fs_info,
279	"csum error at logical %llu mirror %u: metadata %s (level %d) in tree %llu",
280	logical, mirror_num,
281	(ref_level ? "node" : "leaf"),
282	ref_level, ref_root);
283	}
284	} else {
285	struct btrfs_backref_walk_ctx ctx = { 0 };
286	struct data_reloc_warn reloc_warn = { 0 };
287
288	/*
289	* Do not hold the path as later iterate_extent_inodes() call
290	* can be time consuming.
291	*/
292	btrfs_release_path(&path);
293
294	ctx.bytenr = found_key.objectid;
295	ctx.extent_item_pos = logical - found_key.objectid;
296	ctx.fs_info = fs_info;
297
298	reloc_warn.logical = logical;
299	reloc_warn.extent_item_size = found_key.offset;
300	reloc_warn.mirror_num = mirror_num;
301	reloc_warn.fs_info = fs_info;
302
303	iterate_extent_inodes(&ctx, true,
304	data_reloc_print_warning_inode, &reloc_warn);
305	}
306	}
307
308	static void __cold btrfs_print_data_csum_error(struct btrfs_inode *inode,
309	u64 logical_start, u8 csum, u8 csum_expected, int mirror_num)
310	{
311	struct btrfs_root *root = inode->root;
312	const u32 csum_size = root->fs_info->csum_size;
313
314	/* For data reloc tree, it's better to do a backref lookup instead. */
315	if (btrfs_is_data_reloc_root(root))
316	return print_data_reloc_error(inode, logical_start, csum,
317	csum_expected, mirror_num);
318
319	/* Output without objectid, which is more meaningful */
320	if (btrfs_root_id(root) >= BTRFS_LAST_FREE_OBJECTID) {
321	btrfs_warn_rl(root->fs_info,
322	"csum failed root %lld ino %lld off %llu csum " BTRFS_CSUM_FMT " expected csum " BTRFS_CSUM_FMT " mirror %d",
323	btrfs_root_id(root), btrfs_ino(inode),
324	logical_start,
325	BTRFS_CSUM_FMT_VALUE(csum_size, csum),
326	BTRFS_CSUM_FMT_VALUE(csum_size, csum_expected),
327	mirror_num);
328	} else {
329	btrfs_warn_rl(root->fs_info,
330	"csum failed root %llu ino %llu off %llu csum " BTRFS_CSUM_FMT " expected csum " BTRFS_CSUM_FMT " mirror %d",
331	btrfs_root_id(root), btrfs_ino(inode),
332	logical_start,
333	BTRFS_CSUM_FMT_VALUE(csum_size, csum),
334	BTRFS_CSUM_FMT_VALUE(csum_size, csum_expected),
335	mirror_num);
336	}
337	}
338
339	/*
340	* Lock inode i_rwsem based on arguments passed.
341	*
342	* ilock_flags can have the following bit set:
343	*
344	* BTRFS_ILOCK_SHARED - acquire a shared lock on the inode
345	* BTRFS_ILOCK_TRY - try to acquire the lock, if fails on first attempt
346	* return -EAGAIN
347	* BTRFS_ILOCK_MMAP - acquire a write lock on the i_mmap_lock
348	*/
349	int btrfs_inode_lock(struct btrfs_inode *inode, unsigned int ilock_flags)
350	{
351	if (ilock_flags & BTRFS_ILOCK_SHARED) {
352	if (ilock_flags & BTRFS_ILOCK_TRY) {
353	if (!inode_trylock_shared(&inode->vfs_inode))
354	return -EAGAIN;
355	else
356	return 0;
357	}
358	inode_lock_shared(&inode->vfs_inode);
359	} else {
360	if (ilock_flags & BTRFS_ILOCK_TRY) {
361	if (!inode_trylock(&inode->vfs_inode))
362	return -EAGAIN;
363	else
364	return 0;
365	}
366	inode_lock(&inode->vfs_inode);
367	}
368	if (ilock_flags & BTRFS_ILOCK_MMAP)
369	down_write(&inode->i_mmap_lock);
370	return 0;
371	}
372
373	/*
374	* Unlock inode i_rwsem.
375	*
376	* ilock_flags should contain the same bits set as passed to btrfs_inode_lock()
377	* to decide whether the lock acquired is shared or exclusive.
378	*/
379	void btrfs_inode_unlock(struct btrfs_inode *inode, unsigned int ilock_flags)
380	{
381	if (ilock_flags & BTRFS_ILOCK_MMAP)
382	up_write(&inode->i_mmap_lock);
383	if (ilock_flags & BTRFS_ILOCK_SHARED)
384	inode_unlock_shared(&inode->vfs_inode);
385	else
386	inode_unlock(&inode->vfs_inode);
387	}
388
389	/*
390	* Cleanup all submitted ordered extents in specified range to handle errors
391	* from the btrfs_run_delalloc_range() callback.
392	*
393	* NOTE: caller must ensure that when an error happens, it can not call
394	* extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING
395	* and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata
396	* to be released, which we want to happen only when finishing the ordered
397	* extent (btrfs_finish_ordered_io()).
398	*/
399	static inline void btrfs_cleanup_ordered_extents(struct btrfs_inode *inode,
400	u64 offset, u64 bytes)
401	{
402	return btrfs_mark_ordered_io_finished(inode, offset, bytes, false);
403	}
404
405	static int btrfs_dirty_inode(struct btrfs_inode *inode);
406
407	static int btrfs_init_inode_security(struct btrfs_trans_handle *trans,
408	struct btrfs_new_inode_args *args)
409	{
410	int ret;
411
412	if (args->default_acl) {
413	ret = __btrfs_set_acl(trans, args->inode, args->default_acl,
414	ACL_TYPE_DEFAULT);
415	if (ret)
416	return ret;
417	}
418	if (args->acl) {
419	ret = __btrfs_set_acl(trans, args->inode, args->acl, ACL_TYPE_ACCESS);
420	if (ret)
421	return ret;
422	}
423	if (!args->default_acl && !args->acl)
424	cache_no_acl(args->inode);
425	return btrfs_xattr_security_init(trans, args->inode, args->dir,
426	&args->dentry->d_name);
427	}
428
429	/*
430	* this does all the hard work for inserting an inline extent into
431	* the btree. The caller should have done a btrfs_drop_extents so that
432	* no overlapping inline items exist in the btree
433	*/
434	static int insert_inline_extent(struct btrfs_trans_handle *trans,
435	struct btrfs_path *path,
436	struct btrfs_inode *inode, bool extent_inserted,
437	size_t size, size_t compressed_size,
438	int compress_type,
439	struct folio *compressed_folio,
440	bool update_i_size)
441	{
442	struct btrfs_root *root = inode->root;
443	struct extent_buffer *leaf;
444	const u32 sectorsize = trans->fs_info->sectorsize;
445	char *kaddr;
446	unsigned long ptr;
447	struct btrfs_file_extent_item *ei;
448	int ret;
449	size_t cur_size = size;
450	u64 i_size;
451
452	/*
453	* The decompressed size must still be no larger than a sector. Under
454	* heavy race, we can have size == 0 passed in, but that shouldn't be a
455	* big deal and we can continue the insertion.
456	*/
457	ASSERT(size <= sectorsize);
458
459	/*
460	* The compressed size also needs to be no larger than a page.
461	* That's also why we only need one folio as the parameter.
462	*/
463	if (compressed_folio) {
464	ASSERT(compressed_size <= sectorsize);
465	ASSERT(compressed_size <= PAGE_SIZE);
466	} else {
467	ASSERT(compressed_size == 0);
468	}
469
470	if (compressed_size && compressed_folio)
471	cur_size = compressed_size;
472
473	if (!extent_inserted) {
474	struct btrfs_key key;
475	size_t datasize;
476
477	key.objectid = btrfs_ino(inode);
478	key.type = BTRFS_EXTENT_DATA_KEY;
479	key.offset = 0;
480
481	datasize = btrfs_file_extent_calc_inline_size(cur_size);
482	ret = btrfs_insert_empty_item(trans, root, path, &key,
483	datasize);
484	if (ret)
485	return ret;
486	}
487	leaf = path->nodes[0];
488	ei = btrfs_item_ptr(leaf, path->slots[0],
489	struct btrfs_file_extent_item);
490	btrfs_set_file_extent_generation(leaf, ei, trans->transid);
491	btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE);
492	btrfs_set_file_extent_encryption(leaf, ei, 0);
493	btrfs_set_file_extent_other_encoding(leaf, ei, 0);
494	btrfs_set_file_extent_ram_bytes(leaf, ei, size);
495	ptr = btrfs_file_extent_inline_start(ei);
496
497	if (compress_type != BTRFS_COMPRESS_NONE) {
498	kaddr = kmap_local_folio(compressed_folio, 0);
499	write_extent_buffer(leaf, kaddr, ptr, compressed_size);
500	kunmap_local(kaddr);
501
502	btrfs_set_file_extent_compression(leaf, ei,
503	compress_type);
504	} else {
505	struct folio *folio;
506
507	folio = filemap_get_folio(inode->vfs_inode.i_mapping, 0);
508	ASSERT(!IS_ERR(folio));
509	btrfs_set_file_extent_compression(leaf, ei, 0);
510	kaddr = kmap_local_folio(folio, 0);
511	write_extent_buffer(leaf, kaddr, ptr, size);
512	kunmap_local(kaddr);
513	folio_put(folio);
514	}
515	btrfs_release_path(path);
516
517	/*
518	* We align size to sectorsize for inline extents just for simplicity
519	* sake.
520	*/
521	ret = btrfs_inode_set_file_extent_range(inode, 0,
522	ALIGN(size, root->fs_info->sectorsize));
523	if (ret)
524	return ret;
525
526	/*
527	* We're an inline extent, so nobody can extend the file past i_size
528	* without locking a page we already have locked.
529	*
530	* We must do any i_size and inode updates before we unlock the pages.
531	* Otherwise we could end up racing with unlink.
532	*/
533	i_size = i_size_read(&inode->vfs_inode);
534	if (update_i_size && size > i_size) {
535	i_size_write(&inode->vfs_inode, size);
536	i_size = size;
537	}
538	inode->disk_i_size = i_size;
539
540	return 0;
541	}
542
543	static bool can_cow_file_range_inline(struct btrfs_inode *inode,
544	u64 offset, u64 size,
545	size_t compressed_size)
546	{
547	struct btrfs_fs_info *fs_info = inode->root->fs_info;
548	u64 data_len = (compressed_size ?: size);
549
550	/* Inline extents must start at offset 0. */
551	if (offset != 0)
552	return false;
553
554	/*
555	* Even for bs > ps cases, cow_file_range_inline() can only accept a
556	* single folio.
557	*
558	* This can be problematic and cause access beyond page boundary if a
559	* page sized folio is passed into that function.
560	* And encoded write is doing exactly that.
561	* So here limits the inlined extent size to PAGE_SIZE.
562	*/
563	if (size > PAGE_SIZE \|\| compressed_size > PAGE_SIZE)
564	return false;
565
566	/* Inline extents are limited to sectorsize. */
567	if (size > fs_info->sectorsize)
568	return false;
569
570	/* We do not allow a non-compressed extent to be as large as block size. */
571	if (data_len >= fs_info->sectorsize)
572	return false;
573
574	/* We cannot exceed the maximum inline data size. */
575	if (data_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info))
576	return false;
577
578	/* We cannot exceed the user specified max_inline size. */
579	if (data_len > fs_info->max_inline)
580	return false;
581
582	/* Inline extents must be the entirety of the file. */
583	if (size < i_size_read(&inode->vfs_inode))
584	return false;
585
586	/* Encrypted file cannot be inlined. */
587	if (IS_ENCRYPTED(&inode->vfs_inode))
588	return false;
589
590	return true;
591	}
592
593	/*
594	* conditionally insert an inline extent into the file. This
595	* does the checks required to make sure the data is small enough
596	* to fit as an inline extent.
597	*
598	* If being used directly, you must have already checked we're allowed to cow
599	* the range by getting true from can_cow_file_range_inline().
600	*
601	* Return 0 if the inlined extent is created successfully.
602	* Return <0 for critical error, and should be considered as an writeback error.
603	* Return >0 if can not create an inlined extent (mostly due to lack of meta space).
604	*/
605	static noinline int __cow_file_range_inline(struct btrfs_inode *inode,
606	u64 size, size_t compressed_size,
607	int compress_type,
608	struct folio *compressed_folio,
609	bool update_i_size)
610	{
611	struct btrfs_drop_extents_args drop_args = { 0 };
612	struct btrfs_root *root = inode->root;
613	struct btrfs_fs_info *fs_info = root->fs_info;
614	struct btrfs_trans_handle *trans = NULL;
615	u64 data_len = (compressed_size ?: size);
616	int ret;
617	struct btrfs_path *path;
618
619	path = btrfs_alloc_path();
620	if (!path) {
621	ret = -ENOMEM;
622	goto out;
623	}
624
625	trans = btrfs_join_transaction(root);
626	if (IS_ERR(trans)) {
627	ret = PTR_ERR(trans);
628	trans = NULL;
629	goto out;
630	}
631	trans->block_rsv = &inode->block_rsv;
632
633	drop_args.path = path;
634	drop_args.start = 0;
635	drop_args.end = fs_info->sectorsize;
636	drop_args.drop_cache = true;
637	drop_args.replace_extent = true;
638	drop_args.extent_item_size = btrfs_file_extent_calc_inline_size(data_len);
639	ret = btrfs_drop_extents(trans, root, inode, &drop_args);
640	if (unlikely(ret)) {
641	btrfs_abort_transaction(trans, ret);
642	goto out;
643	}
644
645	ret = insert_inline_extent(trans, path, inode, drop_args.extent_inserted,
646	size, compressed_size, compress_type,
647	compressed_folio, update_i_size);
648	if (unlikely(ret && ret != -ENOSPC)) {
649	btrfs_abort_transaction(trans, ret);
650	goto out;
651	} else if (ret == -ENOSPC) {
652	ret = 1;
653	goto out;
654	}
655
656	btrfs_update_inode_bytes(inode, size, drop_args.bytes_found);
657	ret = btrfs_update_inode(trans, inode);
658	if (unlikely(ret && ret != -ENOSPC)) {
659	btrfs_abort_transaction(trans, ret);
660	goto out;
661	} else if (ret == -ENOSPC) {
662	ret = 1;
663	goto out;
664	}
665
666	btrfs_set_inode_full_sync(inode);
667	out:
668	/*
669	* Don't forget to free the reserved space, as for inlined extent
670	* it won't count as data extent, free them directly here.
671	* And at reserve time, it's always aligned to sector size, so
672	* just free one sector here.
673	*
674	* If we fallback to non-inline (ret == 1) due to -ENOSPC, then we need
675	* to keep the data reservation.
676	*/
677	if (ret <= 0)
678	btrfs_qgroup_free_data(inode, NULL, 0, fs_info->sectorsize, NULL);
679	btrfs_free_path(path);
680	if (trans)
681	btrfs_end_transaction(trans);
682	return ret;
683	}
684
685	struct async_extent {
686	u64 start;
687	u64 ram_size;
688	struct compressed_bio *cb;
689	struct list_head list;
690	};
691
692	struct async_chunk {
693	struct btrfs_inode *inode;
694	struct folio *locked_folio;
695	u64 start;
696	u64 end;
697	blk_opf_t write_flags;
698	struct list_head extents;
699	struct cgroup_subsys_state *blkcg_css;
700	struct btrfs_work work;
701	struct async_cow *async_cow;
702	};
703
704	struct async_cow {
705	atomic_t num_chunks;
706	struct async_chunk chunks[];
707	};
708
709	static int add_async_extent(struct async_chunk *cow, u64 start, u64 ram_size,
710	struct compressed_bio *cb)
711	{
712	struct async_extent *async_extent;
713
714	async_extent = kmalloc_obj(*async_extent, GFP_NOFS);
715	if (!async_extent)
716	return -ENOMEM;
717	ASSERT(ram_size < U32_MAX);
718	async_extent->start = start;
719	async_extent->ram_size = ram_size;
720	async_extent->cb = cb;
721	list_add_tail(&async_extent->list, &cow->extents);
722	return 0;
723	}
724
725	/*
726	* Check if the inode needs to be submitted to compression, based on mount
727	* options, defragmentation, properties or heuristics.
728	*/
729	static inline int inode_need_compress(struct btrfs_inode *inode, u64 start,
730	u64 end, bool check_inline)
731	{
732	struct btrfs_fs_info *fs_info = inode->root->fs_info;
733	const u32 blocksize = fs_info->sectorsize;
734
735	ASSERT(IS_ALIGNED(start, blocksize) && IS_ALIGNED(end + 1, blocksize));
736
737	if (unlikely(!btrfs_inode_can_compress(inode))) {
738	DEBUG_WARN("BTRFS: unexpected compression for ino %llu", btrfs_ino(inode));
739	return 0;
740	}
741
742	/*
743	* If the delalloc range is only one fs block and can not be inlined,
744	* do not even bother try compression, as there will be no space saving
745	* and will always fallback to regular write later.
746	*/
747	if (end + 1 - start <= fs_info->sectorsize &&
748	(!check_inline \|\| (start > 0 \|\| end + 1 < inode->disk_i_size)))
749	return 0;
750
751	/* Defrag ioctl takes precedence over mount options and properties. */
752	if (inode->defrag_compress == BTRFS_DEFRAG_DONT_COMPRESS)
753	return 0;
754	if (BTRFS_COMPRESS_NONE < inode->defrag_compress &&
755	inode->defrag_compress < BTRFS_NR_COMPRESS_TYPES)
756	return 1;
757	/* force compress */
758	if (btrfs_test_opt(fs_info, FORCE_COMPRESS))
759	return 1;
760	/* bad compression ratios */
761	if (inode->flags & BTRFS_INODE_NOCOMPRESS)
762	return 0;
763	if (btrfs_test_opt(fs_info, COMPRESS) \|\|
764	inode->flags & BTRFS_INODE_COMPRESS \|\|
765	inode->prop_compress)
766	return btrfs_compress_heuristic(inode, start, end);
767	return 0;
768	}
769
770	static inline void inode_should_defrag(struct btrfs_inode *inode,
771	u64 start, u64 end, u64 num_bytes, u32 small_write)
772	{
773	/* If this is a small write inside eof, kick off a defrag */
774	if (num_bytes < small_write &&
775	(start > 0 \|\| end + 1 < inode->disk_i_size))
776	btrfs_add_inode_defrag(inode, small_write);
777	}
778
779	static int extent_range_clear_dirty_for_io(struct btrfs_inode *inode, u64 start, u64 end)
780	{
781	pgoff_t index = start >> PAGE_SHIFT;
782	const pgoff_t end_index = end >> PAGE_SHIFT;
783	struct folio *folio;
784	int ret = 0;
785
786	while (index <= end_index) {
787	folio = filemap_get_folio(inode->vfs_inode.i_mapping, index);
788	if (IS_ERR(folio)) {
789	if (!ret)
790	ret = PTR_ERR(folio);
791	index++;
792	continue;
793	}
794	/*
795	* We are about to compress the folio, so it must not be mmap
796	* writeable or we could corrupt the data as we attempt to
797	* compress it.
798	*/
799	btrfs_check_folio_write_protected(folio);
800	btrfs_folio_clamp_clear_dirty(inode->root->fs_info, folio, start,
801	end + 1 - start);
802	index = folio_next_index(folio);
803	folio_put(folio);
804	}
805	return ret;
806	}
807
808	static struct folio compressed_bio_last_folio(struct compressed_bio cb)
809	{
810	struct bio *bio = &cb->bbio.bio;
811	struct bio_vec *bvec;
812	phys_addr_t paddr;
813
814	/*
815	* Make sure all folios have the same min_folio_size.
816	*
817	* Otherwise we cannot simply use offset_in_offset(folio, bi_size) to
818	* calculate the end of the last folio.
819	*/
820	if (IS_ENABLED(CONFIG_BTRFS_ASSERT)) {
821	struct btrfs_fs_info *fs_info = cb_to_fs_info(cb);
822	const u32 min_folio_size = btrfs_min_folio_size(fs_info);
823	struct folio_iter fi;
824
825	bio_for_each_folio_all(fi, bio)
826	ASSERT(folio_size(fi.folio) == min_folio_size);
827	}
828
829	/* The bio must not be empty. */
830	ASSERT(bio->bi_vcnt);
831
832	bvec = &bio->bi_io_vec[bio->bi_vcnt - 1];
833	paddr = bvec_phys(bvec) + bvec->bv_len - 1;
834	return page_folio(phys_to_page(paddr));
835	}
836
837	static void round_up_last_block(struct compressed_bio *cb, u32 blocksize)
838	{
839	struct bio *bio = &cb->bbio.bio;
840	struct folio *last_folio = compressed_bio_last_folio(cb);
841	const u32 bio_size = bio->bi_iter.bi_size;
842	const u32 foffset = offset_in_folio(last_folio, bio_size);
843	const u32 padding_len = round_up(foffset, blocksize) - foffset;
844	bool ret;
845
846	if (IS_ALIGNED(bio_size, blocksize))
847	return;
848
849	folio_zero_range(last_folio, foffset, padding_len);
850	ret = bio_add_folio(bio, last_folio, padding_len, foffset);
851	/* The remaining part should be merged thus never fail. */
852	ASSERT(ret);
853	}
854
855	/*
856	* Work queue call back to started compression on a file and pages.
857	*
858	* This is done inside an ordered work queue, and the compression is spread
859	* across many cpus. The actual IO submission is step two, and the ordered work
860	* queue takes care of making sure that happens in the same order things were
861	* put onto the queue by writepages and friends.
862	*
863	* If this code finds it can't get good compression, it puts an entry onto the
864	* work queue to write the uncompressed bytes. This makes sure that both
865	* compressed inodes and uncompressed inodes are written in the same order that
866	* the flusher thread sent them down.
867	*/
868	static void compress_file_range(struct btrfs_work *work)
869	{
870	struct async_chunk *async_chunk =
871	container_of(work, struct async_chunk, work);
872	struct btrfs_inode *inode = async_chunk->inode;
873	struct btrfs_fs_info *fs_info = inode->root->fs_info;
874	struct compressed_bio *cb = NULL;
875	const u32 blocksize = fs_info->sectorsize;
876	u64 start = async_chunk->start;
877	u64 end = async_chunk->end;
878	u64 actual_end;
879	u64 i_size;
880	u32 cur_len;
881	int ret = 0;
882	unsigned long total_compressed = 0;
883	unsigned long total_in = 0;
884	int compress_type = fs_info->compress_type;
885	int compress_level = fs_info->compress_level;
886
887	if (btrfs_is_shutdown(fs_info))
888	goto cleanup_and_bail_uncompressed;
889
890	inode_should_defrag(inode, start, end, end - start + 1, SZ_16K);
891
892	ret = extent_range_clear_dirty_for_io(inode, start, end);
893
894	/*
895	* All the folios should have been locked thus no failure.
896	*
897	* And even if some folios are missing, btrfs_compress_bio()
898	* would handle them correctly, so here just do an ASSERT() check for
899	* early logic errors.
900	*/
901	ASSERT(ret == 0);
902
903	/*
904	* We need to save i_size before now because it could change in between
905	* us evaluating the size and assigning it. This is because we lock and
906	* unlock the page in truncate and fallocate, and then modify the i_size
907	* later on.
908	*
909	* The barriers are to emulate READ_ONCE, remove that once i_size_read
910	* does that for us.
911	*/
912	barrier();
913	i_size = i_size_read(&inode->vfs_inode);
914	barrier();
915	actual_end = min_t(u64, i_size, end + 1);
916	again:
917	total_in = 0;
918	cur_len = min(end + 1 - start, BTRFS_MAX_UNCOMPRESSED);
919	ret = 0;
920	cb = NULL;
921
922	/*
923	* we don't want to send crud past the end of i_size through
924	* compression, that's just a waste of CPU time. So, if the
925	* end of the file is before the start of our current
926	* requested range of bytes, we bail out to the uncompressed
927	* cleanup code that can deal with all of this.
928	*
929	* It isn't really the fastest way to fix things, but this is a
930	* very uncommon corner.
931	*/
932	if (actual_end <= start)
933	goto cleanup_and_bail_uncompressed;
934
935	/*
936	* We do compression for mount -o compress and when the inode has not
937	* been flagged as NOCOMPRESS. This flag can change at any time if we
938	* discover bad compression ratios.
939	*/
940	if (!inode_need_compress(inode, start, end, false))
941	goto cleanup_and_bail_uncompressed;
942
943	if (0 < inode->defrag_compress && inode->defrag_compress < BTRFS_NR_COMPRESS_TYPES) {
944	compress_type = inode->defrag_compress;
945	compress_level = inode->defrag_compress_level;
946	} else if (inode->prop_compress) {
947	compress_type = inode->prop_compress;
948	}
949
950	/* Compression level is applied here. */
951	cb = btrfs_compress_bio(inode, start, cur_len, compress_type,
952	compress_level, async_chunk->write_flags);
953	if (IS_ERR(cb)) {
954	cb = NULL;
955	goto mark_incompressible;
956	}
957
958	total_compressed = cb->bbio.bio.bi_iter.bi_size;
959	total_in = cur_len;
960
961	/*
962	* We aren't doing an inline extent. Round the compressed size up to a
963	* block size boundary so the allocator does sane things.
964	*/
965	round_up_last_block(cb, blocksize);
966	total_compressed = cb->bbio.bio.bi_iter.bi_size;
967	ASSERT(IS_ALIGNED(total_compressed, blocksize));
968
969	/*
970	* One last check to make sure the compression is really a win, compare
971	* the page count read with the blocks on disk, compression must free at
972	* least one sector.
973	*/
974	total_in = round_up(total_in, fs_info->sectorsize);
975	if (total_compressed + blocksize > total_in)
976	goto mark_incompressible;
977
978
979	/*
980	* The async work queues will take care of doing actual allocation on
981	* disk for these compressed pages, and will submit the bios.
982	*/
983	ret = add_async_extent(async_chunk, start, total_in, cb);
984	BUG_ON(ret);
985	if (start + total_in < end) {
986	start += total_in;
987	cond_resched();
988	goto again;
989	}
990	return;
991
992	mark_incompressible:
993	if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) && !inode->prop_compress)
994	inode->flags \|= BTRFS_INODE_NOCOMPRESS;
995	cleanup_and_bail_uncompressed:
996	ret = add_async_extent(async_chunk, start, end - start + 1, NULL);
997	BUG_ON(ret);
998	if (cb)
999	cleanup_compressed_bio(cb);
1000	}
1001
1002	static void submit_uncompressed_range(struct btrfs_inode *inode,
1003	struct async_extent *async_extent,
1004	struct folio *locked_folio)
1005	{
1006	u64 start = async_extent->start;
1007	u64 end = async_extent->start + async_extent->ram_size - 1;
1008	int ret;
1009	struct writeback_control wbc = {
1010	.sync_mode = WB_SYNC_ALL,
1011	.range_start = start,
1012	.range_end = end,
1013	.no_cgroup_owner = 1,
1014	};
1015
1016	wbc_attach_fdatawrite_inode(&wbc, &inode->vfs_inode);
1017	ret = run_delalloc_cow(inode, locked_folio, start, end,
1018	&wbc, false);
1019	wbc_detach_inode(&wbc);
1020	if (ret < 0) {
1021	if (locked_folio)
1022	btrfs_folio_end_lock(inode->root->fs_info, locked_folio,
1023	start, async_extent->ram_size);
1024	btrfs_err_rl(inode->root->fs_info,
1025	"%s failed, root=%llu inode=%llu start=%llu len=%llu: %pe",
1026	__func__, btrfs_root_id(inode->root),
1027	btrfs_ino(inode), start, async_extent->ram_size,
1028	ERR_PTR(ret));
1029	}
1030	}
1031
1032	static void submit_one_async_extent(struct async_chunk *async_chunk,
1033	struct async_extent *async_extent,
1034	u64 *alloc_hint)
1035	{
1036	struct btrfs_inode *inode = async_chunk->inode;
1037	struct extent_io_tree *io_tree = &inode->io_tree;
1038	struct btrfs_root *root = inode->root;
1039	struct btrfs_fs_info *fs_info = root->fs_info;
1040	struct btrfs_ordered_extent *ordered;
1041	struct btrfs_file_extent file_extent;
1042	struct btrfs_key ins;
1043	struct folio *locked_folio = NULL;
1044	struct extent_state *cached = NULL;
1045	struct extent_map *em;
1046	int ret = 0;
1047	u32 compressed_size;
1048	u64 start = async_extent->start;
1049	u64 end = async_extent->start + async_extent->ram_size - 1;
1050
1051	if (async_chunk->blkcg_css)
1052	kthread_associate_blkcg(async_chunk->blkcg_css);
1053
1054	/*
1055	* If async_chunk->locked_folio is in the async_extent range, we need to
1056	* handle it.
1057	*/
1058	if (async_chunk->locked_folio) {
1059	u64 locked_folio_start = folio_pos(async_chunk->locked_folio);
1060	u64 locked_folio_end = locked_folio_start +
1061	folio_size(async_chunk->locked_folio) - 1;
1062
1063	if (!(start >= locked_folio_end \|\| end <= locked_folio_start))
1064	locked_folio = async_chunk->locked_folio;
1065	}
1066
1067	if (!async_extent->cb) {
1068	submit_uncompressed_range(inode, async_extent, locked_folio);
1069	goto done;
1070	}
1071
1072	compressed_size = async_extent->cb->bbio.bio.bi_iter.bi_size;
1073	ret = btrfs_reserve_extent(root, async_extent->ram_size,
1074	compressed_size, compressed_size,
1075	0, *alloc_hint, &ins, true, true);
1076	if (ret) {
1077	/*
1078	* We can't reserve contiguous space for the compressed size.
1079	* Unlikely, but it's possible that we could have enough
1080	* non-contiguous space for the uncompressed size instead. So
1081	* fall back to uncompressed.
1082	*/
1083	submit_uncompressed_range(inode, async_extent, locked_folio);
1084	cleanup_compressed_bio(async_extent->cb);
1085	async_extent->cb = NULL;
1086	goto done;
1087	}
1088
1089	btrfs_lock_extent(io_tree, start, end, &cached);
1090
1091	/* Here we're doing allocation and writeback of the compressed pages */
1092	file_extent.disk_bytenr = ins.objectid;
1093	file_extent.disk_num_bytes = ins.offset;
1094	file_extent.ram_bytes = async_extent->ram_size;
1095	file_extent.num_bytes = async_extent->ram_size;
1096	file_extent.offset = 0;
1097	file_extent.compression = async_extent->cb->compress_type;
1098
1099	async_extent->cb->bbio.bio.bi_iter.bi_sector = ins.objectid >> SECTOR_SHIFT;
1100
1101	em = btrfs_create_io_em(inode, start, &file_extent, BTRFS_ORDERED_COMPRESSED);
1102	if (IS_ERR(em)) {
1103	ret = PTR_ERR(em);
1104	goto out_free_reserve;
1105	}
1106	btrfs_free_extent_map(em);
1107
1108	ordered = btrfs_alloc_ordered_extent(inode, start, &file_extent,
1109	1U << BTRFS_ORDERED_COMPRESSED);
1110	if (IS_ERR(ordered)) {
1111	btrfs_drop_extent_map_range(inode, start, end, false);
1112	ret = PTR_ERR(ordered);
1113	goto out_free_reserve;
1114	}
1115	async_extent->cb->bbio.ordered = ordered;
1116	btrfs_dec_block_group_reservations(fs_info, ins.objectid);
1117
1118	/* Clear dirty, set writeback and unlock the pages. */
1119	extent_clear_unlock_delalloc(inode, start, end,
1120	NULL, &cached, EXTENT_LOCKED \| EXTENT_DELALLOC,
1121	PAGE_UNLOCK \| PAGE_START_WRITEBACK);
1122	btrfs_submit_bbio(&async_extent->cb->bbio, 0);
1123	async_extent->cb = NULL;
1124
1125	*alloc_hint = ins.objectid + ins.offset;
1126	done:
1127	if (async_chunk->blkcg_css)
1128	kthread_associate_blkcg(NULL);
1129	kfree(async_extent);
1130	return;
1131
1132	out_free_reserve:
1133	btrfs_dec_block_group_reservations(fs_info, ins.objectid);
1134	btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, true);
1135	mapping_set_error(inode->vfs_inode.i_mapping, -EIO);
1136	extent_clear_unlock_delalloc(inode, start, end,
1137	NULL, &cached,
1138	EXTENT_LOCKED \| EXTENT_DELALLOC \|
1139	EXTENT_DELALLOC_NEW \|
1140	EXTENT_DEFRAG \| EXTENT_CLEAR_META_RESV,
1141	PAGE_UNLOCK \| PAGE_START_WRITEBACK \|
1142	PAGE_END_WRITEBACK);
1143	if (async_extent->cb)
1144	cleanup_compressed_bio(async_extent->cb);
1145	if (async_chunk->blkcg_css)
1146	kthread_associate_blkcg(NULL);
1147	btrfs_debug(fs_info,
1148	"async extent submission failed root=%lld inode=%llu start=%llu len=%llu ret=%d",
1149	btrfs_root_id(root), btrfs_ino(inode), start,
1150	async_extent->ram_size, ret);
1151	kfree(async_extent);
1152	}
1153
1154	u64 btrfs_get_extent_allocation_hint(struct btrfs_inode *inode, u64 start,
1155	u64 num_bytes)
1156	{
1157	struct extent_map_tree *em_tree = &inode->extent_tree;
1158	struct extent_map *em;
1159	u64 alloc_hint = 0;
1160
1161	read_lock(&em_tree->lock);
1162	em = btrfs_search_extent_mapping(em_tree, start, num_bytes);
1163	if (em) {
1164	/*
1165	* if block start isn't an actual block number then find the
1166	* first block in this inode and use that as a hint. If that
1167	* block is also bogus then just don't worry about it.
1168	*/
1169	if (em->disk_bytenr >= EXTENT_MAP_LAST_BYTE) {
1170	btrfs_free_extent_map(em);
1171	em = btrfs_search_extent_mapping(em_tree, 0, 0);
1172	if (em && em->disk_bytenr < EXTENT_MAP_LAST_BYTE)
1173	alloc_hint = btrfs_extent_map_block_start(em);
1174	if (em)
1175	btrfs_free_extent_map(em);
1176	} else {
1177	alloc_hint = btrfs_extent_map_block_start(em);
1178	btrfs_free_extent_map(em);
1179	}
1180	}
1181	read_unlock(&em_tree->lock);
1182
1183	return alloc_hint;
1184	}
1185
1186	/*
1187	* Handle COW for one range.
1188	*
1189	* @ins: The key representing the allocated range.
1190	* @file_offset: The file offset of the COW range
1191	* @num_bytes: The expected length of the COW range
1192	* The actually allocated length can be smaller than it.
1193	* @min_alloc_size: The minimal extent size.
1194	* @alloc_hint: The hint for the extent allocator.
1195	* @ret_alloc_size: The COW range handles by this function.
1196	*
1197	* Return 0 if everything is fine and update @ret_alloc_size updated. The
1198	* range is still locked, and caller should unlock the range after everything
1199	* is done or for error handling.
1200	*
1201	* Return <0 for error and @is updated for where the extra cleanup should
1202	* happen. The range [file_offset, file_offset + ret_alloc_size) will be
1203	* cleaned up by this function.
1204	*/
1205	static int cow_one_range(struct btrfs_inode inode, struct folio locked_folio,
1206	struct btrfs_key ins, struct extent_state *cached,
1207	u64 file_offset, u32 num_bytes, u32 min_alloc_size,
1208	u64 alloc_hint, u32 *ret_alloc_size)
1209	{
1210	struct btrfs_root *root = inode->root;
1211	struct btrfs_fs_info *fs_info = root->fs_info;
1212	struct btrfs_ordered_extent *ordered;
1213	struct btrfs_file_extent file_extent;
1214	struct extent_map *em;
1215	u32 cur_len = 0;
1216	u64 cur_end;
1217	int ret;
1218
1219	ret = btrfs_reserve_extent(root, num_bytes, num_bytes, min_alloc_size,
1220	0, alloc_hint, ins, true, true);
1221	if (ret < 0) {
1222	*ret_alloc_size = cur_len;
1223	return ret;
1224	}
1225
1226	cur_len = ins->offset;
1227	cur_end = file_offset + cur_len - 1;
1228
1229	file_extent.disk_bytenr = ins->objectid;
1230	file_extent.disk_num_bytes = ins->offset;
1231	file_extent.num_bytes = ins->offset;
1232	file_extent.ram_bytes = ins->offset;
1233	file_extent.offset = 0;
1234	file_extent.compression = BTRFS_COMPRESS_NONE;
1235
1236	/*
1237	* Locked range will be released either during error clean up (inside
1238	* this function or by the caller for previously successful ranges) or
1239	* after the whole range is finished.
1240	*/
1241	btrfs_lock_extent(&inode->io_tree, file_offset, cur_end, cached);
1242	em = btrfs_create_io_em(inode, file_offset, &file_extent, BTRFS_ORDERED_REGULAR);
1243	if (IS_ERR(em)) {
1244	ret = PTR_ERR(em);
1245	goto free_reserved;
1246	}
1247	btrfs_free_extent_map(em);
1248
1249	ordered = btrfs_alloc_ordered_extent(inode, file_offset, &file_extent,
1250	1U << BTRFS_ORDERED_REGULAR);
1251	if (IS_ERR(ordered)) {
1252	btrfs_drop_extent_map_range(inode, file_offset, cur_end, false);
1253	ret = PTR_ERR(ordered);
1254	goto free_reserved;
1255	}
1256
1257	if (btrfs_is_data_reloc_root(root)) {
1258	ret = btrfs_reloc_clone_csums(ordered);
1259
1260	/*
1261	* Only drop cache here, and process as normal.
1262	*
1263	* We must not allow extent_clear_unlock_delalloc() at
1264	* free_reserved label to free meta of this ordered extent, as
1265	* its meta should be freed by btrfs_finish_ordered_io().
1266	*
1267	* So we must continue until @start is increased to
1268	* skip current ordered extent.
1269	*/
1270	if (ret)
1271	btrfs_drop_extent_map_range(inode, file_offset,
1272	cur_end, false);
1273	}
1274	btrfs_put_ordered_extent(ordered);
1275	btrfs_dec_block_group_reservations(fs_info, ins->objectid);
1276	/*
1277	* Error handling for btrfs_reloc_clone_csums().
1278	*
1279	* Treat the range as finished, thus only clear EXTENT_LOCKED \| EXTENT_DELALLOC.
1280	* The accounting will be done by ordered extents.
1281	*/
1282	if (unlikely(ret < 0)) {
1283	btrfs_cleanup_ordered_extents(inode, file_offset, cur_len);
1284	extent_clear_unlock_delalloc(inode, file_offset, cur_end, locked_folio, cached,
1285	EXTENT_LOCKED \| EXTENT_DELALLOC,
1286	PAGE_UNLOCK \| PAGE_START_WRITEBACK \|
1287	PAGE_END_WRITEBACK);
1288	mapping_set_error(inode->vfs_inode.i_mapping, -EIO);
1289	}
1290	*ret_alloc_size = cur_len;
1291	return ret;
1292
1293	free_reserved:
1294	/*
1295	* If we have reserved an extent for the current range and failed to
1296	* create the respective extent map or ordered extent, it means that
1297	* when we reserved the extent we decremented the extent's size from
1298	* the data space_info's bytes_may_use counter and
1299	* incremented the space_info's bytes_reserved counter by the same
1300	* amount.
1301	*
1302	* We must make sure extent_clear_unlock_delalloc() does not try
1303	* to decrement again the data space_info's bytes_may_use counter, which
1304	* will be handled by btrfs_free_reserved_extent().
1305	*
1306	* Therefore we do not pass it the flag EXTENT_CLEAR_DATA_RESV, but only
1307	* EXTENT_CLEAR_META_RESV.
1308	*/
1309	extent_clear_unlock_delalloc(inode, file_offset, cur_end, locked_folio, cached,
1310	EXTENT_LOCKED \| EXTENT_DELALLOC \|
1311	EXTENT_DELALLOC_NEW \|
1312	EXTENT_DEFRAG \| EXTENT_CLEAR_META_RESV,
1313	PAGE_UNLOCK \| PAGE_START_WRITEBACK \|
1314	PAGE_END_WRITEBACK);
1315	btrfs_qgroup_free_data(inode, NULL, file_offset, cur_len, NULL);
1316	btrfs_dec_block_group_reservations(fs_info, ins->objectid);
1317	btrfs_free_reserved_extent(fs_info, ins->objectid, ins->offset, true);
1318	mapping_set_error(inode->vfs_inode.i_mapping, -EIO);
1319	*ret_alloc_size = cur_len;
1320	/*
1321	* We should not return -EAGAIN where it's a special return code for
1322	* zoned to catch btrfs_reserved_extent().
1323	*/
1324	ASSERT(ret != -EAGAIN);
1325	return ret;
1326	}
1327
1328	/*
1329	* when extent_io.c finds a delayed allocation range in the file,
1330	* the call backs end up in this code. The basic idea is to
1331	* allocate extents on disk for the range, and create ordered data structs
1332	* in ram to track those extents.
1333	*
1334	* locked_folio is the folio that writepage had locked already. We use
1335	* it to make sure we don't do extra locks or unlocks.
1336	*
1337	* When this function fails, it unlocks all folios except @locked_folio.
1338	*
1339	* When this function succeed and creates a normal extent, the folio locking
1340	* status depends on the passed in flags:
1341	*
1342	* - If COW_FILE_RANGE_KEEP_LOCKED flag is set, all folios are kept locked.
1343	* - Else all folios except for @locked_folio are unlocked.
1344	*
1345	* When a failure happens in the second or later iteration of the
1346	* while-loop, the ordered extents created in previous iterations are cleaned up.
1347	*/
1348	static noinline int cow_file_range(struct btrfs_inode *inode,
1349	struct folio *locked_folio, u64 start,
1350	u64 end, u64 *done_offset,
1351	unsigned long flags)
1352	{
1353	struct btrfs_root *root = inode->root;
1354	struct btrfs_fs_info *fs_info = root->fs_info;
1355	struct extent_state *cached = NULL;
1356	u64 alloc_hint = 0;
1357	u64 orig_start = start;
1358	u64 num_bytes;
1359	u32 min_alloc_size;
1360	u32 blocksize = fs_info->sectorsize;
1361	u32 cur_alloc_size = 0;
1362	struct btrfs_key ins;
1363	unsigned clear_bits;
1364	unsigned long page_ops;
1365	int ret = 0;
1366
1367	if (btrfs_is_shutdown(fs_info)) {
1368	ret = -EIO;
1369	goto out_unlock;
1370	}
1371
1372	if (btrfs_is_free_space_inode(inode)) {
1373	ret = -EINVAL;
1374	goto out_unlock;
1375	}
1376
1377	num_bytes = ALIGN(end - start + 1, blocksize);
1378	num_bytes = max(blocksize, num_bytes);
1379	ASSERT(num_bytes <= btrfs_super_total_bytes(fs_info->super_copy));
1380
1381	inode_should_defrag(inode, start, end, num_bytes, SZ_64K);
1382	alloc_hint = btrfs_get_extent_allocation_hint(inode, start, num_bytes);
1383
1384	/*
1385	* We're not doing compressed IO, don't unlock the first page (which
1386	* the caller expects to stay locked), don't clear any dirty bits and
1387	* don't set any writeback bits.
1388	*
1389	* Do set the Ordered (Private2) bit so we know this page was properly
1390	* setup for writepage.
1391	*/
1392	page_ops = ((flags & COW_FILE_RANGE_KEEP_LOCKED) ? 0 : PAGE_UNLOCK);
1393
1394	/*
1395	* Relocation relies on the relocated extents to have exactly the same
1396	* size as the original extents. Normally writeback for relocation data
1397	* extents follows a NOCOW path because relocation preallocates the
1398	* extents. However, due to an operation such as scrub turning a block
1399	* group to RO mode, it may fallback to COW mode, so we must make sure
1400	* an extent allocated during COW has exactly the requested size and can
1401	* not be split into smaller extents, otherwise relocation breaks and
1402	* fails during the stage where it updates the bytenr of file extent
1403	* items.
1404	*/
1405	if (btrfs_is_data_reloc_root(root))
1406	min_alloc_size = num_bytes;
1407	else
1408	min_alloc_size = fs_info->sectorsize;
1409
1410	while (num_bytes > 0) {
1411	ret = cow_one_range(inode, locked_folio, &ins, &cached, start,
1412	num_bytes, min_alloc_size, alloc_hint, &cur_alloc_size);
1413
1414	if (ret == -EAGAIN) {
1415	/*
1416	* cow_one_range() only returns -EAGAIN for zoned
1417	* file systems (from btrfs_reserve_extent()), which
1418	* is an indication that there are
1419	* no active zones to allocate from at the moment.
1420	*
1421	* If this is the first loop iteration, wait for at
1422	* least one zone to finish before retrying the
1423	* allocation. Otherwise ask the caller to write out
1424	* the already allocated blocks before coming back to
1425	* us, or return -ENOSPC if it can't handle retries.
1426	*/
1427	ASSERT(btrfs_is_zoned(fs_info));
1428	if (start == orig_start) {
1429	wait_on_bit_io(&inode->root->fs_info->flags,
1430	BTRFS_FS_NEED_ZONE_FINISH,
1431	TASK_UNINTERRUPTIBLE);
1432	continue;
1433	}
1434	if (done_offset) {
1435	/*
1436	* Move @end to the end of the processed range,
1437	* and exit the loop to unlock the processed extents.
1438	*/
1439	end = start - 1;
1440	ret = 0;
1441	break;
1442	}
1443	ret = -ENOSPC;
1444	}
1445	if (ret < 0)
1446	goto out_unlock;
1447
1448	/* We should not allocate an extent larger than requested.*/
1449	ASSERT(cur_alloc_size <= num_bytes);
1450
1451	num_bytes -= cur_alloc_size;
1452	alloc_hint = ins.objectid + ins.offset;
1453	start += cur_alloc_size;
1454	cur_alloc_size = 0;
1455	}
1456	extent_clear_unlock_delalloc(inode, orig_start, end, locked_folio, &cached,
1457	EXTENT_LOCKED \| EXTENT_DELALLOC, page_ops);
1458	if (done_offset)
1459	*done_offset = end;
1460	return ret;
1461
1462	out_unlock:
1463	/*
1464	* Now, we have three regions to clean up:
1465	*
1466	* \|-------(1)----\|---(2)---\|-------------(3)----------\|
1467	* `- orig_start `- start `- start + cur_alloc_size `- end
1468	*
1469	* We process each region below.
1470	*/
1471
1472	/*
1473	* For the range (1). We have already instantiated the ordered extents
1474	* for this region, thus we need to cleanup those ordered extents.
1475	* EXTENT_DELALLOC_NEW \| EXTENT_DEFRAG \| EXTENT_CLEAR_META_RESV
1476	* are also handled by the ordered extents cleanup.
1477	*
1478	* So here we only clear EXTENT_LOCKED and EXTENT_DELALLOC flag, and
1479	* finish the writeback of the involved folios, which will be never submitted.
1480	*/
1481	if (orig_start < start) {
1482	clear_bits = EXTENT_LOCKED \| EXTENT_DELALLOC;
1483	page_ops = PAGE_UNLOCK \| PAGE_START_WRITEBACK \| PAGE_END_WRITEBACK;
1484
1485	if (!locked_folio)
1486	mapping_set_error(inode->vfs_inode.i_mapping, ret);
1487
1488	btrfs_cleanup_ordered_extents(inode, orig_start, start - orig_start);
1489	extent_clear_unlock_delalloc(inode, orig_start, start - 1,
1490	locked_folio, NULL, clear_bits, page_ops);
1491	}
1492
1493	clear_bits = EXTENT_LOCKED \| EXTENT_DELALLOC \| EXTENT_DELALLOC_NEW \|
1494	EXTENT_DEFRAG \| EXTENT_CLEAR_META_RESV;
1495	page_ops = PAGE_UNLOCK \| PAGE_START_WRITEBACK \| PAGE_END_WRITEBACK;
1496
1497	/*
1498	* For the range (2) the error handling is done by cow_one_range() itself.
1499	* Nothing needs to be done.
1500	*
1501	* For the range (3). We never touched the region. In addition to the
1502	* clear_bits above, we add EXTENT_CLEAR_DATA_RESV to release the data
1503	* space_info's bytes_may_use counter, reserved in
1504	* btrfs_check_data_free_space().
1505	*/
1506	if (start + cur_alloc_size < end) {
1507	clear_bits \|= EXTENT_CLEAR_DATA_RESV;
1508	extent_clear_unlock_delalloc(inode, start + cur_alloc_size,
1509	end, locked_folio,
1510	&cached, clear_bits, page_ops);
1511	btrfs_qgroup_free_data(inode, NULL, start + cur_alloc_size,
1512	end - start - cur_alloc_size + 1, NULL);
1513	}
1514	btrfs_err(fs_info,
1515	"%s failed, root=%llu inode=%llu start=%llu len=%llu cur_offset=%llu cur_alloc_size=%u: %pe",
1516	__func__, btrfs_root_id(inode->root),
1517	btrfs_ino(inode), orig_start, end + 1 - orig_start,
1518	start, cur_alloc_size, ERR_PTR(ret));
1519	return ret;
1520	}
1521
1522	/*
1523	* Phase two of compressed writeback. This is the ordered portion of the code,
1524	* which only gets called in the order the work was queued. We walk all the
1525	* async extents created by compress_file_range and send them down to the disk.
1526	*
1527	* If called with @do_free == true then it'll try to finish the work and free
1528	* the work struct eventually.
1529	*/
1530	static noinline void submit_compressed_extents(struct btrfs_work *work, bool do_free)
1531	{
1532	struct async_chunk *async_chunk = container_of(work, struct async_chunk,
1533	work);
1534	struct btrfs_fs_info *fs_info = btrfs_work_owner(work);
1535	struct async_extent *async_extent;
1536	unsigned long nr_pages;
1537	u64 alloc_hint = 0;
1538
1539	if (do_free) {
1540	struct async_cow *async_cow;
1541
1542	btrfs_add_delayed_iput(async_chunk->inode);
1543	if (async_chunk->blkcg_css)
1544	css_put(async_chunk->blkcg_css);
1545
1546	async_cow = async_chunk->async_cow;
1547	if (atomic_dec_and_test(&async_cow->num_chunks))
1548	kvfree(async_cow);
1549	return;
1550	}
1551
1552	nr_pages = (async_chunk->end - async_chunk->start + PAGE_SIZE) >>
1553	PAGE_SHIFT;
1554
1555	while (!list_empty(&async_chunk->extents)) {
1556	async_extent = list_first_entry(&async_chunk->extents,
1557	struct async_extent, list);
1558	list_del(&async_extent->list);
1559	submit_one_async_extent(async_chunk, async_extent, &alloc_hint);
1560	}
1561
1562	/* atomic_sub_return implies a barrier */
1563	if (atomic_sub_return(nr_pages, &fs_info->async_delalloc_pages) <
1564	5 * SZ_1M)
1565	cond_wake_up_nomb(&fs_info->async_submit_wait);
1566	}
1567
1568	static bool run_delalloc_compressed(struct btrfs_inode *inode,
1569	struct folio *locked_folio, u64 start,
1570	u64 end, struct writeback_control *wbc)
1571	{
1572	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1573	struct cgroup_subsys_state *blkcg_css = wbc_blkcg_css(wbc);
1574	struct async_cow *ctx;
1575	struct async_chunk *async_chunk;
1576	unsigned long nr_pages;
1577	u64 num_chunks = DIV_ROUND_UP(end - start, BTRFS_COMPRESSION_CHUNK_SIZE);
1578	int i;
1579	unsigned nofs_flag;
1580	const blk_opf_t write_flags = wbc_to_write_flags(wbc);
1581
1582	nofs_flag = memalloc_nofs_save();
1583	ctx = kvmalloc_flex(*ctx, chunks, num_chunks);
1584	memalloc_nofs_restore(nofs_flag);
1585	if (!ctx)
1586	return false;
1587
1588	set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, &inode->runtime_flags);
1589
1590	async_chunk = ctx->chunks;
1591	atomic_set(&ctx->num_chunks, num_chunks);
1592
1593	for (i = 0; i < num_chunks; i++) {
1594	u64 cur_end = min(end, start + BTRFS_COMPRESSION_CHUNK_SIZE - 1);
1595
1596	/*
1597	* igrab is called higher up in the call chain, take only the
1598	* lightweight reference for the callback lifetime
1599	*/
1600	ihold(&inode->vfs_inode);
1601	async_chunk[i].async_cow = ctx;
1602	async_chunk[i].inode = inode;
1603	async_chunk[i].start = start;
1604	async_chunk[i].end = cur_end;
1605	async_chunk[i].write_flags = write_flags;
1606	INIT_LIST_HEAD(&async_chunk[i].extents);
1607
1608	/*
1609	* The locked_folio comes all the way from writepage and its
1610	* the original folio we were actually given. As we spread
1611	* this large delalloc region across multiple async_chunk
1612	* structs, only the first struct needs a pointer to
1613	* locked_folio.
1614	*
1615	* This way we don't need racey decisions about who is supposed
1616	* to unlock it.
1617	*/
1618	if (locked_folio) {
1619	/*
1620	* Depending on the compressibility, the pages might or
1621	* might not go through async. We want all of them to
1622	* be accounted against wbc once. Let's do it here
1623	* before the paths diverge. wbc accounting is used
1624	* only for foreign writeback detection and doesn't
1625	* need full accuracy. Just account the whole thing
1626	* against the first page.
1627	*/
1628	wbc_account_cgroup_owner(wbc, locked_folio,
1629	cur_end - start);
1630	async_chunk[i].locked_folio = locked_folio;
1631	locked_folio = NULL;
1632	} else {
1633	async_chunk[i].locked_folio = NULL;
1634	}
1635
1636	if (blkcg_css != blkcg_root_css) {
1637	css_get(blkcg_css);
1638	async_chunk[i].blkcg_css = blkcg_css;
1639	async_chunk[i].write_flags \|= REQ_BTRFS_CGROUP_PUNT;
1640	} else {
1641	async_chunk[i].blkcg_css = NULL;
1642	}
1643
1644	btrfs_init_work(&async_chunk[i].work, compress_file_range,
1645	submit_compressed_extents);
1646
1647	nr_pages = DIV_ROUND_UP(cur_end - start, PAGE_SIZE);
1648	atomic_add(nr_pages, &fs_info->async_delalloc_pages);
1649
1650	btrfs_queue_work(fs_info->delalloc_workers, &async_chunk[i].work);
1651
1652	start = cur_end + 1;
1653	}
1654	return true;
1655	}
1656
1657	/*
1658	* Run the delalloc range from start to end, and write back any dirty pages
1659	* covered by the range.
1660	*/
1661	static noinline int run_delalloc_cow(struct btrfs_inode *inode,
1662	struct folio *locked_folio, u64 start,
1663	u64 end, struct writeback_control *wbc,
1664	bool pages_dirty)
1665	{
1666	u64 done_offset = end;
1667	int ret;
1668
1669	while (start <= end) {
1670	ret = cow_file_range(inode, locked_folio, start, end,
1671	&done_offset, COW_FILE_RANGE_KEEP_LOCKED);
1672	if (ret)
1673	return ret;
1674	extent_write_locked_range(&inode->vfs_inode, locked_folio,
1675	start, done_offset, wbc, pages_dirty);
1676	start = done_offset + 1;
1677	}
1678
1679	return 1;
1680	}
1681
1682	static int fallback_to_cow(struct btrfs_inode *inode,
1683	struct folio *locked_folio, const u64 start,
1684	const u64 end)
1685	{
1686	const bool is_space_ino = btrfs_is_free_space_inode(inode);
1687	const bool is_reloc_ino = btrfs_is_data_reloc_root(inode->root);
1688	const u64 range_bytes = end + 1 - start;
1689	struct extent_io_tree *io_tree = &inode->io_tree;
1690	struct extent_state *cached_state = NULL;
1691	u64 range_start = start;
1692	u64 count;
1693	int ret;
1694
1695	/*
1696	* If EXTENT_NORESERVE is set it means that when the buffered write was
1697	* made we had not enough available data space and therefore we did not
1698	* reserve data space for it, since we though we could do NOCOW for the
1699	* respective file range (either there is prealloc extent or the inode
1700	* has the NOCOW bit set).
1701	*
1702	* However when we need to fallback to COW mode (because for example the
1703	* block group for the corresponding extent was turned to RO mode by a
1704	* scrub or relocation) we need to do the following:
1705	*
1706	* 1) We increment the bytes_may_use counter of the data space info.
1707	* If COW succeeds, it allocates a new data extent and after doing
1708	* that it decrements the space info's bytes_may_use counter and
1709	* increments its bytes_reserved counter by the same amount (we do
1710	* this at btrfs_add_reserved_bytes()). So we need to increment the
1711	* bytes_may_use counter to compensate (when space is reserved at
1712	* buffered write time, the bytes_may_use counter is incremented);
1713	*
1714	* 2) We clear the EXTENT_NORESERVE bit from the range. We do this so
1715	* that if the COW path fails for any reason, it decrements (through
1716	* extent_clear_unlock_delalloc()) the bytes_may_use counter of the
1717	* data space info, which we incremented in the step above.
1718	*
1719	* If we need to fallback to cow and the inode corresponds to a free
1720	* space cache inode or an inode of the data relocation tree, we must
1721	* also increment bytes_may_use of the data space_info for the same
1722	* reason. Space caches and relocated data extents always get a prealloc
1723	* extent for them, however scrub or balance may have set the block
1724	* group that contains that extent to RO mode and therefore force COW
1725	* when starting writeback.
1726	*/
1727	btrfs_lock_extent(io_tree, start, end, &cached_state);
1728	count = btrfs_count_range_bits(io_tree, &range_start, end, range_bytes,
1729	EXTENT_NORESERVE, false, NULL);
1730	if (count > 0 \|\| is_space_ino \|\| is_reloc_ino) {
1731	u64 bytes = count;
1732	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1733	struct btrfs_space_info *sinfo = fs_info->data_sinfo;
1734
1735	if (is_space_ino \|\| is_reloc_ino)
1736	bytes = range_bytes;
1737
1738	spin_lock(&sinfo->lock);
1739	btrfs_space_info_update_bytes_may_use(sinfo, bytes);
1740	spin_unlock(&sinfo->lock);
1741
1742	if (count > 0)
1743	btrfs_clear_extent_bit(io_tree, start, end, EXTENT_NORESERVE,
1744	&cached_state);
1745	}
1746	btrfs_unlock_extent(io_tree, start, end, &cached_state);
1747
1748	/*
1749	* Don't try to create inline extents, as a mix of inline extent that
1750	* is written out and unlocked directly and a normal NOCOW extent
1751	* doesn't work.
1752	*
1753	* And here we do not unlock the folio after a successful run.
1754	* The folios will be unlocked after everything is finished, or by error handling.
1755	*
1756	* This is to ensure error handling won't need to clear dirty/ordered flags without
1757	* a locked folio, which can race with writeback.
1758	*/
1759	ret = cow_file_range(inode, locked_folio, start, end, NULL,
1760	COW_FILE_RANGE_KEEP_LOCKED);
1761	ASSERT(ret != 1);
1762	return ret;
1763	}
1764
1765	struct can_nocow_file_extent_args {
1766	/* Input fields. */
1767
1768	/* Start file offset of the range we want to NOCOW. */
1769	u64 start;
1770	/* End file offset (inclusive) of the range we want to NOCOW. */
1771	u64 end;
1772	bool writeback_path;
1773	/*
1774	* Free the path passed to can_nocow_file_extent() once it's not needed
1775	* anymore.
1776	*/
1777	bool free_path;
1778
1779	/*
1780	* Output fields. Only set when can_nocow_file_extent() returns 1.
1781	* The expected file extent for the NOCOW write.
1782	*/
1783	struct btrfs_file_extent file_extent;
1784	};
1785
1786	/*
1787	* Check if we can NOCOW the file extent that the path points to.
1788	* This function may return with the path released, so the caller should check
1789	* if path->nodes[0] is NULL or not if it needs to use the path afterwards.
1790	*
1791	* Returns: < 0 on error
1792	* 0 if we can not NOCOW
1793	* 1 if we can NOCOW
1794	*/
1795	static int can_nocow_file_extent(struct btrfs_path *path,
1796	struct btrfs_key *key,
1797	struct btrfs_inode *inode,
1798	struct can_nocow_file_extent_args *args)
1799	{
1800	const bool is_freespace_inode = btrfs_is_free_space_inode(inode);
1801	struct extent_buffer *leaf = path->nodes[0];
1802	struct btrfs_root *root = inode->root;
1803	struct btrfs_file_extent_item *fi;
1804	struct btrfs_root *csum_root;
1805	u64 io_start;
1806	u64 extent_end;
1807	u8 extent_type;
1808	int can_nocow = 0;
1809	int ret = 0;
1810	bool nowait = path->nowait;
1811
1812	/* If there are pending snapshots for this root, we must do COW. */
1813	if (args->writeback_path && !is_freespace_inode &&
1814	atomic_read(&root->snapshot_force_cow))
1815	goto out;
1816
1817	fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
1818	extent_type = btrfs_file_extent_type(leaf, fi);
1819
1820	if (extent_type == BTRFS_FILE_EXTENT_INLINE)
1821	goto out;
1822
1823	if (!(inode->flags & BTRFS_INODE_NODATACOW) &&
1824	extent_type == BTRFS_FILE_EXTENT_REG)
1825	goto out;
1826
1827	/*
1828	* If the extent was created before the generation where the last snapshot
1829	* for its subvolume was created, then this implies the extent is shared,
1830	* hence we must COW.
1831	*/
1832	if (btrfs_file_extent_generation(leaf, fi) <=
1833	btrfs_root_last_snapshot(&root->root_item))
1834	goto out;
1835
1836	/* An explicit hole, must COW. */
1837	if (btrfs_file_extent_disk_bytenr(leaf, fi) == 0)
1838	goto out;
1839
1840	/* Compressed/encrypted/encoded extents must be COWed. */
1841	if (btrfs_file_extent_compression(leaf, fi) \|\|
1842	btrfs_file_extent_encryption(leaf, fi) \|\|
1843	btrfs_file_extent_other_encoding(leaf, fi))
1844	goto out;
1845
1846	extent_end = btrfs_file_extent_end(path);
1847
1848	args->file_extent.disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
1849	args->file_extent.disk_num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
1850	args->file_extent.ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
1851	args->file_extent.offset = btrfs_file_extent_offset(leaf, fi);
1852	args->file_extent.compression = btrfs_file_extent_compression(leaf, fi);
1853
1854	/*
1855	* The following checks can be expensive, as they need to take other
1856	* locks and do btree or rbtree searches, so release the path to avoid
1857	* blocking other tasks for too long.
1858	*/
1859	btrfs_release_path(path);
1860
1861	ret = btrfs_cross_ref_exist(inode, key->offset - args->file_extent.offset,
1862	args->file_extent.disk_bytenr, path);
1863	WARN_ON_ONCE(ret > 0 && is_freespace_inode);
1864	if (ret != 0)
1865	goto out;
1866
1867	if (args->free_path) {
1868	/*
1869	* We don't need the path anymore, plus through the
1870	* btrfs_lookup_csums_list() call below we will end up allocating
1871	* another path. So free the path to avoid unnecessary extra
1872	* memory usage.
1873	*/
1874	btrfs_free_path(path);
1875	path = NULL;
1876	}
1877
1878	args->file_extent.num_bytes = min(args->end + 1, extent_end) - args->start;
1879	args->file_extent.offset += args->start - key->offset;
1880	io_start = args->file_extent.disk_bytenr + args->file_extent.offset;
1881
1882	/*
1883	* Force COW if csums exist in the range. This ensures that csums for a
1884	* given extent are either valid or do not exist.
1885	*/
1886
1887	csum_root = btrfs_csum_root(root->fs_info, io_start);
1888	if (unlikely(!csum_root)) {
1889	btrfs_err(root->fs_info,
1890	"missing csum root for extent at bytenr %llu", io_start);
1891	ret = -EUCLEAN;
1892	goto out;
1893	}
1894
1895	ret = btrfs_lookup_csums_list(csum_root, io_start,
1896	io_start + args->file_extent.num_bytes - 1,
1897	NULL, nowait);
1898	WARN_ON_ONCE(ret > 0 && is_freespace_inode);
1899	if (ret != 0)
1900	goto out;
1901
1902	can_nocow = 1;
1903	out:
1904	if (args->free_path && path)
1905	btrfs_free_path(path);
1906
1907	return ret < 0 ? ret : can_nocow;
1908	}
1909
1910	static int nocow_one_range(struct btrfs_inode inode, struct folio locked_folio,
1911	struct extent_state **cached,
1912	struct can_nocow_file_extent_args *nocow_args,
1913	u64 file_pos, bool is_prealloc)
1914	{
1915	struct btrfs_ordered_extent *ordered;
1916	const u64 len = nocow_args->file_extent.num_bytes;
1917	const u64 end = file_pos + len - 1;
1918	int ret = 0;
1919
1920	btrfs_lock_extent(&inode->io_tree, file_pos, end, cached);
1921
1922	if (is_prealloc) {
1923	struct extent_map *em;
1924
1925	em = btrfs_create_io_em(inode, file_pos, &nocow_args->file_extent,
1926	BTRFS_ORDERED_PREALLOC);
1927	if (IS_ERR(em)) {
1928	ret = PTR_ERR(em);
1929	goto error;
1930	}
1931	btrfs_free_extent_map(em);
1932	}
1933
1934	ordered = btrfs_alloc_ordered_extent(inode, file_pos, &nocow_args->file_extent,
1935	is_prealloc
1936	? (1U << BTRFS_ORDERED_PREALLOC)
1937	: (1U << BTRFS_ORDERED_NOCOW));
1938	if (IS_ERR(ordered)) {
1939	if (is_prealloc)
1940	btrfs_drop_extent_map_range(inode, file_pos, end, false);
1941	ret = PTR_ERR(ordered);
1942	goto error;
1943	}
1944
1945	if (btrfs_is_data_reloc_root(inode->root))
1946	/*
1947	* Errors are handled later, as we must prevent
1948	* extent_clear_unlock_delalloc() in error handler from freeing
1949	* metadata of the created ordered extent.
1950	*/
1951	ret = btrfs_reloc_clone_csums(ordered);
1952	btrfs_put_ordered_extent(ordered);
1953
1954	if (ret < 0)
1955	goto error;
1956	extent_clear_unlock_delalloc(inode, file_pos, end, locked_folio, cached,
1957	EXTENT_LOCKED \| EXTENT_DELALLOC \|
1958	EXTENT_CLEAR_DATA_RESV, 0);
1959	return ret;
1960
1961	error:
1962	btrfs_cleanup_ordered_extents(inode, file_pos, len);
1963	extent_clear_unlock_delalloc(inode, file_pos, end, locked_folio, cached,
1964	EXTENT_LOCKED \| EXTENT_DELALLOC \|
1965	EXTENT_CLEAR_DATA_RESV,
1966	PAGE_UNLOCK \| PAGE_START_WRITEBACK \|
1967	PAGE_END_WRITEBACK);
1968	btrfs_err(inode->root->fs_info,
1969	"%s failed, root=%lld inode=%llu start=%llu len=%llu: %pe",
1970	__func__, btrfs_root_id(inode->root), btrfs_ino(inode),
1971	file_pos, len, ERR_PTR(ret));
1972	return ret;
1973	}
1974
1975	/*
1976	* When nocow writeback calls back. This checks for snapshots or COW copies
1977	* of the extents that exist in the file, and COWs the file as required.
1978	*
1979	* If no cow copies or snapshots exist, we write directly to the existing
1980	* blocks on disk
1981	*/
1982	static noinline int run_delalloc_nocow(struct btrfs_inode *inode,
1983	struct folio *locked_folio,
1984	const u64 start, const u64 end)
1985	{
1986	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1987	struct btrfs_root *root = inode->root;
1988	struct btrfs_path *path = NULL;
1989	u64 cow_start = (u64)-1;
1990	/*
1991	* If not 0, represents the inclusive end of the last fallback_to_cow()
1992	* range. Only for error handling.
1993	*
1994	* The same for nocow_end, it's to avoid double cleaning up the range
1995	* already cleaned by nocow_one_range().
1996	*/
1997	u64 cow_end = 0;
1998	u64 nocow_end = 0;
1999	u64 cur_offset = start;
2000	int ret;
2001	bool check_prev = true;
2002	u64 ino = btrfs_ino(inode);
2003	struct can_nocow_file_extent_args nocow_args = { 0 };
2004	/* The range that has ordered extent(s). */
2005	u64 oe_cleanup_start;
2006	u64 oe_cleanup_len = 0;
2007	/* The range that is untouched. */
2008	u64 untouched_start;
2009	u64 untouched_len = 0;
2010
2011	/*
2012	* Normally on a zoned device we're only doing COW writes, but in case
2013	* of relocation on a zoned filesystem serializes I/O so that we're only
2014	* writing sequentially and can end up here as well.
2015	*/
2016	ASSERT(!btrfs_is_zoned(fs_info) \|\| btrfs_is_data_reloc_root(root));
2017
2018	if (btrfs_is_shutdown(fs_info)) {
2019	ret = -EIO;
2020	goto error;
2021	}
2022	path = btrfs_alloc_path();
2023	if (!path) {
2024	ret = -ENOMEM;
2025	goto error;
2026	}
2027
2028	nocow_args.end = end;
2029	nocow_args.writeback_path = true;
2030
2031	while (cur_offset <= end) {
2032	struct btrfs_block_group *nocow_bg = NULL;
2033	struct btrfs_key found_key;
2034	struct btrfs_file_extent_item *fi;
2035	struct extent_buffer *leaf;
2036	struct extent_state *cached_state = NULL;
2037	u64 extent_end;
2038	int extent_type;
2039
2040	ret = btrfs_lookup_file_extent(NULL, root, path, ino,
2041	cur_offset, 0);
2042	if (ret < 0)
2043	goto error;
2044
2045	/*
2046	* If there is no extent for our range when doing the initial
2047	* search, then go back to the previous slot as it will be the
2048	* one containing the search offset
2049	*/
2050	if (ret > 0 && path->slots[0] > 0 && check_prev) {
2051	leaf = path->nodes[0];
2052	btrfs_item_key_to_cpu(leaf, &found_key,
2053	path->slots[0] - 1);
2054	if (found_key.objectid == ino &&
2055	found_key.type == BTRFS_EXTENT_DATA_KEY)
2056	path->slots[0]--;
2057	}
2058	check_prev = false;
2059	next_slot:
2060	/* Go to next leaf if we have exhausted the current one */
2061	leaf = path->nodes[0];
2062	if (path->slots[0] >= btrfs_header_nritems(leaf)) {
2063	ret = btrfs_next_leaf(root, path);
2064	if (ret < 0)
2065	goto error;
2066	if (ret > 0)
2067	break;
2068	leaf = path->nodes[0];
2069	}
2070
2071	btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
2072
2073	/* Didn't find anything for our INO */
2074	if (found_key.objectid > ino)
2075	break;
2076	/*
2077	* Keep searching until we find an EXTENT_ITEM or there are no
2078	* more extents for this inode
2079	*/
2080	if (WARN_ON_ONCE(found_key.objectid < ino) \|\|
2081	found_key.type < BTRFS_EXTENT_DATA_KEY) {
2082	path->slots[0]++;
2083	goto next_slot;
2084	}
2085
2086	/* Found key is not EXTENT_DATA_KEY or starts after req range */
2087	if (found_key.type > BTRFS_EXTENT_DATA_KEY \|\|
2088	found_key.offset > end)
2089	break;
2090
2091	/*
2092	* If the found extent starts after requested offset, then
2093	* adjust cur_offset to be right before this extent begins.
2094	*/
2095	if (found_key.offset > cur_offset) {
2096	if (cow_start == (u64)-1)
2097	cow_start = cur_offset;
2098	cur_offset = found_key.offset;
2099	goto next_slot;
2100	}
2101
2102	/*
2103	* Found extent which begins before our range and potentially
2104	* intersect it
2105	*/
2106	fi = btrfs_item_ptr(leaf, path->slots[0],
2107	struct btrfs_file_extent_item);
2108	extent_type = btrfs_file_extent_type(leaf, fi);
2109	/* If this is triggered then we have a memory corruption. */
2110	ASSERT(extent_type < BTRFS_NR_FILE_EXTENT_TYPES);
2111	if (WARN_ON(extent_type >= BTRFS_NR_FILE_EXTENT_TYPES)) {
2112	ret = -EUCLEAN;
2113	goto error;
2114	}
2115	extent_end = btrfs_file_extent_end(path);
2116
2117	/*
2118	* If the extent we got ends before our current offset, skip to
2119	* the next extent.
2120	*/
2121	if (extent_end <= cur_offset) {
2122	path->slots[0]++;
2123	goto next_slot;
2124	}
2125
2126	nocow_args.start = cur_offset;
2127	ret = can_nocow_file_extent(path, &found_key, inode, &nocow_args);
2128	if (ret < 0)
2129	goto error;
2130	if (ret == 0)
2131	goto must_cow;
2132
2133	ret = 0;
2134	nocow_bg = btrfs_inc_nocow_writers(fs_info,
2135	nocow_args.file_extent.disk_bytenr +
2136	nocow_args.file_extent.offset);
2137	if (!nocow_bg) {
2138	must_cow:
2139	/*
2140	* If we can't perform NOCOW writeback for the range,
2141	* then record the beginning of the range that needs to
2142	* be COWed. It will be written out before the next
2143	* NOCOW range if we find one, or when exiting this
2144	* loop.
2145	*/
2146	if (cow_start == (u64)-1)
2147	cow_start = cur_offset;
2148	cur_offset = extent_end;
2149	if (cur_offset > end)
2150	break;
2151	if (!path->nodes[0])
2152	continue;
2153	path->slots[0]++;
2154	goto next_slot;
2155	}
2156
2157	/*
2158	* COW range from cow_start to found_key.offset - 1. As the key
2159	* will contain the beginning of the first extent that can be
2160	* NOCOW, following one which needs to be COW'ed
2161	*/
2162	if (cow_start != (u64)-1) {
2163	ret = fallback_to_cow(inode, locked_folio, cow_start,
2164	found_key.offset - 1);
2165	if (ret) {
2166	cow_end = found_key.offset - 1;
2167	btrfs_dec_nocow_writers(nocow_bg);
2168	goto error;
2169	}
2170	cow_start = (u64)-1;
2171	}
2172
2173	ret = nocow_one_range(inode, locked_folio, &cached_state,
2174	&nocow_args, cur_offset,
2175	extent_type == BTRFS_FILE_EXTENT_PREALLOC);
2176	btrfs_dec_nocow_writers(nocow_bg);
2177	if (ret < 0) {
2178	nocow_end = cur_offset + nocow_args.file_extent.num_bytes - 1;
2179	goto error;
2180	}
2181	cur_offset = extent_end;
2182	}
2183	btrfs_release_path(path);
2184
2185	if (cur_offset <= end && cow_start == (u64)-1)
2186	cow_start = cur_offset;
2187
2188	if (cow_start != (u64)-1) {
2189	ret = fallback_to_cow(inode, locked_folio, cow_start, end);
2190	if (ret) {
2191	cow_end = end;
2192	goto error;
2193	}
2194	cow_start = (u64)-1;
2195	}
2196
2197	/*
2198	* Everything is finished without an error, can unlock the folios now.
2199	*
2200	* No need to touch the io tree range nor set folio ordered flag, as
2201	* fallback_to_cow() and nocow_one_range() have already handled them.
2202	*/
2203	extent_clear_unlock_delalloc(inode, start, end, locked_folio, NULL, 0, PAGE_UNLOCK);
2204
2205	btrfs_free_path(path);
2206	return 0;
2207
2208	error:
2209	if (cow_start == (u64)-1) {
2210	/*
2211	* case a)
2212	* start cur_offset end
2213	* \| OE cleanup \| Untouched \|
2214	*
2215	* We finished a fallback_to_cow() or nocow_one_range() call,
2216	* but failed to check the next range.
2217	*
2218	* or
2219	* start cur_offset nocow_end end
2220	* \| OE cleanup \| Skip \| Untouched \|
2221	*
2222	* nocow_one_range() failed, the range [cur_offset, nocow_end] is
2223	* already cleaned up.
2224	*/
2225	oe_cleanup_start = start;
2226	oe_cleanup_len = cur_offset - start;
2227	if (nocow_end)
2228	untouched_start = nocow_end + 1;
2229	else
2230	untouched_start = cur_offset;
2231	untouched_len = end + 1 - untouched_start;
2232	} else if (cow_start != (u64)-1 && cow_end == 0) {
2233	/*
2234	* case b)
2235	* start cow_start cur_offset end
2236	* \| OE cleanup \| Untouched \|
2237	*
2238	* We got a range that needs COW, but before we hit the next NOCOW range,
2239	* thus [cow_start, cur_offset) doesn't yet have any OE.
2240	*/
2241	oe_cleanup_start = start;
2242	oe_cleanup_len = cow_start - start;
2243	untouched_start = cow_start;
2244	untouched_len = end + 1 - untouched_start;
2245	} else {
2246	/*
2247	* case c)
2248	* start cow_start cow_end end
2249	* \| OE cleanup \| Skip \| Untouched \|
2250	*
2251	* fallback_to_cow() failed, and fallback_to_cow() will do the
2252	* cleanup for its range, we shouldn't touch the range
2253	* [cow_start, cow_end].
2254	*/
2255	ASSERT(cow_start != (u64)-1 && cow_end != 0);
2256	oe_cleanup_start = start;
2257	oe_cleanup_len = cow_start - start;
2258	untouched_start = cow_end + 1;
2259	untouched_len = end + 1 - untouched_start;
2260	}
2261
2262	if (oe_cleanup_len) {
2263	const u64 oe_cleanup_end = oe_cleanup_start + oe_cleanup_len - 1;
2264	btrfs_cleanup_ordered_extents(inode, oe_cleanup_start, oe_cleanup_len);
2265	extent_clear_unlock_delalloc(inode, oe_cleanup_start, oe_cleanup_end,
2266	locked_folio, NULL,
2267	EXTENT_LOCKED \| EXTENT_DELALLOC,
2268	PAGE_UNLOCK \| PAGE_START_WRITEBACK \|
2269	PAGE_END_WRITEBACK);
2270	}
2271
2272	if (untouched_len) {
2273	struct extent_state *cached = NULL;
2274	const u64 untouched_end = untouched_start + untouched_len - 1;
2275
2276	/*
2277	* We need to lock the extent here because we're clearing DELALLOC and
2278	* we're not locked at this point.
2279	*/
2280	btrfs_lock_extent(&inode->io_tree, untouched_start, untouched_end, &cached);
2281	extent_clear_unlock_delalloc(inode, untouched_start, untouched_end,
2282	locked_folio, &cached,
2283	EXTENT_LOCKED \| EXTENT_DELALLOC \|
2284	EXTENT_DEFRAG \|
2285	EXTENT_DO_ACCOUNTING, PAGE_UNLOCK \|
2286	PAGE_START_WRITEBACK \|
2287	PAGE_END_WRITEBACK);
2288	btrfs_qgroup_free_data(inode, NULL, untouched_start, untouched_len, NULL);
2289	}
2290	btrfs_free_path(path);
2291	btrfs_err(fs_info,
2292	"%s failed, root=%llu inode=%llu start=%llu len=%llu cur_offset=%llu oe_cleanup=%llu oe_cleanup_len=%llu untouched_start=%llu untouched_len=%llu: %pe",
2293	__func__, btrfs_root_id(inode->root), btrfs_ino(inode),
2294	start, end + 1 - start, cur_offset, oe_cleanup_start, oe_cleanup_len,
2295	untouched_start, untouched_len, ERR_PTR(ret));
2296	return ret;
2297	}
2298
2299	static bool should_nocow(struct btrfs_inode *inode, u64 start, u64 end)
2300	{
2301	if (inode->flags & (BTRFS_INODE_NODATACOW \| BTRFS_INODE_PREALLOC)) {
2302	if (data_race(inode->defrag_bytes) &&
2303	btrfs_test_range_bit_exists(&inode->io_tree, start, end, EXTENT_DEFRAG))
2304	return false;
2305	return true;
2306	}
2307	return false;
2308	}
2309
2310	/*
2311	* Return 0 if an inlined extent is created successfully.
2312	* Return <0 if critical error happened.
2313	* Return >0 if an inline extent can not be created.
2314	*/
2315	static int run_delalloc_inline(struct btrfs_inode inode, struct folio locked_folio)
2316	{
2317	struct btrfs_fs_info *fs_info = inode->root->fs_info;
2318	struct compressed_bio *cb = NULL;
2319	struct extent_state *cached = NULL;
2320	const u64 i_size = i_size_read(&inode->vfs_inode);
2321	const u32 blocksize = fs_info->sectorsize;
2322	int compress_type = fs_info->compress_type;
2323	int compress_level = fs_info->compress_level;
2324	u32 compressed_size = 0;
2325	int ret;
2326
2327	ASSERT(folio_pos(locked_folio) == 0);
2328	/*
2329	* If an mmap writer could modify the folio while we copy it into an
2330	* inline extent we might see only part of their modification then
2331	* wrongly mark it clean again after copying, losing that write. So the
2332	* folio must be write protected here.
2333	*/
2334	btrfs_check_folio_write_protected(locked_folio);
2335
2336	if (btrfs_inode_can_compress(inode) &&
2337	inode_need_compress(inode, 0, blocksize - 1, true)) {
2338	if (inode->defrag_compress > 0 &&
2339	inode->defrag_compress < BTRFS_NR_COMPRESS_TYPES) {
2340	compress_type = inode->defrag_compress;
2341	compress_level = inode->defrag_compress_level;
2342	} else if (inode->prop_compress) {
2343	compress_type = inode->prop_compress;
2344	}
2345	/*
2346	* We need to pass blocksize and not i_size, otherwise we can't
2347	* create compressed inline extents for data smaller than sector
2348	* size with lzo.
2349	*/
2350	cb = btrfs_compress_bio(inode, 0, blocksize, compress_type, compress_level, 0);
2351	if (IS_ERR(cb)) {
2352	cb = NULL;
2353	/* Just fall back to non-compressed case. */
2354	} else {
2355	compressed_size = cb->bbio.bio.bi_iter.bi_size;
2356	/*
2357	* If we did not save space, it's pointless and wasteful
2358	* to have an inline compressed extent, so fallback to
2359	* an uncompressed inline extent.
2360	*/
2361	if (compressed_size >= i_size) {
2362	cleanup_compressed_bio(cb);
2363	cb = NULL;
2364	compressed_size = 0;
2365	}
2366	}
2367	}
2368	if (!can_cow_file_range_inline(inode, 0, i_size, compressed_size)) {
2369	if (cb)
2370	cleanup_compressed_bio(cb);
2371	return 1;
2372	}
2373
2374	btrfs_lock_extent(&inode->io_tree, 0, blocksize - 1, &cached);
2375	if (cb) {
2376	ret = __cow_file_range_inline(inode, i_size, compressed_size, compress_type,
2377	bio_first_folio_all(&cb->bbio.bio), false);
2378	cleanup_compressed_bio(cb);
2379	cb = NULL;
2380	} else {
2381	ret = __cow_file_range_inline(inode, i_size, 0, BTRFS_COMPRESS_NONE,
2382	NULL, false);
2383	}
2384	/*
2385	* We failed to insert inline extent due to lack of meta space.
2386	* Just unlock the extent io range and fallback to regular COW/NOCOW path.
2387	*/
2388	if (ret > 0) {
2389	btrfs_unlock_extent(&inode->io_tree, 0, blocksize - 1, &cached);
2390	return ret;
2391	}
2392
2393	/*
2394	* In the successful case (ret == 0 here), btrfs_run_delalloc_range()
2395	* will return 1.
2396	*
2397	* Quite a bit further up the callstack in extent_writepage(), ret == 1
2398	* is treated as a short circuited success and does not unlock the folio,
2399	* so we must do it here.
2400	*
2401	* For failure case, the @locked_folio does get unlocked by
2402	* btrfs_folio_end_lock_bitmap(), so we must not unlock it here.
2403	*
2404	* So if ret == 0, we let extent_clear_unlock_delalloc() to unlock the
2405	* folio by passing NULL as @locked_folio.
2406	* Otherwise pass @locked_folio as usual.
2407	*/
2408	if (ret == 0)
2409	locked_folio = NULL;
2410	extent_clear_unlock_delalloc(inode, 0, blocksize - 1, locked_folio, &cached,
2411	EXTENT_DELALLOC \| EXTENT_DELALLOC_NEW \| EXTENT_DEFRAG \|
2412	EXTENT_DO_ACCOUNTING \| EXTENT_LOCKED,
2413	PAGE_UNLOCK \| PAGE_START_WRITEBACK \| PAGE_END_WRITEBACK);
2414	return ret;
2415	}
2416
2417	/*
2418	* Function to process delayed allocation (create CoW) for ranges which are
2419	* being touched for the first time.
2420	*/
2421	int btrfs_run_delalloc_range(struct btrfs_inode inode, struct folio locked_folio,
2422	u64 start, u64 end, struct writeback_control *wbc)
2423	{
2424	const bool zoned = btrfs_is_zoned(inode->root->fs_info);
2425
2426	/*
2427	* The range must cover part of the @locked_folio, or a return of 1
2428	* can confuse the caller.
2429	*/
2430	ASSERT(!(end <= folio_pos(locked_folio) \|\|
2431	start >= folio_next_pos(locked_folio)));
2432
2433	if (start == 0 && end + 1 <= inode->root->fs_info->sectorsize &&
2434	end + 1 >= inode->disk_i_size) {
2435	int ret;
2436
2437	ret = run_delalloc_inline(inode, locked_folio);
2438	if (ret < 0)
2439	return ret;
2440	if (ret == 0)
2441	return 1;
2442	/*
2443	* Continue regular handling if we can not create an
2444	* inlined extent.
2445	*/
2446	}
2447
2448	if (should_nocow(inode, start, end))
2449	return run_delalloc_nocow(inode, locked_folio, start, end);
2450
2451	if (btrfs_inode_can_compress(inode) &&
2452	inode_need_compress(inode, start, end, false) &&
2453	run_delalloc_compressed(inode, locked_folio, start, end, wbc))
2454	return 1;
2455
2456	if (zoned)
2457	return run_delalloc_cow(inode, locked_folio, start, end, wbc, true);
2458	else
2459	return cow_file_range(inode, locked_folio, start, end, NULL, 0);
2460	}
2461
2462	void btrfs_split_delalloc_extent(struct btrfs_inode *inode,
2463	struct extent_state *orig, u64 split)
2464	{
2465	struct btrfs_fs_info *fs_info = inode->root->fs_info;
2466	u64 size;
2467
2468	lockdep_assert_held(&inode->io_tree.lock);
2469
2470	/* not delalloc, ignore it */
2471	if (!(orig->state & EXTENT_DELALLOC))
2472	return;
2473
2474	size = orig->end - orig->start + 1;
2475	if (size > fs_info->max_extent_size) {
2476	u32 num_extents;
2477	u64 new_size;
2478
2479	/*
2480	* See the explanation in btrfs_merge_delalloc_extent, the same
2481	* applies here, just in reverse.
2482	*/
2483	new_size = orig->end - split + 1;
2484	num_extents = count_max_extents(fs_info, new_size);
2485	new_size = split - orig->start;
2486	num_extents += count_max_extents(fs_info, new_size);
2487	if (count_max_extents(fs_info, size) >= num_extents)
2488	return;
2489	}
2490
2491	spin_lock(&inode->lock);
2492	btrfs_mod_outstanding_extents(inode, 1);
2493	spin_unlock(&inode->lock);
2494	}
2495
2496	/*
2497	* Handle merged delayed allocation extents so we can keep track of new extents
2498	* that are just merged onto old extents, such as when we are doing sequential
2499	* writes, so we can properly account for the metadata space we'll need.
2500	*/
2501	void btrfs_merge_delalloc_extent(struct btrfs_inode inode, struct extent_state new,
2502	struct extent_state *other)
2503	{
2504	struct btrfs_fs_info *fs_info = inode->root->fs_info;
2505	u64 new_size, old_size;
2506	u32 num_extents;
2507
2508	lockdep_assert_held(&inode->io_tree.lock);
2509
2510	/* not delalloc, ignore it */
2511	if (!(other->state & EXTENT_DELALLOC))
2512	return;
2513
2514	if (new->start > other->start)
2515	new_size = new->end - other->start + 1;
2516	else
2517	new_size = other->end - new->start + 1;
2518
2519	/* we're not bigger than the max, unreserve the space and go */
2520	if (new_size <= fs_info->max_extent_size) {
2521	spin_lock(&inode->lock);
2522	btrfs_mod_outstanding_extents(inode, -1);
2523	spin_unlock(&inode->lock);
2524	return;
2525	}
2526
2527	/*
2528	* We have to add up either side to figure out how many extents were
2529	* accounted for before we merged into one big extent. If the number of
2530	* extents we accounted for is <= the amount we need for the new range
2531	* then we can return, otherwise drop. Think of it like this
2532	*
2533	* [ 4k][MAX_SIZE]
2534	*
2535	* So we've grown the extent by a MAX_SIZE extent, this would mean we
2536	* need 2 outstanding extents, on one side we have 1 and the other side
2537	* we have 1 so they are == and we can return. But in this case
2538	*
2539	* [MAX_SIZE+4k][MAX_SIZE+4k]
2540	*
2541	* Each range on their own accounts for 2 extents, but merged together
2542	* they are only 3 extents worth of accounting, so we need to drop in
2543	* this case.
2544	*/
2545	old_size = other->end - other->start + 1;
2546	num_extents = count_max_extents(fs_info, old_size);
2547	old_size = new->end - new->start + 1;
2548	num_extents += count_max_extents(fs_info, old_size);
2549	if (count_max_extents(fs_info, new_size) >= num_extents)
2550	return;
2551
2552	spin_lock(&inode->lock);
2553	btrfs_mod_outstanding_extents(inode, -1);
2554	spin_unlock(&inode->lock);
2555	}
2556
2557	static void btrfs_add_delalloc_inode(struct btrfs_inode *inode)
2558	{
2559	struct btrfs_root *root = inode->root;
2560	struct btrfs_fs_info *fs_info = root->fs_info;
2561
2562	spin_lock(&root->delalloc_lock);
2563	ASSERT(list_empty(&inode->delalloc_inodes));
2564	list_add_tail(&inode->delalloc_inodes, &root->delalloc_inodes);
2565	root->nr_delalloc_inodes++;
2566	if (root->nr_delalloc_inodes == 1) {
2567	spin_lock(&fs_info->delalloc_root_lock);
2568	ASSERT(list_empty(&root->delalloc_root));
2569	list_add_tail(&root->delalloc_root, &fs_info->delalloc_roots);
2570	spin_unlock(&fs_info->delalloc_root_lock);
2571	}
2572	spin_unlock(&root->delalloc_lock);
2573	}
2574
2575	void btrfs_del_delalloc_inode(struct btrfs_inode *inode)
2576	{
2577	struct btrfs_root *root = inode->root;
2578	struct btrfs_fs_info *fs_info = root->fs_info;
2579
2580	lockdep_assert_held(&root->delalloc_lock);
2581
2582	/*
2583	* We may be called after the inode was already deleted from the list,
2584	* namely in the transaction abort path btrfs_destroy_delalloc_inodes(),
2585	* and then later through btrfs_clear_delalloc_extent() while the inode
2586	* still has ->delalloc_bytes > 0.
2587	*/
2588	if (!list_empty(&inode->delalloc_inodes)) {
2589	list_del_init(&inode->delalloc_inodes);
2590	root->nr_delalloc_inodes--;
2591	if (!root->nr_delalloc_inodes) {
2592	ASSERT(list_empty(&root->delalloc_inodes));
2593	spin_lock(&fs_info->delalloc_root_lock);
2594	ASSERT(!list_empty(&root->delalloc_root));
2595	list_del_init(&root->delalloc_root);
2596	spin_unlock(&fs_info->delalloc_root_lock);
2597	}
2598	}
2599	}
2600
2601	/*
2602	* Properly track delayed allocation bytes in the inode and to maintain the
2603	* list of inodes that have pending delalloc work to be done.
2604	*/
2605	void btrfs_set_delalloc_extent(struct btrfs_inode inode, struct extent_state state,
2606	u32 bits)
2607	{
2608	struct btrfs_fs_info *fs_info = inode->root->fs_info;
2609
2610	lockdep_assert_held(&inode->io_tree.lock);
2611
2612	WARN_ON((bits & EXTENT_DEFRAG) && !(bits & EXTENT_DELALLOC));
2613	/*
2614	* set_bit and clear bit hooks normally require _irqsave/restore
2615	* but in this case, we are only testing for the DELALLOC
2616	* bit, which is only set or cleared with irqs on
2617	*/
2618	if (!(state->state & EXTENT_DELALLOC) && (bits & EXTENT_DELALLOC)) {
2619	u64 len = state->end + 1 - state->start;
2620	u64 prev_delalloc_bytes;
2621	u32 num_extents = count_max_extents(fs_info, len);
2622
2623	spin_lock(&inode->lock);
2624	btrfs_mod_outstanding_extents(inode, num_extents);
2625	spin_unlock(&inode->lock);
2626
2627	/* For sanity tests */
2628	if (btrfs_is_testing(fs_info))
2629	return;
2630
2631	percpu_counter_add_batch(&fs_info->delalloc_bytes, len,
2632	fs_info->delalloc_batch);
2633	spin_lock(&inode->lock);
2634	prev_delalloc_bytes = inode->delalloc_bytes;
2635	inode->delalloc_bytes += len;
2636	if (bits & EXTENT_DEFRAG)
2637	inode->defrag_bytes += len;
2638	spin_unlock(&inode->lock);
2639
2640	/*
2641	* We don't need to be under the protection of the inode's lock,
2642	* because we are called while holding the inode's io_tree lock
2643	* and are therefore protected against concurrent calls of this
2644	* function and btrfs_clear_delalloc_extent().
2645	*/
2646	if (!btrfs_is_free_space_inode(inode) && prev_delalloc_bytes == 0)
2647	btrfs_add_delalloc_inode(inode);
2648	}
2649
2650	if (!(state->state & EXTENT_DELALLOC_NEW) &&
2651	(bits & EXTENT_DELALLOC_NEW)) {
2652	spin_lock(&inode->lock);
2653	inode->new_delalloc_bytes += state->end + 1 - state->start;
2654	spin_unlock(&inode->lock);
2655	}
2656	}
2657
2658	/*
2659	* Once a range is no longer delalloc this function ensures that proper
2660	* accounting happens.
2661	*/
2662	void btrfs_clear_delalloc_extent(struct btrfs_inode *inode,
2663	struct extent_state *state, u32 bits)
2664	{
2665	struct btrfs_fs_info *fs_info = inode->root->fs_info;
2666	u64 len = state->end + 1 - state->start;
2667	u32 num_extents = count_max_extents(fs_info, len);
2668
2669	lockdep_assert_held(&inode->io_tree.lock);
2670
2671	if ((state->state & EXTENT_DEFRAG) && (bits & EXTENT_DEFRAG)) {
2672	spin_lock(&inode->lock);
2673	inode->defrag_bytes -= len;
2674	spin_unlock(&inode->lock);
2675	}
2676
2677	/*
2678	* set_bit and clear bit hooks normally require _irqsave/restore
2679	* but in this case, we are only testing for the DELALLOC
2680	* bit, which is only set or cleared with irqs on
2681	*/
2682	if ((state->state & EXTENT_DELALLOC) && (bits & EXTENT_DELALLOC)) {
2683	struct btrfs_root *root = inode->root;
2684	u64 new_delalloc_bytes;
2685
2686	spin_lock(&inode->lock);
2687	btrfs_mod_outstanding_extents(inode, -num_extents);
2688	spin_unlock(&inode->lock);
2689
2690	/*
2691	* We don't reserve metadata space for space cache inodes so we
2692	* don't need to call delalloc_release_metadata if there is an
2693	* error.
2694	*/
2695	if (bits & EXTENT_CLEAR_META_RESV &&
2696	root != fs_info->tree_root)
2697	btrfs_delalloc_release_metadata(inode, len, true);
2698
2699	/* For sanity tests. */
2700	if (btrfs_is_testing(fs_info))
2701	return;
2702
2703	if (!btrfs_is_data_reloc_root(root) &&
2704	!btrfs_is_free_space_inode(inode) &&
2705	!(state->state & EXTENT_NORESERVE) &&
2706	(bits & EXTENT_CLEAR_DATA_RESV))
2707	btrfs_free_reserved_data_space_noquota(inode, len);
2708
2709	percpu_counter_add_batch(&fs_info->delalloc_bytes, -len,
2710	fs_info->delalloc_batch);
2711	spin_lock(&inode->lock);
2712	inode->delalloc_bytes -= len;
2713	new_delalloc_bytes = inode->delalloc_bytes;
2714	spin_unlock(&inode->lock);
2715
2716	/*
2717	* We don't need to be under the protection of the inode's lock,
2718	* because we are called while holding the inode's io_tree lock
2719	* and are therefore protected against concurrent calls of this
2720	* function and btrfs_set_delalloc_extent().
2721	*/
2722	if (!btrfs_is_free_space_inode(inode) && new_delalloc_bytes == 0) {
2723	spin_lock(&root->delalloc_lock);
2724	btrfs_del_delalloc_inode(inode);
2725	spin_unlock(&root->delalloc_lock);
2726	}
2727	}
2728
2729	if ((state->state & EXTENT_DELALLOC_NEW) &&
2730	(bits & EXTENT_DELALLOC_NEW)) {
2731	spin_lock(&inode->lock);
2732	ASSERT(inode->new_delalloc_bytes >= len);
2733	inode->new_delalloc_bytes -= len;
2734	if (bits & EXTENT_ADD_INODE_BYTES)
2735	inode_add_bytes(&inode->vfs_inode, len);
2736	spin_unlock(&inode->lock);
2737	}
2738	}
2739
2740	/*
2741	* Given an ordered extent and insert all its checksums into the csum tree.
2742	*
2743	* This happens at IO completion time based on sums calculated at bio
2744	* submission time.
2745	*/
2746	static int add_pending_csums(struct btrfs_trans_handle *trans,
2747	struct btrfs_ordered_extent *oe)
2748	{
2749	struct btrfs_ordered_sum *sum;
2750	struct btrfs_root *csum_root = NULL;
2751	int ret;
2752
2753	list_for_each_entry(sum, &oe->csum_list, list) {
2754	if (!csum_root) {
2755	csum_root = btrfs_csum_root(trans->fs_info,
2756	sum->logical);
2757	if (unlikely(!csum_root)) {
2758	btrfs_err(trans->fs_info,
2759	"missing csum root for extent at bytenr %llu",
2760	sum->logical);
2761	return -EUCLEAN;
2762	}
2763	}
2764	trans->adding_csums = true;
2765	ret = btrfs_insert_data_csums(trans, csum_root, sum);
2766	trans->adding_csums = false;
2767	if (ret)
2768	return ret;
2769	}
2770	return 0;
2771	}
2772
2773	static int btrfs_find_new_delalloc_bytes(struct btrfs_inode *inode,
2774	const u64 start,
2775	const u64 len,
2776	struct extent_state **cached_state)
2777	{
2778	u64 search_start = start;
2779	const u64 end = start + len - 1;
2780
2781	while (search_start < end) {
2782	const u64 search_len = end - search_start + 1;
2783	struct extent_map *em;
2784	u64 em_len;
2785	int ret = 0;
2786
2787	em = btrfs_get_extent(inode, NULL, search_start, search_len);
2788	if (IS_ERR(em))
2789	return PTR_ERR(em);
2790
2791	if (em->disk_bytenr != EXTENT_MAP_HOLE)
2792	goto next;
2793
2794	em_len = em->len;
2795	if (em->start < search_start)
2796	em_len -= search_start - em->start;
2797	if (em_len > search_len)
2798	em_len = search_len;
2799
2800	ret = btrfs_set_extent_bit(&inode->io_tree, search_start,
2801	search_start + em_len - 1,
2802	EXTENT_DELALLOC_NEW, cached_state);
2803	next:
2804	search_start = btrfs_extent_map_end(em);
2805	btrfs_free_extent_map(em);
2806	if (ret)
2807	return ret;
2808	}
2809	return 0;
2810	}
2811
2812	int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
2813	unsigned int extra_bits,
2814	struct extent_state **cached_state)
2815	{
2816	const u32 blocksize = inode->root->fs_info->sectorsize;
2817
2818	/* Basic alignment check. */
2819	ASSERT(IS_ALIGNED(start, blocksize), "start=%llu blocksize=%u",
2820	start, blocksize);
2821	ASSERT(IS_ALIGNED(end + 1, blocksize), "inclusive end=%llu blocksize=%u",
2822	end, blocksize);
2823
2824	if (start >= i_size_read(&inode->vfs_inode) &&
2825	!(inode->flags & BTRFS_INODE_PREALLOC)) {
2826	/*
2827	* There can't be any extents following eof in this case so just
2828	* set the delalloc new bit for the range directly.
2829	*/
2830	extra_bits \|= EXTENT_DELALLOC_NEW;
2831	} else {
2832	int ret;
2833
2834	ret = btrfs_find_new_delalloc_bytes(inode, start,
2835	end + 1 - start,
2836	cached_state);
2837	if (ret)
2838	return ret;
2839	}
2840
2841	return btrfs_set_extent_bit(&inode->io_tree, start, end,
2842	EXTENT_DELALLOC \| extra_bits, cached_state);
2843	}
2844
2845	struct btrfs_writepage_fixup {
2846	struct folio *folio;
2847	struct btrfs_inode *inode;
2848	struct work_struct work;
2849	};
2850
2851	/*
2852	* Do the real fixup work of reserving space for the blocks a folio's fixup
2853	* state records. Queued by writepage_fixup() when writeback found the bits set.
2854	*
2855	* Since the fixup can be cancelled by a task dirtying with a reservation, we must
2856	* re-check the state of fixup under the folio lock.
2857	*/
2858	static void btrfs_writepage_fixup_worker(struct work_struct *work)
2859	{
2860	struct btrfs_writepage_fixup *fixup =
2861	container_of(work, struct btrfs_writepage_fixup, work);
2862	struct extent_state *cached_state = NULL;
2863	struct extent_changeset *data_reserved = NULL;
2864	unsigned long delalloc_bitmap[BITS_TO_LONGS(BTRFS_MAX_BLOCKS_PER_FOLIO)] = { 0 };
2865	struct folio *folio = fixup->folio;
2866	struct btrfs_inode *inode = fixup->inode;
2867	struct btrfs_fs_info *fs_info = inode->root->fs_info;
2868	const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio);
2869	const u32 sectorsize = fs_info->sectorsize;
2870	const u64 page_start = folio_pos(folio);
2871	const u64 page_end = folio_next_pos(folio) - 1;
2872	unsigned int start_bit;
2873	unsigned int end_bit;
2874	unsigned int bit;
2875	bool reserved;
2876	int ret;
2877
2878	/*
2879	* We would prefer to reserve under the folio lock when we know exactly
2880	* which blocks need a reservation. Unfortunately, since the reservation
2881	* can go into flushers which can go into writeback, which takes folio
2882	* locks, that is not possible. Therefore, we have to reserve for the
2883	* whole folio here, then release what we didn't end up needing once we
2884	* figure it out.
2885	*
2886	* Also note the slightly strange error checking. If fixup is actually
2887	* not set, we don't need to mark an error on the mapping. So hang on to
2888	* ret until after we lock and find out if we actually care.
2889	*/
2890	ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start,
2891	folio_size(folio));
2892	reserved = (ret == 0);
2893	again:
2894	folio_lock(folio);
2895
2896	if (!folio->mapping \|\| !folio_test_fixup_pending(folio)) {
2897	ret = 0;
2898	goto out;
2899	}
2900	if (ret)
2901	goto out;
2902
2903	btrfs_lock_extent(&inode->io_tree, page_start, page_end, &cached_state);
2904
2905	for (bit = 0; bit < blocks_per_folio; bit++) {
2906	struct btrfs_ordered_extent *ordered;
2907	const u64 start = page_start + (bit << fs_info->sectorsize_bits);
2908
2909	if (test_bit(bit, delalloc_bitmap))
2910	continue;
2911	if (!btrfs_folio_test_fixup(fs_info, folio, start, sectorsize))
2912	continue;
2913	/*
2914	* Any task that sets EXTENT_DELALLOC clears the fixup bits
2915	* under the folio lock, so it should be impossible to observe
2916	* both under the lock. Setting delalloc twice would wrongly
2917	* double account the space.
2918	*/
2919	if (IS_ENABLED(CONFIG_BTRFS_DEBUG) &&
2920	unlikely(btrfs_test_range_bit_exists(&inode->io_tree, start,
2921	start + sectorsize - 1,
2922	EXTENT_DELALLOC))) {
2923	DEBUG_WARN("fixup worker: delalloc and fixup conflict. ino %llu start %llu",
2924	btrfs_ino(inode), start);
2925	btrfs_folio_clear_fixup(fs_info, folio, start, sectorsize);
2926	continue;
2927	}
2928	ordered = btrfs_lookup_ordered_range(inode, start, sectorsize);
2929	if (ordered) {
2930	trace_btrfs_writepage_fixup_defer(inode, ordered);
2931	btrfs_unlock_extent(&inode->io_tree, page_start,
2932	page_end, &cached_state);
2933	folio_unlock(folio);
2934	btrfs_start_ordered_extent(ordered);
2935	btrfs_put_ordered_extent(ordered);
2936	goto again;
2937	}
2938	ret = btrfs_set_extent_delalloc(inode, start,
2939	start + sectorsize - 1, 0,
2940	&cached_state);
2941	if (ret)
2942	break;
2943	trace_btrfs_writepage_fixup_reserve(inode, start, sectorsize);
2944	btrfs_folio_clear_fixup(fs_info, folio, start, sectorsize);
2945	set_bit(bit, delalloc_bitmap);
2946	}
2947
2948	btrfs_unlock_extent(&inode->io_tree, page_start, page_end, &cached_state);
2949	out:
2950	if (ret < 0) {
2951	/* Failure here is analogous to failure in writeback. */
2952	mapping_set_error(folio->mapping, ret);
2953	btrfs_folio_clear_fixup_dirty(fs_info, folio, page_start,
2954	folio_size(folio));
2955	}
2956	if (reserved) {
2957	btrfs_delalloc_release_extents(inode, folio_size(folio));
2958	for_each_clear_bitrange(start_bit, end_bit, delalloc_bitmap,
2959	blocks_per_folio)
2960	btrfs_delalloc_release_space(inode, data_reserved,
2961	page_start + (start_bit << fs_info->sectorsize_bits),
2962	(end_bit - start_bit) << fs_info->sectorsize_bits,
2963	true);
2964	}
2965	folio_unlock(folio);
2966	folio_put(folio);
2967	kfree(fixup);
2968	extent_changeset_free(data_reserved);
2969	btrfs_add_delayed_iput(inode);
2970	}
2971
2972	/*
2973	* Queue space reservation fixup work for blocks dirtied without a space reservation.
2974	*
2975	* Should be used by writeback while holding the folio locked.
2976	*
2977	* If we fail to queue fixup, then the folio state is unchanged and a future
2978	* writeback pass will still see it.
2979	*/
2980	void btrfs_queue_writepage_fixup(struct btrfs_inode inode, struct folio folio)
2981	{
2982	struct btrfs_fs_info *fs_info = inode->root->fs_info;
2983	struct btrfs_writepage_fixup *fixup;
2984
2985	/*
2986	* Disallow queueing more fixup during unmount to break the cycle
2987	* of writeback queuing fixup queuing writeback etc.
2988	*
2989	* If it actually hit, then something which was fixup wasn't written
2990	* which we should warn about.
2991	*/
2992	if (btrfs_fs_closing(fs_info)) {
2993	btrfs_warn_rl(fs_info,
2994	"dropping unqueued fixup blocks at unmount. root %lld ino %llu folio %llu",
2995	btrfs_root_id(inode->root), btrfs_ino(inode),
2996	folio_pos(folio));
2997	btrfs_folio_clear_fixup_dirty(fs_info, folio,
2998	folio_pos(folio), folio_size(folio));
2999	return;
3000	}
3001
3002	fixup = kzalloc_obj(*fixup, GFP_NOFS);
3003	if (!fixup)
3004	return;
3005
3006	/*
3007	* This is called from within extent_write_cache_pages() which
3008	* has successfully done an igrab(). But that will be released at the
3009	* end of the writeback pass. We need to extend it for the worker as well.
3010	*/
3011	ihold(&inode->vfs_inode);
3012	folio_get(folio);
3013	INIT_WORK(&fixup->work, btrfs_writepage_fixup_worker);
3014	fixup->folio = folio;
3015	fixup->inode = inode;
3016	queue_work(fs_info->fixup_workers, &fixup->work);
3017	}
3018
3019	/*
3020	* Clear the old accounting flags and set EXTENT_DELALLOC for the range.
3021	*
3022	* Return <0 for error, in that case no range has EXTENT_DELALLOC bit cleared or set.
3023	*/
3024	int btrfs_reset_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
3025	unsigned int extra_bits, struct extent_state **cached_state)
3026	{
3027	const u32 blocksize = inode->root->fs_info->sectorsize;
3028
3029	/* The @extra_bits can only be EXTENT_NORESERVE for now. */
3030	ASSERT(!(extra_bits & ~EXTENT_NORESERVE), "extra_bits=0x%x", extra_bits);
3031
3032	/* Basic alignment check. */
3033	ASSERT(IS_ALIGNED(start, blocksize), "start=%llu blocksize=%u",
3034	start, blocksize);
3035	ASSERT(IS_ALIGNED(end + 1, blocksize), "inclusive end=%llu blocksize=%u",
3036	end, blocksize);
3037
3038	/*
3039	* Check and set DELALLOC_NEW flag, this needs to search tree thus can
3040	* fail early. Thus we want to do this before clearing EXTENT_DELALLOC.
3041	*/
3042	if (start >= i_size_read(&inode->vfs_inode) &&
3043	!(inode->flags & BTRFS_INODE_PREALLOC)) {
3044	/*
3045	* There can't be any extents following EOF in this case so just
3046	* set the delalloc new bit for the range directly.
3047	*/
3048	extra_bits \|= EXTENT_DELALLOC_NEW;
3049	} else {
3050	int ret;
3051
3052	ret = btrfs_find_new_delalloc_bytes(inode, start, end + 1 - start,
3053	NULL);
3054	if (unlikely(ret))
3055	return ret;
3056	}
3057	/* Clear the old accounting as the range may already be dirty. */
3058	btrfs_clear_extent_bit(&inode->io_tree, start, end,
3059	EXTENT_DELALLOC \| EXTENT_DO_ACCOUNTING \|
3060	EXTENT_DEFRAG, cached_state);
3061	return btrfs_set_extent_bit(&inode->io_tree, start, end,
3062	EXTENT_DELALLOC \| extra_bits, cached_state);
3063	}
3064
3065	static int insert_reserved_file_extent(struct btrfs_trans_handle *trans,
3066	struct btrfs_inode *inode, u64 file_pos,
3067	struct btrfs_file_extent_item *stack_fi,
3068	const bool update_inode_bytes,
3069	u64 qgroup_reserved)
3070	{
3071	struct btrfs_root *root = inode->root;
3072	const u32 sectorsize = root->fs_info->sectorsize;
3073	BTRFS_PATH_AUTO_FREE(path);
3074	struct extent_buffer *leaf;
3075	struct btrfs_key ins;
3076	u64 disk_num_bytes = btrfs_stack_file_extent_disk_num_bytes(stack_fi);
3077	u64 disk_bytenr = btrfs_stack_file_extent_disk_bytenr(stack_fi);
3078	u64 offset = btrfs_stack_file_extent_offset(stack_fi);
3079	u64 num_bytes = btrfs_stack_file_extent_num_bytes(stack_fi);
3080	u64 ram_bytes = btrfs_stack_file_extent_ram_bytes(stack_fi);
3081	struct btrfs_drop_extents_args drop_args = { 0 };
3082	int ret;
3083
3084	path = btrfs_alloc_path();
3085	if (!path)
3086	return -ENOMEM;
3087
3088	/*
3089	* we may be replacing one extent in the tree with another.
3090	* The new extent is pinned in the extent map, and we don't want
3091	* to drop it from the cache until it is completely in the btree.
3092	*
3093	* So, tell btrfs_drop_extents to leave this extent in the cache.
3094	* the caller is expected to unpin it and allow it to be merged
3095	* with the others.
3096	*/
3097	drop_args.path = path;
3098	drop_args.start = file_pos;
3099	drop_args.end = file_pos + num_bytes;
3100	drop_args.replace_extent = true;
3101	drop_args.extent_item_size = sizeof(*stack_fi);
3102	ret = btrfs_drop_extents(trans, root, inode, &drop_args);
3103	if (ret)
3104	return ret;
3105
3106	if (!drop_args.extent_inserted) {
3107	ins.objectid = btrfs_ino(inode);
3108	ins.type = BTRFS_EXTENT_DATA_KEY;
3109	ins.offset = file_pos;
3110
3111	ret = btrfs_insert_empty_item(trans, root, path, &ins,
3112	sizeof(*stack_fi));
3113	if (ret)
3114	return ret;
3115	}
3116	leaf = path->nodes[0];
3117	btrfs_set_stack_file_extent_generation(stack_fi, trans->transid);
3118	write_extent_buffer(leaf, stack_fi,
3119	btrfs_item_ptr_offset(leaf, path->slots[0]),
3120	sizeof(struct btrfs_file_extent_item));
3121
3122	btrfs_release_path(path);
3123
3124	/*
3125	* If we dropped an inline extent here, we know the range where it is
3126	* was not marked with the EXTENT_DELALLOC_NEW bit, so we update the
3127	* number of bytes only for that range containing the inline extent.
3128	* The remaining of the range will be processed when clearing the
3129	* EXTENT_DELALLOC_BIT bit through the ordered extent completion.
3130	*/
3131	if (file_pos == 0 && !IS_ALIGNED(drop_args.bytes_found, sectorsize)) {
3132	u64 inline_size = round_down(drop_args.bytes_found, sectorsize);
3133
3134	inline_size = drop_args.bytes_found - inline_size;
3135	btrfs_update_inode_bytes(inode, sectorsize, inline_size);
3136	drop_args.bytes_found -= inline_size;
3137	num_bytes -= sectorsize;
3138	}
3139
3140	if (update_inode_bytes)
3141	btrfs_update_inode_bytes(inode, num_bytes, drop_args.bytes_found);
3142
3143	ins.objectid = disk_bytenr;
3144	ins.type = BTRFS_EXTENT_ITEM_KEY;
3145	ins.offset = disk_num_bytes;
3146
3147	ret = btrfs_inode_set_file_extent_range(inode, file_pos, ram_bytes);
3148	if (ret)
3149	return ret;
3150
3151	return btrfs_alloc_reserved_file_extent(trans, root, btrfs_ino(inode),
3152	file_pos - offset,
3153	qgroup_reserved, &ins);
3154	}
3155
3156	static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info,
3157	u64 start, u64 len)
3158	{
3159	struct btrfs_block_group *cache;
3160
3161	cache = btrfs_lookup_block_group(fs_info, start);
3162	ASSERT(cache);
3163
3164	spin_lock(&cache->lock);
3165	cache->delalloc_bytes -= len;
3166	spin_unlock(&cache->lock);
3167
3168	btrfs_put_block_group(cache);
3169	}
3170
3171	static int insert_ordered_extent_file_extent(struct btrfs_trans_handle *trans,
3172	struct btrfs_ordered_extent *oe)
3173	{
3174	struct btrfs_file_extent_item stack_fi;
3175	bool update_inode_bytes;
3176	u64 num_bytes = oe->num_bytes;
3177	u64 ram_bytes = oe->ram_bytes;
3178
3179	memset(&stack_fi, 0, sizeof(stack_fi));
3180	btrfs_set_stack_file_extent_type(&stack_fi, BTRFS_FILE_EXTENT_REG);
3181	btrfs_set_stack_file_extent_disk_bytenr(&stack_fi, oe->disk_bytenr);
3182	btrfs_set_stack_file_extent_disk_num_bytes(&stack_fi,
3183	oe->disk_num_bytes);
3184	btrfs_set_stack_file_extent_offset(&stack_fi, oe->offset);
3185	if (test_bit(BTRFS_ORDERED_TRUNCATED, &oe->flags))
3186	num_bytes = oe->truncated_len;
3187	btrfs_set_stack_file_extent_num_bytes(&stack_fi, num_bytes);
3188	btrfs_set_stack_file_extent_ram_bytes(&stack_fi, ram_bytes);
3189	btrfs_set_stack_file_extent_compression(&stack_fi, oe->compress_type);
3190	/* Encryption and other encoding is reserved and all 0 */
3191
3192	/*
3193	* For delalloc, when completing an ordered extent we update the inode's
3194	* bytes when clearing the range in the inode's io tree, so pass false
3195	* as the argument 'update_inode_bytes' to insert_reserved_file_extent(),
3196	* except if the ordered extent was truncated.
3197	*/
3198	update_inode_bytes = test_bit(BTRFS_ORDERED_DIRECT, &oe->flags) \|\|
3199	test_bit(BTRFS_ORDERED_ENCODED, &oe->flags) \|\|
3200	test_bit(BTRFS_ORDERED_TRUNCATED, &oe->flags);
3201
3202	return insert_reserved_file_extent(trans, oe->inode,
3203	oe->file_offset, &stack_fi,
3204	update_inode_bytes, oe->qgroup_rsv);
3205	}
3206
3207	/*
3208	* As ordered data IO finishes, this gets called so we can finish
3209	* an ordered extent if the range of bytes in the file it covers are
3210	* fully written.
3211	*/
3212	int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent)
3213	{
3214	struct btrfs_inode *inode = ordered_extent->inode;
3215	struct btrfs_root *root = inode->root;
3216	struct btrfs_fs_info *fs_info = root->fs_info;
3217	struct btrfs_trans_handle *trans = NULL;
3218	struct extent_io_tree *io_tree = &inode->io_tree;
3219	struct extent_state *cached_state = NULL;
3220	u64 start, end;
3221	int compress_type = 0;
3222	int ret = 0;
3223	u64 logical_len = ordered_extent->num_bytes;
3224	bool freespace_inode;
3225	bool truncated = false;
3226	bool clear_reserved_extent = true;
3227	unsigned int clear_bits = 0;
3228
3229	start = ordered_extent->file_offset;
3230	end = start + ordered_extent->num_bytes - 1;
3231
3232	if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
3233	!test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags) &&
3234	!test_bit(BTRFS_ORDERED_DIRECT, &ordered_extent->flags) &&
3235	!test_bit(BTRFS_ORDERED_ENCODED, &ordered_extent->flags))
3236	clear_bits \|= EXTENT_DELALLOC_NEW;
3237
3238	if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags))
3239	clear_bits \|= EXTENT_DEFRAG;
3240
3241	freespace_inode = btrfs_is_free_space_inode(inode);
3242	if (!freespace_inode)
3243	btrfs_lockdep_acquire(fs_info, btrfs_ordered_extent);
3244
3245	if (unlikely(test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags))) {
3246	ret = -EIO;
3247	goto out;
3248	}
3249
3250	ret = btrfs_zone_finish_endio(fs_info, ordered_extent->disk_bytenr,
3251	ordered_extent->disk_num_bytes);
3252	if (ret)
3253	goto out;
3254
3255	if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) {
3256	truncated = true;
3257	logical_len = ordered_extent->truncated_len;
3258	/* Truncated the entire extent, don't bother adding */
3259	if (!logical_len)
3260	goto out;
3261	}
3262
3263	/*
3264	* If it's a COW write we need to lock the extent range as we will be
3265	* inserting/replacing file extent items and unpinning an extent map.
3266	* This must be taken before joining a transaction, as it's a higher
3267	* level lock (like the inode's VFS lock), otherwise we can run into an
3268	* ABBA deadlock with other tasks (transactions work like a lock,
3269	* depending on their current state).
3270	*/
3271	if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) {
3272	clear_bits \|= EXTENT_LOCKED \| EXTENT_FINISHING_ORDERED;
3273	btrfs_lock_extent_bits(io_tree, start, end,
3274	EXTENT_LOCKED \| EXTENT_FINISHING_ORDERED,
3275	&cached_state);
3276	}
3277
3278	if (freespace_inode)
3279	trans = btrfs_join_transaction_spacecache(root);
3280	else
3281	trans = btrfs_join_transaction(root);
3282	if (IS_ERR(trans)) {
3283	ret = PTR_ERR(trans);
3284	trans = NULL;
3285	goto out;
3286	}
3287
3288	trans->block_rsv = &inode->block_rsv;
3289
3290	ret = btrfs_insert_raid_extent(trans, ordered_extent);
3291	if (unlikely(ret)) {
3292	btrfs_abort_transaction(trans, ret);
3293	goto out;
3294	}
3295
3296	if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) {
3297	/* Logic error */
3298	ASSERT(list_empty(&ordered_extent->csum_list));
3299	if (unlikely(!list_empty(&ordered_extent->csum_list))) {
3300	ret = -EINVAL;
3301	btrfs_abort_transaction(trans, ret);
3302	goto out;
3303	}
3304
3305	btrfs_inode_safe_disk_i_size_write(inode, 0);
3306	ret = btrfs_update_inode_fallback(trans, inode);
3307	if (unlikely(ret)) {
3308	/* -ENOMEM or corruption */
3309	btrfs_abort_transaction(trans, ret);
3310	}
3311	goto out;
3312	}
3313
3314	if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags))
3315	compress_type = ordered_extent->compress_type;
3316	if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {
3317	BUG_ON(compress_type);
3318	ret = btrfs_mark_extent_written(trans, inode,
3319	ordered_extent->file_offset,
3320	ordered_extent->file_offset +
3321	logical_len);
3322	btrfs_zoned_release_data_reloc_bg(fs_info, ordered_extent->disk_bytenr,
3323	ordered_extent->disk_num_bytes);
3324	if (unlikely(ret < 0)) {
3325	btrfs_abort_transaction(trans, ret);
3326	goto out;
3327	}
3328	} else {
3329	BUG_ON(root == fs_info->tree_root);
3330	ret = insert_ordered_extent_file_extent(trans, ordered_extent);
3331	if (unlikely(ret < 0)) {
3332	btrfs_abort_transaction(trans, ret);
3333	goto out;
3334	}
3335	clear_reserved_extent = false;
3336	btrfs_release_delalloc_bytes(fs_info,
3337	ordered_extent->disk_bytenr,
3338	ordered_extent->disk_num_bytes);
3339	}
3340
3341	ret = btrfs_unpin_extent_cache(inode, ordered_extent->file_offset,
3342	ordered_extent->num_bytes, trans->transid);
3343	if (unlikely(ret < 0)) {
3344	btrfs_abort_transaction(trans, ret);
3345	goto out;
3346	}
3347
3348	ret = add_pending_csums(trans, ordered_extent);
3349	if (unlikely(ret)) {
3350	btrfs_abort_transaction(trans, ret);
3351	goto out;
3352	}
3353
3354	/*
3355	* If this is a new delalloc range, clear its new delalloc flag to
3356	* update the inode's number of bytes. This needs to be done first
3357	* before updating the inode item.
3358	*/
3359	if ((clear_bits & EXTENT_DELALLOC_NEW) &&
3360	!test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags))
3361	btrfs_clear_extent_bit(&inode->io_tree, start, end,
3362	EXTENT_DELALLOC_NEW \| EXTENT_ADD_INODE_BYTES,
3363	&cached_state);
3364
3365	btrfs_inode_safe_disk_i_size_write(inode, 0);
3366	ret = btrfs_update_inode_fallback(trans, inode);
3367	if (unlikely(ret)) { /* -ENOMEM or corruption */
3368	btrfs_abort_transaction(trans, ret);
3369	goto out;
3370	}
3371	out:
3372	if (clear_bits)
3373	btrfs_clear_extent_bit(&inode->io_tree, start, end, clear_bits,
3374	&cached_state);
3375
3376	if (trans)
3377	btrfs_end_transaction(trans);
3378
3379	if (ret \|\| truncated) {
3380	/*
3381	* If we failed to finish this ordered extent for any reason we
3382	* need to make sure BTRFS_ORDERED_IOERR is set on the ordered
3383	* extent, and mark the inode with the error if it wasn't
3384	* already set. Any error during writeback would have already
3385	* set the mapping error, so we need to set it if we're the ones
3386	* marking this ordered extent as failed.
3387	*/
3388	if (ret)
3389	btrfs_mark_ordered_extent_error(ordered_extent);
3390
3391	/*
3392	* Drop extent maps for the part of the extent we didn't write.
3393	*
3394	* We have an exception here for the free_space_inode, this is
3395	* because when we do btrfs_get_extent() on the free space inode
3396	* we will search the commit root. If this is a new block group
3397	* we won't find anything, and we will trip over the assert in
3398	* writepage where we do ASSERT(em->block_start !=
3399	* EXTENT_MAP_HOLE).
3400	*
3401	* Theoretically we could also skip this for any NOCOW extent as
3402	* we don't mess with the extent map tree in the NOCOW case, but
3403	* for now simply skip this if we are the free space inode.
3404	*/
3405	if (!btrfs_is_free_space_inode(inode)) {
3406	u64 unwritten_start = start;
3407
3408	if (truncated)
3409	unwritten_start += logical_len;
3410
3411	btrfs_drop_extent_map_range(inode, unwritten_start,
3412	end, false);
3413	}
3414
3415	/*
3416	* If the ordered extent had an IOERR or something else went
3417	* wrong we need to return the space for this ordered extent
3418	* back to the allocator. We only free the extent in the
3419	* truncated case if we didn't write out the extent at all.
3420	*
3421	* If we made it past insert_reserved_file_extent before we
3422	* errored out then we don't need to do this as the accounting
3423	* has already been done.
3424	*/
3425	if ((ret \|\| !logical_len) &&
3426	clear_reserved_extent &&
3427	!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
3428	!test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {
3429	/*
3430	* Discard the range before returning it back to the
3431	* free space pool
3432	*/
3433	if (ret && btrfs_test_opt(fs_info, DISCARD_SYNC))
3434	btrfs_discard_extent(fs_info,
3435	ordered_extent->disk_bytenr,
3436	ordered_extent->disk_num_bytes,
3437	NULL, true);
3438	btrfs_free_reserved_extent(fs_info,
3439	ordered_extent->disk_bytenr,
3440	ordered_extent->disk_num_bytes, true);
3441	/*
3442	* Actually free the qgroup rsv which was released when
3443	* the ordered extent was created.
3444	*/
3445	btrfs_qgroup_free_refroot(fs_info, btrfs_root_id(inode->root),
3446	ordered_extent->qgroup_rsv,
3447	BTRFS_QGROUP_RSV_DATA);
3448	}
3449	}
3450
3451	/*
3452	* This needs to be done to make sure anybody waiting knows we are done
3453	* updating everything for this ordered extent.
3454	*/
3455	btrfs_remove_ordered_extent(ordered_extent);
3456
3457	/* Cleanup any remaining biocs attached to the OE. */
3458	btrfs_cleanup_ordered_bioc_list(ordered_extent);
3459
3460	/* once for us */
3461	btrfs_put_ordered_extent(ordered_extent);
3462	/* once for the tree */
3463	btrfs_put_ordered_extent(ordered_extent);
3464
3465	return ret;
3466	}
3467
3468	int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered)
3469	{
3470	if (btrfs_is_zoned(ordered->inode->root->fs_info) &&
3471	!test_bit(BTRFS_ORDERED_IOERR, &ordered->flags) &&
3472	list_empty(&ordered->bioc_list))
3473	btrfs_finish_ordered_zoned(ordered);
3474	return btrfs_finish_one_ordered(ordered);
3475	}
3476
3477	/* Generate data checksum for a single fs block, pointed to by @orig_iter. */
3478	void btrfs_csum_one_bio_block(struct btrfs_fs_info fs_info, struct bio bio,
3479	const struct bvec_iter orig_iter, u8 csum)
3480	{
3481	struct btrfs_csum_ctx cctx;
3482	struct bvec_iter iter = *orig_iter;
3483	const u32 blocksize = fs_info->sectorsize;
3484	u32 cur = 0;
3485
3486	btrfs_csum_init(&cctx, fs_info->csum_type);
3487	while (cur < blocksize) {
3488	struct page *page = bio_iter_page(bio, iter);
3489	const u32 pg_off = bio_iter_offset(bio, iter);
3490	const u32 cur_len = min(bio_iter_len(bio, iter), blocksize - cur);
3491	void *kaddr;
3492
3493	kaddr = kmap_local_page(page) + pg_off;
3494	btrfs_csum_update(&cctx, kaddr, cur_len);
3495	kunmap_local(kaddr);
3496
3497	bio_advance_iter_single(bio, &iter, cur_len);
3498	cur += cur_len;
3499	}
3500	btrfs_csum_final(&cctx, csum);
3501	}
3502
3503	/*
3504	* Verify the checksum of a single data sector, which can be scattered at
3505	* different noncontiguous pages.
3506	*
3507	* @bbio: btrfs_io_bio which contains the csum
3508	* @orig_iter: bvec iter pointing to the start of the block
3509	* @dev: device the sector is on (optional)
3510	*
3511	* Check if the checksum on a data block is valid. When a checksum mismatch is
3512	* detected, report the error and fill the corrupted range with zero.
3513	*
3514	* Return %true if the sector is ok or had no checksum to start with, else %false.
3515	*/
3516	bool btrfs_bio_data_csum_ok(struct btrfs_bio *bbio,
3517	const struct bvec_iter *orig_iter,
3518	struct btrfs_device *dev)
3519	{
3520	struct btrfs_inode *inode = bbio->inode;
3521	struct btrfs_fs_info *fs_info = inode->root->fs_info;
3522	struct bvec_iter iter = *orig_iter;
3523	const u32 blocksize = fs_info->sectorsize;
3524	const u32 bio_offset = (iter.bi_sector - bbio->saved_iter.bi_sector) << SECTOR_SHIFT;
3525	u64 file_offset = bbio->file_offset + bio_offset;
3526	u64 end = file_offset + blocksize - 1;
3527	u8 *csum_expected;
3528	u8 csum[BTRFS_CSUM_SIZE];
3529	u32 cur = 0;
3530
3531	ASSERT(iter.bi_sector >= bbio->saved_iter.bi_sector);
3532
3533	if (!bbio->csum)
3534	return true;
3535
3536	if (btrfs_is_data_reloc_root(inode->root) &&
3537	btrfs_test_range_bit(&inode->io_tree, file_offset, end, EXTENT_NODATASUM,
3538	NULL)) {
3539	/* Skip the range without csum for data reloc inode */
3540	btrfs_clear_extent_bit(&inode->io_tree, file_offset, end,
3541	EXTENT_NODATASUM, NULL);
3542	return true;
3543	}
3544
3545	csum_expected = bbio->csum + (bio_offset >> fs_info->sectorsize_bits) *
3546	fs_info->csum_size;
3547	btrfs_csum_one_bio_block(fs_info, &bbio->bio, orig_iter, csum);
3548	if (unlikely(memcmp(csum, csum_expected, fs_info->csum_size) != 0))
3549	goto zeroit;
3550	return true;
3551
3552	zeroit:
3553	btrfs_print_data_csum_error(inode, file_offset, csum, csum_expected,
3554	bbio->mirror_num);
3555	if (dev)
3556	btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS);
3557	while (cur < blocksize) {
3558	struct page *page = bio_iter_page(&bbio->bio, iter);
3559	const u32 pg_off = bio_iter_offset(&bbio->bio, iter);
3560	const u32 cur_len = min(bio_iter_len(&bbio->bio, iter), blocksize - cur);
3561
3562	memzero_page(page, pg_off, cur_len);
3563
3564	bio_advance_iter_single(&bbio->bio, &iter, cur_len);
3565	cur += cur_len;
3566	}
3567	return false;
3568	}
3569
3570	/*
3571	* Perform a delayed iput on @inode.
3572	*
3573	* @inode: The inode we want to perform iput on
3574	*
3575	* This function uses the generic vfs_inode::i_count to track whether we should
3576	* just decrement it (in case it's > 1) or if this is the last iput then link
3577	* the inode to the delayed iput machinery. Delayed iputs are processed at
3578	* transaction commit time/superblock commit/cleaner kthread.
3579	*/
3580	void btrfs_add_delayed_iput(struct btrfs_inode *inode)
3581	{
3582	struct btrfs_fs_info *fs_info = inode->root->fs_info;
3583	unsigned long flags;
3584
3585	if (atomic_add_unless(&inode->vfs_inode.i_count, -1, 1))
3586	return;
3587
3588	WARN_ON_ONCE(test_bit(BTRFS_FS_STATE_NO_DELAYED_IPUT, &fs_info->fs_state));
3589	atomic_inc(&fs_info->nr_delayed_iputs);
3590	/*
3591	* Need to be irq safe here because we can be called from either an irq
3592	* context (see bio.c and btrfs_put_ordered_extent()) or a non-irq
3593	* context.
3594	*/
3595	spin_lock_irqsave(&fs_info->delayed_iput_lock, flags);
3596	ASSERT(list_empty(&inode->delayed_iput));
3597	list_add_tail(&inode->delayed_iput, &fs_info->delayed_iputs);
3598	spin_unlock_irqrestore(&fs_info->delayed_iput_lock, flags);
3599	if (!test_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags))
3600	wake_up_process(fs_info->cleaner_kthread);
3601	}
3602
3603	static void run_delayed_iput_locked(struct btrfs_fs_info *fs_info,
3604	struct btrfs_inode *inode)
3605	{
3606	list_del_init(&inode->delayed_iput);
3607	spin_unlock_irq(&fs_info->delayed_iput_lock);
3608	iput(&inode->vfs_inode);
3609	if (atomic_dec_and_test(&fs_info->nr_delayed_iputs))
3610	wake_up(&fs_info->delayed_iputs_wait);
3611	spin_lock_irq(&fs_info->delayed_iput_lock);
3612	}
3613
3614	static void btrfs_run_delayed_iput(struct btrfs_fs_info *fs_info,
3615	struct btrfs_inode *inode)
3616	{
3617	if (!list_empty(&inode->delayed_iput)) {
3618	spin_lock_irq(&fs_info->delayed_iput_lock);
3619	if (!list_empty(&inode->delayed_iput))
3620	run_delayed_iput_locked(fs_info, inode);
3621	spin_unlock_irq(&fs_info->delayed_iput_lock);
3622	}
3623	}
3624
3625	void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info)
3626	{
3627	/*
3628	* btrfs_put_ordered_extent() can run in irq context (see bio.c), which
3629	* calls btrfs_add_delayed_iput() and that needs to lock
3630	* fs_info->delayed_iput_lock. So we need to disable irqs here to
3631	* prevent a deadlock.
3632	*/
3633	spin_lock_irq(&fs_info->delayed_iput_lock);
3634	while (!list_empty(&fs_info->delayed_iputs)) {
3635	struct btrfs_inode *inode;
3636
3637	inode = list_first_entry(&fs_info->delayed_iputs,
3638	struct btrfs_inode, delayed_iput);
3639	run_delayed_iput_locked(fs_info, inode);
3640	if (need_resched()) {
3641	spin_unlock_irq(&fs_info->delayed_iput_lock);
3642	cond_resched();
3643	spin_lock_irq(&fs_info->delayed_iput_lock);
3644	}
3645	}
3646	spin_unlock_irq(&fs_info->delayed_iput_lock);
3647	}
3648
3649	/*
3650	* Wait for flushing all delayed iputs
3651	*
3652	* @fs_info: the filesystem
3653	*
3654	* This will wait on any delayed iputs that are currently running with KILLABLE
3655	* set. Once they are all done running we will return, unless we are killed in
3656	* which case we return EINTR. This helps in user operations like fallocate etc
3657	* that might get blocked on the iputs.
3658	*
3659	* Return EINTR if we were killed, 0 if nothing's pending
3660	*/
3661	int btrfs_wait_on_delayed_iputs(struct btrfs_fs_info *fs_info)
3662	{
3663	int ret = wait_event_killable(fs_info->delayed_iputs_wait,
3664	atomic_read(&fs_info->nr_delayed_iputs) == 0);
3665	if (ret)
3666	return -EINTR;
3667	return 0;
3668	}
3669
3670	/*
3671	* This creates an orphan entry for the given inode in case something goes wrong
3672	* in the middle of an unlink.
3673	*/
3674	int btrfs_orphan_add(struct btrfs_trans_handle *trans,
3675	struct btrfs_inode *inode)
3676	{
3677	int ret;
3678
3679	ret = btrfs_insert_orphan_item(trans, inode->root, btrfs_ino(inode));
3680	if (unlikely(ret && ret != -EEXIST)) {
3681	btrfs_abort_transaction(trans, ret);
3682	return ret;
3683	}
3684
3685	return 0;
3686	}
3687
3688	/*
3689	* We have done the delete so we can go ahead and remove the orphan item for
3690	* this particular inode.
3691	*/
3692	static int btrfs_orphan_del(struct btrfs_trans_handle *trans,
3693	struct btrfs_inode *inode)
3694	{
3695	return btrfs_del_orphan_item(trans, inode->root, btrfs_ino(inode));
3696	}
3697
3698	/*
3699	* this cleans up any orphans that may be left on the list from the last use
3700	* of this root.
3701	*/
3702	int btrfs_orphan_cleanup(struct btrfs_root *root)
3703	{
3704	struct btrfs_fs_info *fs_info = root->fs_info;
3705	BTRFS_PATH_AUTO_FREE(path);
3706	struct extent_buffer *leaf;
3707	struct btrfs_key key, found_key;
3708	struct btrfs_trans_handle *trans;
3709	u64 last_objectid = 0;
3710	int ret = 0, nr_unlink = 0;
3711
3712	if (test_and_set_bit(BTRFS_ROOT_ORPHAN_CLEANUP, &root->state))
3713	return 0;
3714
3715	path = btrfs_alloc_path();
3716	if (!path) {
3717	ret = -ENOMEM;
3718	goto out;
3719	}
3720	path->reada = READA_BACK;
3721
3722	key.objectid = BTRFS_ORPHAN_OBJECTID;
3723	key.type = BTRFS_ORPHAN_ITEM_KEY;
3724	key.offset = (u64)-1;
3725
3726	while (1) {
3727	struct btrfs_inode *inode;
3728
3729	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3730	if (ret < 0)
3731	goto out;
3732
3733	/*
3734	* if ret == 0 means we found what we were searching for, which
3735	* is weird, but possible, so only screw with path if we didn't
3736	* find the key and see if we have stuff that matches
3737	*/
3738	if (ret > 0) {
3739	ret = 0;
3740	if (path->slots[0] == 0)
3741	break;
3742	path->slots[0]--;
3743	}
3744
3745	/* pull out the item */
3746	leaf = path->nodes[0];
3747	btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
3748
3749	/* make sure the item matches what we want */
3750	if (found_key.objectid != BTRFS_ORPHAN_OBJECTID)
3751	break;
3752	if (found_key.type != BTRFS_ORPHAN_ITEM_KEY)
3753	break;
3754
3755	/* release the path since we're done with it */
3756	btrfs_release_path(path);
3757
3758	/*
3759	* this is where we are basically btrfs_lookup, without the
3760	* crossing root thing. we store the inode number in the
3761	* offset of the orphan item.
3762	*/
3763
3764	if (found_key.offset == last_objectid) {
3765	/*
3766	* We found the same inode as before. This means we were
3767	* not able to remove its items via eviction triggered
3768	* by an iput(). A transaction abort may have happened,
3769	* due to -ENOSPC for example, so try to grab the error
3770	* that lead to a transaction abort, if any.
3771	*/
3772	btrfs_err(fs_info,
3773	"Error removing orphan entry, stopping orphan cleanup");
3774	ret = BTRFS_FS_ERROR(fs_info) ?: -EINVAL;
3775	goto out;
3776	}
3777
3778	last_objectid = found_key.offset;
3779
3780	found_key.objectid = found_key.offset;
3781	found_key.type = BTRFS_INODE_ITEM_KEY;
3782	found_key.offset = 0;
3783	inode = btrfs_iget(last_objectid, root);
3784	if (IS_ERR(inode)) {
3785	ret = PTR_ERR(inode);
3786	inode = NULL;
3787	if (ret != -ENOENT)
3788	goto out;
3789	}
3790
3791	if (!inode && root == fs_info->tree_root) {
3792	struct btrfs_root *dead_root;
3793	bool is_dead_root = false;
3794
3795	/*
3796	* This is an orphan in the tree root. Currently these
3797	* could come from 2 sources:
3798	* a) a root (snapshot/subvolume) deletion in progress
3799	* b) a free space cache inode
3800	* We need to distinguish those two, as the orphan item
3801	* for a root must not get deleted before the deletion
3802	* of the snapshot/subvolume's tree completes.
3803	*
3804	* btrfs_find_orphan_roots() ran before us, which has
3805	* found all deleted roots and loaded them into
3806	* fs_info->fs_roots_radix. So here we can find if an
3807	* orphan item corresponds to a deleted root by looking
3808	* up the root from that radix tree.
3809	*/
3810
3811	spin_lock(&fs_info->fs_roots_radix_lock);
3812	dead_root = radix_tree_lookup(&fs_info->fs_roots_radix,
3813	(unsigned long)found_key.objectid);
3814	if (dead_root && btrfs_root_refs(&dead_root->root_item) == 0)
3815	is_dead_root = true;
3816	spin_unlock(&fs_info->fs_roots_radix_lock);
3817
3818	if (is_dead_root) {
3819	/* prevent this orphan from being found again */
3820	key.offset = found_key.objectid - 1;
3821	continue;
3822	}
3823
3824	}
3825
3826	/*
3827	* If we have an inode with links, there are a couple of
3828	* possibilities:
3829	*
3830	* 1. We were halfway through creating fsverity metadata for the
3831	* file. In that case, the orphan item represents incomplete
3832	* fsverity metadata which must be cleaned up with
3833	* btrfs_drop_verity_items and deleting the orphan item.
3834
3835	* 2. Old kernels (before v3.12) used to create an
3836	* orphan item for truncate indicating that there were possibly
3837	* extent items past i_size that needed to be deleted. In v3.12,
3838	* truncate was changed to update i_size in sync with the extent
3839	* items, but the (useless) orphan item was still created. Since
3840	* v4.18, we don't create the orphan item for truncate at all.
3841	*
3842	* So, this item could mean that we need to do a truncate, but
3843	* only if this filesystem was last used on a pre-v3.12 kernel
3844	* and was not cleanly unmounted. The odds of that are quite
3845	* slim, and it's a pain to do the truncate now, so just delete
3846	* the orphan item.
3847	*
3848	* It's also possible that this orphan item was supposed to be
3849	* deleted but wasn't. The inode number may have been reused,
3850	* but either way, we can delete the orphan item.
3851	*/
3852	if (!inode \|\| inode->vfs_inode.i_nlink) {
3853	if (inode) {
3854	ret = btrfs_drop_verity_items(inode);
3855	iput(&inode->vfs_inode);
3856	inode = NULL;
3857	if (ret)
3858	goto out;
3859	}
3860	/* Only deletes the orphan. */
3861	trans = btrfs_start_transaction_fallback_global_rsv(root, 1);
3862	if (IS_ERR(trans)) {
3863	ret = PTR_ERR(trans);
3864	goto out;
3865	}
3866	btrfs_debug(fs_info, "auto deleting %Lu",
3867	found_key.objectid);
3868	ret = btrfs_del_orphan_item(trans, root,
3869	found_key.objectid);
3870	btrfs_end_transaction(trans);
3871	if (ret)
3872	goto out;
3873	continue;
3874	}
3875
3876	nr_unlink++;
3877
3878	/* this will do delete_inode and everything for us */
3879	iput(&inode->vfs_inode);
3880	}
3881	/* release the path since we're done with it */
3882	btrfs_release_path(path);
3883
3884	if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) {
3885	trans = btrfs_join_transaction(root);
3886	if (!IS_ERR(trans))
3887	btrfs_end_transaction(trans);
3888	}
3889
3890	if (nr_unlink)
3891	btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink);
3892
3893	out:
3894	if (ret)
3895	btrfs_err(fs_info, "could not do orphan cleanup %pe", ERR_PTR(ret));
3896	return ret;
3897	}
3898
3899	/*
3900	* Look ahead in the leaf for xattrs. If we don't find any then we know there
3901	* can't be any ACLs.
3902	*
3903	* @leaf: the eb leaf where to search
3904	* @slot: the slot the inode is in
3905	* @objectid: the objectid of the inode
3906	*
3907	* Return true if there is xattr/ACL, false otherwise.
3908	*/
3909	static noinline bool acls_after_inode_item(struct extent_buffer *leaf,
3910	int slot, u64 objectid,
3911	int *first_xattr_slot)
3912	{
3913	u32 nritems = btrfs_header_nritems(leaf);
3914	struct btrfs_key found_key;
3915	static u64 xattr_access = 0;
3916	static u64 xattr_default = 0;
3917	int scanned = 0;
3918
3919	if (!xattr_access) {
3920	xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS,
3921	strlen(XATTR_NAME_POSIX_ACL_ACCESS));
3922	xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT,
3923	strlen(XATTR_NAME_POSIX_ACL_DEFAULT));
3924	}
3925
3926	slot++;
3927	*first_xattr_slot = -1;
3928	while (slot < nritems) {
3929	btrfs_item_key_to_cpu(leaf, &found_key, slot);
3930
3931	/* We found a different objectid, there must be no ACLs. */
3932	if (found_key.objectid != objectid)
3933	return false;
3934
3935	/* We found an xattr, assume we've got an ACL. */
3936	if (found_key.type == BTRFS_XATTR_ITEM_KEY) {
3937	if (*first_xattr_slot == -1)
3938	*first_xattr_slot = slot;
3939	if (found_key.offset == xattr_access \|\|
3940	found_key.offset == xattr_default)
3941	return true;
3942	}
3943
3944	/*
3945	* We found a key greater than an xattr key, there can't be any
3946	* ACLs later on.
3947	*/
3948	if (found_key.type > BTRFS_XATTR_ITEM_KEY)
3949	return false;
3950
3951	slot++;
3952	scanned++;
3953
3954	/*
3955	* The item order goes like:
3956	* - inode
3957	* - inode backrefs
3958	* - xattrs
3959	* - extents,
3960	*
3961	* so if there are lots of hard links to an inode there can be
3962	* a lot of backrefs. Don't waste time searching too hard,
3963	* this is just an optimization.
3964	*/
3965	if (scanned >= 8)
3966	break;
3967	}
3968	/*
3969	* We hit the end of the leaf before we found an xattr or something
3970	* larger than an xattr. We have to assume the inode has ACLs.
3971	*/
3972	if (*first_xattr_slot == -1)
3973	*first_xattr_slot = slot;
3974	return true;
3975	}
3976
3977	static int btrfs_init_file_extent_tree(struct btrfs_inode *inode)
3978	{
3979	struct btrfs_fs_info *fs_info = inode->root->fs_info;
3980
3981	if (WARN_ON_ONCE(inode->file_extent_tree))
3982	return 0;
3983	if (btrfs_fs_incompat(fs_info, NO_HOLES))
3984	return 0;
3985	if (!S_ISREG(inode->vfs_inode.i_mode))
3986	return 0;
3987	if (btrfs_is_free_space_inode(inode))
3988	return 0;
3989
3990	inode->file_extent_tree = kmalloc_obj(struct extent_io_tree);
3991	if (!inode->file_extent_tree)
3992	return -ENOMEM;
3993
3994	btrfs_extent_io_tree_init(fs_info, inode->file_extent_tree,
3995	IO_TREE_INODE_FILE_EXTENT);
3996	/* Lockdep class is set only for the file extent tree. */
3997	lockdep_set_class(&inode->file_extent_tree->lock, &file_extent_tree_class);
3998
3999	return 0;
4000	}
4001
4002	static int btrfs_add_inode_to_root(struct btrfs_inode *inode, bool prealloc)
4003	{
4004	struct btrfs_root *root = inode->root;
4005	struct btrfs_inode *existing;
4006	const u64 ino = btrfs_ino(inode);
4007	int ret;
4008
4009	if (inode_unhashed(&inode->vfs_inode))
4010	return 0;
4011
4012	if (prealloc) {
4013	ret = xa_reserve(&root->inodes, ino, GFP_NOFS);
4014	if (ret)
4015	return ret;
4016	}
4017
4018	existing = xa_store(&root->inodes, ino, inode, GFP_ATOMIC);
4019
4020	if (xa_is_err(existing)) {
4021	ret = xa_err(existing);
4022	ASSERT(ret != -EINVAL);
4023	ASSERT(ret != -ENOMEM);
4024	return ret;
4025	} else if (existing) {
4026	WARN_ON(!(inode_state_read_once(&existing->vfs_inode) & (I_WILL_FREE \| I_FREEING)));
4027	}
4028
4029	return 0;
4030	}
4031
4032	/*
4033	* Read a locked inode from the btree into the in-memory inode and add it to
4034	* its root list/tree.
4035	*
4036	* On failure clean up the inode.
4037	*/
4038	static int btrfs_read_locked_inode(struct btrfs_inode inode, struct btrfs_path path)
4039	{
4040	struct btrfs_root *root = inode->root;
4041	struct btrfs_fs_info *fs_info = root->fs_info;
4042	struct extent_buffer *leaf;
4043	struct btrfs_inode_item *inode_item;
4044	struct inode *vfs_inode = &inode->vfs_inode;
4045	struct btrfs_key location;
4046	unsigned long ptr;
4047	int maybe_acls;
4048	u32 rdev;
4049	int ret;
4050	bool filled = false;
4051	int first_xattr_slot;
4052
4053	ret = btrfs_fill_inode(inode, &rdev);
4054	if (!ret)
4055	filled = true;
4056
4057	ASSERT(path);
4058
4059	btrfs_get_inode_key(inode, &location);
4060
4061	ret = btrfs_lookup_inode(NULL, root, path, &location, 0);
4062	if (ret) {
4063	/*
4064	* ret > 0 can come from btrfs_search_slot called by
4065	* btrfs_lookup_inode(), this means the inode was not found.
4066	*/
4067	if (ret > 0)
4068	ret = -ENOENT;
4069	goto out;
4070	}
4071
4072	leaf = path->nodes[0];
4073
4074	if (filled)
4075	goto cache_index;
4076
4077	inode_item = btrfs_item_ptr(leaf, path->slots[0],
4078	struct btrfs_inode_item);
4079	vfs_inode->i_mode = btrfs_inode_mode(leaf, inode_item);
4080	set_nlink(vfs_inode, btrfs_inode_nlink(leaf, inode_item));
4081	i_uid_write(vfs_inode, btrfs_inode_uid(leaf, inode_item));
4082	i_gid_write(vfs_inode, btrfs_inode_gid(leaf, inode_item));
4083	btrfs_i_size_write(inode, btrfs_inode_size(leaf, inode_item));
4084
4085	inode_set_atime(vfs_inode, btrfs_timespec_sec(leaf, &inode_item->atime),
4086	btrfs_timespec_nsec(leaf, &inode_item->atime));
4087
4088	inode_set_mtime(vfs_inode, btrfs_timespec_sec(leaf, &inode_item->mtime),
4089	btrfs_timespec_nsec(leaf, &inode_item->mtime));
4090
4091	inode_set_ctime(vfs_inode, btrfs_timespec_sec(leaf, &inode_item->ctime),
4092	btrfs_timespec_nsec(leaf, &inode_item->ctime));
4093
4094	inode->i_otime_sec = btrfs_timespec_sec(leaf, &inode_item->otime);
4095	inode->i_otime_nsec = btrfs_timespec_nsec(leaf, &inode_item->otime);
4096
4097	inode_set_bytes(vfs_inode, btrfs_inode_nbytes(leaf, inode_item));
4098	inode->generation = btrfs_inode_generation(leaf, inode_item);
4099	inode->last_trans = btrfs_inode_transid(leaf, inode_item);
4100
4101	inode_set_iversion_queried(vfs_inode, btrfs_inode_sequence(leaf, inode_item));
4102	vfs_inode->i_generation = inode->generation;
4103	vfs_inode->i_rdev = 0;
4104	rdev = btrfs_inode_rdev(leaf, inode_item);
4105
4106	if (S_ISDIR(vfs_inode->i_mode))
4107	inode->index_cnt = (u64)-1;
4108
4109	btrfs_inode_split_flags(btrfs_inode_flags(leaf, inode_item),
4110	&inode->flags, &inode->ro_flags);
4111
4112	cache_index:
4113	btrfs_update_inode_mapping_flags(inode);
4114	btrfs_set_inode_mapping_order(inode);
4115
4116	/*
4117	* If we were modified in the current generation and evicted from memory
4118	* and then re-read we need to do a full sync since we don't have any
4119	* idea about which extents were modified before we were evicted from
4120	* cache.
4121	*
4122	* This is required for both inode re-read from disk and delayed inode
4123	* in the delayed_nodes xarray.
4124	*/
4125	if (inode->last_trans == btrfs_get_fs_generation(fs_info))
4126	set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags);
4127
4128	/*
4129	* We don't persist the id of the transaction where an unlink operation
4130	* against the inode was last made. So here we assume the inode might
4131	* have been evicted, and therefore the exact value of last_unlink_trans
4132	* lost, and set it to last_trans to avoid metadata inconsistencies
4133	* between the inode and its parent if the inode is fsync'ed and the log
4134	* replayed. For example, in the scenario:
4135	*
4136	* touch mydir/foo
4137	* ln mydir/foo mydir/bar
4138	* sync
4139	* unlink mydir/bar
4140	* echo 2 > /proc/sys/vm/drop_caches # evicts inode
4141	* xfs_io -c fsync mydir/foo
4142	* <power failure>
4143	* mount fs, triggers fsync log replay
4144	*
4145	* We must make sure that when we fsync our inode foo we also log its
4146	* parent inode, otherwise after log replay the parent still has the
4147	* dentry with the "bar" name but our inode foo has a link count of 1
4148	* and doesn't have an inode ref with the name "bar" anymore.
4149	*
4150	* Setting last_unlink_trans to last_trans is a pessimistic approach,
4151	* but it guarantees correctness at the expense of occasional full
4152	* transaction commits on fsync if our inode is a directory, or if our
4153	* inode is not a directory, logging its parent unnecessarily.
4154	*/
4155	inode->last_unlink_trans = inode->last_trans;
4156
4157	/*
4158	* Same logic as for last_unlink_trans. We don't persist the generation
4159	* of the last transaction where this inode was used for a reflink
4160	* operation, so after eviction and reloading the inode we must be
4161	* pessimistic and assume the last transaction that modified the inode.
4162	*/
4163	inode->last_reflink_trans = inode->last_trans;
4164
4165	path->slots[0]++;
4166	if (vfs_inode->i_nlink != 1 \|\|
4167	path->slots[0] >= btrfs_header_nritems(leaf))
4168	goto cache_acl;
4169
4170	btrfs_item_key_to_cpu(leaf, &location, path->slots[0]);
4171	if (location.objectid != btrfs_ino(inode))
4172	goto cache_acl;
4173
4174	ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
4175	if (location.type == BTRFS_INODE_REF_KEY) {
4176	struct btrfs_inode_ref *ref;
4177
4178	ref = (struct btrfs_inode_ref *)ptr;
4179	inode->dir_index = btrfs_inode_ref_index(leaf, ref);
4180	} else if (location.type == BTRFS_INODE_EXTREF_KEY) {
4181	struct btrfs_inode_extref *extref;
4182
4183	extref = (struct btrfs_inode_extref *)ptr;
4184	inode->dir_index = btrfs_inode_extref_index(leaf, extref);
4185	}
4186	cache_acl:
4187	/*
4188	* try to precache a NULL acl entry for files that don't have
4189	* any xattrs or acls
4190	*/
4191	maybe_acls = acls_after_inode_item(leaf, path->slots[0],
4192	btrfs_ino(inode), &first_xattr_slot);
4193	if (first_xattr_slot != -1) {
4194	path->slots[0] = first_xattr_slot;
4195	ret = btrfs_load_inode_props(inode, path);
4196	if (ret)
4197	btrfs_err(fs_info,
4198	"error loading props for ino %llu (root %llu): %pe",
4199	btrfs_ino(inode), btrfs_root_id(root), ERR_PTR(ret));
4200	}
4201
4202	/*
4203	* We don't need the path anymore, so release it to avoid holding a read
4204	* lock on a leaf while calling btrfs_init_file_extent_tree(), which can
4205	* allocate memory that triggers reclaim (GFP_KERNEL) and cause a locking
4206	* dependency.
4207	*/
4208	btrfs_release_path(path);
4209
4210	ret = btrfs_init_file_extent_tree(inode);
4211	if (ret)
4212	goto out;
4213	btrfs_inode_set_file_extent_range(inode, 0,
4214	round_up(i_size_read(vfs_inode), fs_info->sectorsize));
4215
4216	if (!maybe_acls)
4217	cache_no_acl(vfs_inode);
4218
4219	switch (vfs_inode->i_mode & S_IFMT) {
4220	case S_IFREG:
4221	vfs_inode->i_mapping->a_ops = &btrfs_aops;
4222	vfs_inode->i_fop = &btrfs_file_operations;
4223	vfs_inode->i_op = &btrfs_file_inode_operations;
4224	break;
4225	case S_IFDIR:
4226	vfs_inode->i_fop = &btrfs_dir_file_operations;
4227	vfs_inode->i_op = &btrfs_dir_inode_operations;
4228	break;
4229	case S_IFLNK:
4230	vfs_inode->i_op = &btrfs_symlink_inode_operations;
4231	inode_nohighmem(vfs_inode);
4232	vfs_inode->i_mapping->a_ops = &btrfs_aops;
4233	break;
4234	default:
4235	vfs_inode->i_op = &btrfs_special_inode_operations;
4236	init_special_inode(vfs_inode, vfs_inode->i_mode, rdev);
4237	break;
4238	}
4239
4240	btrfs_sync_inode_flags_to_i_flags(inode);
4241
4242	ret = btrfs_add_inode_to_root(inode, true);
4243	if (ret)
4244	goto out;
4245
4246	return 0;
4247	out:
4248	/*
4249	* We may have a read locked leaf and iget_failed() triggers inode
4250	* eviction which needs to release the delayed inode and that needs
4251	* to lock the delayed inode's mutex. This can cause a ABBA deadlock
4252	* with a task running delayed items, as that require first locking
4253	* the delayed inode's mutex and then modifying its subvolume btree.
4254	* So release the path before iget_failed().
4255	*/
4256	btrfs_release_path(path);
4257	iget_failed(vfs_inode);
4258	return ret;
4259	}
4260
4261	/*
4262	* given a leaf and an inode, copy the inode fields into the leaf
4263	*/
4264	static void fill_inode_item(struct btrfs_trans_handle *trans,
4265	struct extent_buffer *leaf,
4266	struct btrfs_inode_item *item,
4267	struct inode *inode)
4268	{
4269	u64 flags;
4270
4271	btrfs_set_inode_uid(leaf, item, i_uid_read(inode));
4272	btrfs_set_inode_gid(leaf, item, i_gid_read(inode));
4273	btrfs_set_inode_size(leaf, item, BTRFS_I(inode)->disk_i_size);
4274	btrfs_set_inode_mode(leaf, item, inode->i_mode);
4275	btrfs_set_inode_nlink(leaf, item, inode->i_nlink);
4276
4277	btrfs_set_timespec_sec(leaf, &item->atime, inode_get_atime_sec(inode));
4278	btrfs_set_timespec_nsec(leaf, &item->atime, inode_get_atime_nsec(inode));
4279
4280	btrfs_set_timespec_sec(leaf, &item->mtime, inode_get_mtime_sec(inode));
4281	btrfs_set_timespec_nsec(leaf, &item->mtime, inode_get_mtime_nsec(inode));
4282
4283	btrfs_set_timespec_sec(leaf, &item->ctime, inode_get_ctime_sec(inode));
4284	btrfs_set_timespec_nsec(leaf, &item->ctime, inode_get_ctime_nsec(inode));
4285
4286	btrfs_set_timespec_sec(leaf, &item->otime, BTRFS_I(inode)->i_otime_sec);
4287	btrfs_set_timespec_nsec(leaf, &item->otime, BTRFS_I(inode)->i_otime_nsec);
4288
4289	btrfs_set_inode_nbytes(leaf, item, inode_get_bytes(inode));
4290	btrfs_set_inode_generation(leaf, item, BTRFS_I(inode)->generation);
4291	btrfs_set_inode_sequence(leaf, item, inode_peek_iversion(inode));
4292	btrfs_set_inode_transid(leaf, item, trans->transid);
4293	btrfs_set_inode_rdev(leaf, item, inode->i_rdev);
4294	flags = btrfs_inode_combine_flags(BTRFS_I(inode)->flags,
4295	BTRFS_I(inode)->ro_flags);
4296	btrfs_set_inode_flags(leaf, item, flags);
4297	btrfs_set_inode_block_group(leaf, item, 0);
4298	}
4299
4300	/*
4301	* copy everything in the in-memory inode into the btree.
4302	*/
4303	static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans,
4304	struct btrfs_inode *inode)
4305	{
4306	struct btrfs_inode_item *inode_item;
4307	BTRFS_PATH_AUTO_FREE(path);
4308	struct extent_buffer *leaf;
4309	struct btrfs_key key;
4310	int ret;
4311
4312	path = btrfs_alloc_path();
4313	if (!path)
4314	return -ENOMEM;
4315
4316	btrfs_get_inode_key(inode, &key);
4317	ret = btrfs_lookup_inode(trans, inode->root, path, &key, 1);
4318	if (ret) {
4319	if (ret > 0)
4320	ret = -ENOENT;
4321	return ret;
4322	}
4323
4324	leaf = path->nodes[0];
4325	inode_item = btrfs_item_ptr(leaf, path->slots[0],
4326	struct btrfs_inode_item);
4327
4328	fill_inode_item(trans, leaf, inode_item, &inode->vfs_inode);
4329	btrfs_set_inode_last_trans(trans, inode);
4330	return 0;
4331	}
4332
4333	/*
4334	* copy everything in the in-memory inode into the btree.
4335	*/
4336	int btrfs_update_inode(struct btrfs_trans_handle *trans,
4337	struct btrfs_inode *inode)
4338	{
4339	struct btrfs_root *root = inode->root;
4340	struct btrfs_fs_info *fs_info = root->fs_info;
4341	int ret;
4342
4343	/*
4344	* If the inode is a free space inode, we can deadlock during commit
4345	* if we put it into the delayed code.
4346	*
4347	* The data relocation inode should also be directly updated
4348	* without delay
4349	*/
4350	if (!btrfs_is_free_space_inode(inode)
4351	&& !btrfs_is_data_reloc_root(root)
4352	&& !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
4353	btrfs_update_root_times(trans, root);
4354
4355	ret = btrfs_delayed_update_inode(trans, inode);
4356	if (!ret)
4357	btrfs_set_inode_last_trans(trans, inode);
4358	return ret;
4359	}
4360
4361	return btrfs_update_inode_item(trans, inode);
4362	}
4363
4364	int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans,
4365	struct btrfs_inode *inode)
4366	{
4367	int ret;
4368
4369	ret = btrfs_update_inode(trans, inode);
4370	if (ret == -ENOSPC)
4371	return btrfs_update_inode_item(trans, inode);
4372	return ret;
4373	}
4374
4375	static void update_time_after_link_or_unlink(struct btrfs_inode *dir)
4376	{
4377	struct timespec64 now;
4378
4379	/*
4380	* If we are replaying a log tree, we do not want to update the mtime
4381	* and ctime of the parent directory with the current time, since the
4382	* log replay procedure is responsible for setting them to their correct
4383	* values (the ones it had when the fsync was done).
4384	*/
4385	if (test_bit(BTRFS_FS_LOG_RECOVERING, &dir->root->fs_info->flags))
4386	return;
4387
4388	now = inode_set_ctime_current(&dir->vfs_inode);
4389	inode_set_mtime_to_ts(&dir->vfs_inode, now);
4390	}
4391
4392	/*
4393	* unlink helper that gets used here in inode.c and in the tree logging
4394	* recovery code. It remove a link in a directory with a given name, and
4395	* also drops the back refs in the inode to the directory
4396	*/
4397	static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans,
4398	struct btrfs_inode *dir,
4399	struct btrfs_inode *inode,
4400	const struct fscrypt_str *name,
4401	struct btrfs_rename_ctx *rename_ctx)
4402	{
4403	struct btrfs_root *root = dir->root;
4404	struct btrfs_fs_info *fs_info = root->fs_info;
4405	struct btrfs_path *path;
4406	int ret = 0;
4407	struct btrfs_dir_item *di;
4408	u64 index;
4409	u64 ino = btrfs_ino(inode);
4410	u64 dir_ino = btrfs_ino(dir);
4411
4412	path = btrfs_alloc_path();
4413	if (!path)
4414	return -ENOMEM;
4415
4416	di = btrfs_lookup_dir_item(trans, root, path, dir_ino, name, -1);
4417	if (IS_ERR_OR_NULL(di)) {
4418	btrfs_free_path(path);
4419	return di ? PTR_ERR(di) : -ENOENT;
4420	}
4421	ret = btrfs_delete_one_dir_name(trans, root, path, di);
4422	/*
4423	* Down the call chains below we'll also need to allocate a path, so no
4424	* need to hold on to this one for longer than necessary.
4425	*/
4426	btrfs_free_path(path);
4427	if (ret)
4428	return ret;
4429
4430	/*
4431	* If we don't have dir index, we have to get it by looking up
4432	* the inode ref, since we get the inode ref, remove it directly,
4433	* it is unnecessary to do delayed deletion.
4434	*
4435	* But if we have dir index, needn't search inode ref to get it.
4436	* Since the inode ref is close to the inode item, it is better
4437	* that we delay to delete it, and just do this deletion when
4438	* we update the inode item.
4439	*/
4440	if (inode->dir_index) {
4441	ret = btrfs_delayed_delete_inode_ref(inode);
4442	if (!ret) {
4443	index = inode->dir_index;
4444	goto skip_backref;
4445	}
4446	}
4447
4448	ret = btrfs_del_inode_ref(trans, root, name, ino, dir_ino, &index);
4449	if (unlikely(ret)) {
4450	btrfs_crit(fs_info,
4451	"failed to delete reference to %.*s, root %llu inode %llu parent %llu",
4452	name->len, name->name, btrfs_root_id(root), ino, dir_ino);
4453	btrfs_abort_transaction(trans, ret);
4454	return ret;
4455	}
4456	skip_backref:
4457	if (rename_ctx)
4458	rename_ctx->index = index;
4459
4460	ret = btrfs_delete_delayed_dir_index(trans, dir, index);
4461	if (unlikely(ret)) {
4462	btrfs_abort_transaction(trans, ret);
4463	return ret;
4464	}
4465
4466	/*
4467	* If we are in a rename context, we don't need to update anything in the
4468	* log. That will be done later during the rename by btrfs_log_new_name().
4469	* Besides that, doing it here would only cause extra unnecessary btree
4470	* operations on the log tree, increasing latency for applications.
4471	*/
4472	if (!rename_ctx) {
4473	btrfs_del_inode_ref_in_log(trans, name, inode, dir);
4474	btrfs_del_dir_entries_in_log(trans, name, dir, index);
4475	}
4476
4477	/*
4478	* If we have a pending delayed iput we could end up with the final iput
4479	* being run in btrfs-cleaner context. If we have enough of these built
4480	* up we can end up burning a lot of time in btrfs-cleaner without any
4481	* way to throttle the unlinks. Since we're currently holding a ref on
4482	* the inode we can run the delayed iput here without any issues as the
4483	* final iput won't be done until after we drop the ref we're currently
4484	* holding.
4485	*/
4486	btrfs_run_delayed_iput(fs_info, inode);
4487
4488	btrfs_i_size_write(dir, dir->vfs_inode.i_size - name->len * 2);
4489	inode_inc_iversion(&inode->vfs_inode);
4490	inode_set_ctime_current(&inode->vfs_inode);
4491	inode_inc_iversion(&dir->vfs_inode);
4492	update_time_after_link_or_unlink(dir);
4493
4494	return btrfs_update_inode(trans, dir);
4495	}
4496
4497	int btrfs_unlink_inode(struct btrfs_trans_handle *trans,
4498	struct btrfs_inode dir, struct btrfs_inode inode,
4499	const struct fscrypt_str *name)
4500	{
4501	int ret;
4502
4503	ret = __btrfs_unlink_inode(trans, dir, inode, name, NULL);
4504	if (!ret) {
4505	drop_nlink(&inode->vfs_inode);
4506	ret = btrfs_update_inode(trans, inode);
4507	}
4508	return ret;
4509	}
4510
4511	/*
4512	* helper to start transaction for unlink and rmdir.
4513	*
4514	* unlink and rmdir are special in btrfs, they do not always free space, so
4515	* if we cannot make our reservations the normal way try and see if there is
4516	* plenty of slack room in the global reserve to migrate, otherwise we cannot
4517	* allow the unlink to occur.
4518	*/
4519	static struct btrfs_trans_handle __unlink_start_trans(struct btrfs_inode dir)
4520	{
4521	struct btrfs_root *root = dir->root;
4522
4523	return btrfs_start_transaction_fallback_global_rsv(root,
4524	BTRFS_UNLINK_METADATA_UNITS);
4525	}
4526
4527	static int btrfs_unlink(struct inode dir, struct dentry dentry)
4528	{
4529	struct btrfs_trans_handle *trans;
4530	struct inode *inode = d_inode(dentry);
4531	int ret;
4532	struct fscrypt_name fname;
4533
4534	ret = fscrypt_setup_filename(dir, &dentry->d_name, 1, &fname);
4535	if (ret)
4536	return ret;
4537
4538	/* This needs to handle no-key deletions later on */
4539
4540	trans = __unlink_start_trans(BTRFS_I(dir));
4541	if (IS_ERR(trans)) {
4542	ret = PTR_ERR(trans);
4543	goto fscrypt_free;
4544	}
4545
4546	btrfs_record_unlink_dir(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)),
4547	false);
4548
4549	ret = btrfs_unlink_inode(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)),
4550	&fname.disk_name);
4551	if (ret)
4552	goto end_trans;
4553
4554	if (inode->i_nlink == 0) {
4555	ret = btrfs_orphan_add(trans, BTRFS_I(inode));
4556	if (ret)
4557	goto end_trans;
4558	}
4559
4560	end_trans:
4561	btrfs_end_transaction(trans);
4562	btrfs_btree_balance_dirty(BTRFS_I(dir)->root->fs_info);
4563	fscrypt_free:
4564	fscrypt_free_filename(&fname);
4565	return ret;
4566	}
4567
4568	static int btrfs_unlink_subvol(struct btrfs_trans_handle *trans,
4569	struct btrfs_inode dir, struct dentry dentry)
4570	{
4571	struct btrfs_root *root = dir->root;
4572	struct btrfs_inode *inode = BTRFS_I(d_inode(dentry));
4573	BTRFS_PATH_AUTO_FREE(path);
4574	struct extent_buffer *leaf;
4575	struct btrfs_dir_item *di;
4576	struct btrfs_key key;
4577	u64 index;
4578	int ret;
4579	u64 objectid;
4580	u64 dir_ino = btrfs_ino(dir);
4581	struct fscrypt_name fname;
4582
4583	ret = fscrypt_setup_filename(&dir->vfs_inode, &dentry->d_name, 1, &fname);
4584	if (ret)
4585	return ret;
4586
4587	/* This needs to handle no-key deletions later on */
4588
4589	if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID) {
4590	objectid = btrfs_root_id(inode->root);
4591	} else if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) {
4592	objectid = inode->ref_root_id;
4593	} else {
4594	WARN_ON(1);
4595	fscrypt_free_filename(&fname);
4596	return -EINVAL;
4597	}
4598
4599	path = btrfs_alloc_path();
4600	if (!path) {
4601	ret = -ENOMEM;
4602	goto out;
4603	}
4604
4605	di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
4606	&fname.disk_name, -1);
4607	if (IS_ERR_OR_NULL(di)) {
4608	ret = di ? PTR_ERR(di) : -ENOENT;
4609	goto out;
4610	}
4611
4612	leaf = path->nodes[0];
4613	btrfs_dir_item_key_to_cpu(leaf, di, &key);
4614	WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY \|\| key.objectid != objectid);
4615	ret = btrfs_delete_one_dir_name(trans, root, path, di);
4616	if (unlikely(ret)) {
4617	btrfs_abort_transaction(trans, ret);
4618	goto out;
4619	}
4620	btrfs_release_path(path);
4621
4622	/*
4623	* This is a placeholder inode for a subvolume we didn't have a
4624	* reference to at the time of the snapshot creation. In the meantime
4625	* we could have renamed the real subvol link into our snapshot, so
4626	* depending on btrfs_del_root_ref to return -ENOENT here is incorrect.
4627	* Instead simply lookup the dir_index_item for this entry so we can
4628	* remove it. Otherwise we know we have a ref to the root and we can
4629	* call btrfs_del_root_ref, and it _shouldn't_ fail.
4630	*/
4631	if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) {
4632	di = btrfs_search_dir_index_item(root, path, dir_ino, &fname.disk_name);
4633	if (IS_ERR(di)) {
4634	ret = PTR_ERR(di);
4635	btrfs_abort_transaction(trans, ret);
4636	goto out;
4637	}
4638
4639	leaf = path->nodes[0];
4640	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
4641	index = key.offset;
4642	btrfs_release_path(path);
4643	} else {
4644	ret = btrfs_del_root_ref(trans, objectid,
4645	btrfs_root_id(root), dir_ino,
4646	&index, &fname.disk_name);
4647	if (unlikely(ret)) {
4648	btrfs_abort_transaction(trans, ret);
4649	goto out;
4650	}
4651	}
4652
4653	ret = btrfs_delete_delayed_dir_index(trans, dir, index);
4654	if (unlikely(ret)) {
4655	btrfs_abort_transaction(trans, ret);
4656	goto out;
4657	}
4658
4659	btrfs_i_size_write(dir, dir->vfs_inode.i_size - fname.disk_name.len * 2);
4660	inode_inc_iversion(&dir->vfs_inode);
4661	inode_set_mtime_to_ts(&dir->vfs_inode, inode_set_ctime_current(&dir->vfs_inode));
4662	ret = btrfs_update_inode_fallback(trans, dir);
4663	if (ret)
4664	btrfs_abort_transaction(trans, ret);
4665	out:
4666	fscrypt_free_filename(&fname);
4667	return ret;
4668	}
4669
4670	/*
4671	* Helper to check if the subvolume references other subvolumes or if it's
4672	* default.
4673	*/
4674	static noinline int may_destroy_subvol(struct btrfs_root *root)
4675	{
4676	struct btrfs_fs_info *fs_info = root->fs_info;
4677	BTRFS_PATH_AUTO_FREE(path);
4678	struct btrfs_dir_item *di;
4679	struct btrfs_key key;
4680	struct fscrypt_str name = FSTR_INIT("default", 7);
4681	u64 dir_id;
4682	int ret;
4683
4684	path = btrfs_alloc_path();
4685	if (!path)
4686	return -ENOMEM;
4687
4688	/* Make sure this root isn't set as the default subvol */
4689	dir_id = btrfs_super_root_dir(fs_info->super_copy);
4690	di = btrfs_lookup_dir_item(NULL, fs_info->tree_root, path,
4691	dir_id, &name, 0);
4692	if (!IS_ERR_OR_NULL(di)) {
4693	btrfs_dir_item_key_to_cpu(path->nodes[0], di, &key);
4694	if (key.objectid == btrfs_root_id(root)) {
4695	ret = -EPERM;
4696	btrfs_err(fs_info,
4697	"deleting default subvolume %llu is not allowed",
4698	key.objectid);
4699	return ret;
4700	}
4701	btrfs_release_path(path);
4702	}
4703
4704	key.objectid = btrfs_root_id(root);
4705	key.type = BTRFS_ROOT_REF_KEY;
4706	key.offset = (u64)-1;
4707
4708	ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
4709	if (ret < 0)
4710	return ret;
4711	if (unlikely(ret == 0)) {
4712	/*
4713	* Key with offset -1 found, there would have to exist a root
4714	* with such id, but this is out of valid range.
4715	*/
4716	return -EUCLEAN;
4717	}
4718
4719	ret = 0;
4720	if (path->slots[0] > 0) {
4721	path->slots[0]--;
4722	btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
4723	if (key.objectid == btrfs_root_id(root) && key.type == BTRFS_ROOT_REF_KEY)
4724	ret = -ENOTEMPTY;
4725	}
4726
4727	return ret;
4728	}
4729
4730	/* Delete all dentries for inodes belonging to the root */
4731	static void btrfs_prune_dentries(struct btrfs_root *root)
4732	{
4733	struct btrfs_fs_info *fs_info = root->fs_info;
4734	struct btrfs_inode *inode;
4735	u64 min_ino = 0;
4736
4737	if (!BTRFS_FS_ERROR(fs_info))
4738	WARN_ON(btrfs_root_refs(&root->root_item) != 0);
4739
4740	inode = btrfs_find_first_inode(root, min_ino);
4741	while (inode) {
4742	if (icount_read_once(&inode->vfs_inode) > 1)
4743	d_prune_aliases(&inode->vfs_inode);
4744
4745	min_ino = btrfs_ino(inode) + 1;
4746	/*
4747	* btrfs_drop_inode() will have it removed from the inode
4748	* cache when its usage count hits zero.
4749	*/
4750	iput(&inode->vfs_inode);
4751	cond_resched();
4752	inode = btrfs_find_first_inode(root, min_ino);
4753	}
4754	}
4755
4756	int btrfs_delete_subvolume(struct btrfs_inode dir, struct dentry dentry)
4757	{
4758	struct btrfs_root *root = dir->root;
4759	struct btrfs_fs_info *fs_info = root->fs_info;
4760	struct inode *inode = d_inode(dentry);
4761	struct btrfs_root *dest = BTRFS_I(inode)->root;
4762	struct btrfs_trans_handle *trans;
4763	struct btrfs_block_rsv block_rsv;
4764	u64 root_flags;
4765	u64 qgroup_reserved = 0;
4766	int ret;
4767
4768	down_write(&fs_info->subvol_sem);
4769
4770	/*
4771	* Don't allow to delete a subvolume with send in progress. This is
4772	* inside the inode lock so the error handling that has to drop the bit
4773	* again is not run concurrently.
4774	*/
4775	spin_lock(&dest->root_item_lock);
4776	if (dest->send_in_progress) {
4777	spin_unlock(&dest->root_item_lock);
4778	btrfs_warn(fs_info,
4779	"attempt to delete subvolume %llu during send",
4780	btrfs_root_id(dest));
4781	ret = -EPERM;
4782	goto out_up_write;
4783	}
4784	if (atomic_read(&dest->nr_swapfiles)) {
4785	spin_unlock(&dest->root_item_lock);
4786	btrfs_warn(fs_info,
4787	"attempt to delete subvolume %llu with active swapfile",
4788	btrfs_root_id(dest));
4789	ret = -EPERM;
4790	goto out_up_write;
4791	}
4792	root_flags = btrfs_root_flags(&dest->root_item);
4793	btrfs_set_root_flags(&dest->root_item,
4794	root_flags \| BTRFS_ROOT_SUBVOL_DEAD);
4795	spin_unlock(&dest->root_item_lock);
4796
4797	ret = may_destroy_subvol(dest);
4798	if (ret)
4799	goto out_undead;
4800
4801	btrfs_init_block_rsv(&block_rsv, BTRFS_BLOCK_RSV_TEMP);
4802	/*
4803	* One for dir inode,
4804	* two for dir entries,
4805	* two for root ref/backref.
4806	*/
4807	ret = btrfs_subvolume_reserve_metadata(root, &block_rsv, 5, true);
4808	if (ret)
4809	goto out_undead;
4810	qgroup_reserved = block_rsv.qgroup_rsv_reserved;
4811
4812	trans = btrfs_start_transaction(root, 0);
4813	if (IS_ERR(trans)) {
4814	ret = PTR_ERR(trans);
4815	goto out_release;
4816	}
4817	btrfs_qgroup_convert_reserved_meta(root, qgroup_reserved);
4818	qgroup_reserved = 0;
4819	trans->block_rsv = &block_rsv;
4820	trans->bytes_reserved = block_rsv.size;
4821
4822	btrfs_record_snapshot_destroy(trans, dir);
4823
4824	ret = btrfs_unlink_subvol(trans, dir, dentry);
4825	if (unlikely(ret)) {
4826	btrfs_abort_transaction(trans, ret);
4827	goto out_end_trans;
4828	}
4829
4830	ret = btrfs_record_root_in_trans(trans, dest);
4831	if (unlikely(ret)) {
4832	btrfs_abort_transaction(trans, ret);
4833	goto out_end_trans;
4834	}
4835
4836	memset(&dest->root_item.drop_progress, 0,
4837	sizeof(dest->root_item.drop_progress));
4838	btrfs_set_root_drop_level(&dest->root_item, 0);
4839	btrfs_set_root_refs(&dest->root_item, 0);
4840
4841	if (!test_and_set_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &dest->state)) {
4842	ret = btrfs_insert_orphan_item(trans,
4843	fs_info->tree_root,
4844	btrfs_root_id(dest));
4845	if (unlikely(ret)) {
4846	btrfs_abort_transaction(trans, ret);
4847	goto out_end_trans;
4848	}
4849	}
4850
4851	ret = btrfs_uuid_tree_remove(trans, dest->root_item.uuid,
4852	BTRFS_UUID_KEY_SUBVOL, btrfs_root_id(dest));
4853	if (unlikely(ret && ret != -ENOENT)) {
4854	btrfs_abort_transaction(trans, ret);
4855	goto out_end_trans;
4856	}
4857	if (!btrfs_is_empty_uuid(dest->root_item.received_uuid)) {
4858	ret = btrfs_uuid_tree_remove(trans,
4859	dest->root_item.received_uuid,
4860	BTRFS_UUID_KEY_RECEIVED_SUBVOL,
4861	btrfs_root_id(dest));
4862	if (unlikely(ret && ret != -ENOENT)) {
4863	btrfs_abort_transaction(trans, ret);
4864	goto out_end_trans;
4865	}
4866	}
4867
4868	free_anon_bdev(dest->anon_dev);
4869	dest->anon_dev = 0;
4870	out_end_trans:
4871	trans->block_rsv = NULL;
4872	trans->bytes_reserved = 0;
4873	ret = btrfs_end_transaction(trans);
4874	inode->i_flags \|= S_DEAD;
4875	out_release:
4876	btrfs_block_rsv_release(fs_info, &block_rsv, (u64)-1, NULL);
4877	if (qgroup_reserved)
4878	btrfs_qgroup_free_meta_prealloc(root, qgroup_reserved);
4879	out_undead:
4880	if (ret) {
4881	spin_lock(&dest->root_item_lock);
4882	root_flags = btrfs_root_flags(&dest->root_item);
4883	btrfs_set_root_flags(&dest->root_item,
4884	root_flags & ~BTRFS_ROOT_SUBVOL_DEAD);
4885	spin_unlock(&dest->root_item_lock);
4886	}
4887	out_up_write:
4888	up_write(&fs_info->subvol_sem);
4889	if (!ret) {
4890	d_invalidate(dentry);
4891	btrfs_prune_dentries(dest);
4892	ASSERT(dest->send_in_progress == 0);
4893	}
4894
4895	return ret;
4896	}
4897
4898	static int btrfs_rmdir(struct inode vfs_dir, struct dentry dentry)
4899	{
4900	struct btrfs_inode *dir = BTRFS_I(vfs_dir);
4901	struct btrfs_inode *inode = BTRFS_I(d_inode(dentry));
4902	struct btrfs_fs_info *fs_info = inode->root->fs_info;
4903	int ret = 0;
4904	struct btrfs_trans_handle *trans;
4905	struct fscrypt_name fname;
4906
4907	if (inode->vfs_inode.i_size > BTRFS_EMPTY_DIR_SIZE)
4908	return -ENOTEMPTY;
4909	if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID) {
4910	if (unlikely(btrfs_fs_incompat(fs_info, EXTENT_TREE_V2))) {
4911	btrfs_err(fs_info,
4912	"extent tree v2 doesn't support snapshot deletion yet");
4913	return -EOPNOTSUPP;
4914	}
4915	return btrfs_delete_subvolume(dir, dentry);
4916	}
4917
4918	ret = fscrypt_setup_filename(vfs_dir, &dentry->d_name, 1, &fname);
4919	if (ret)
4920	return ret;
4921
4922	/* This needs to handle no-key deletions later on */
4923
4924	trans = __unlink_start_trans(dir);
4925	if (IS_ERR(trans)) {
4926	ret = PTR_ERR(trans);
4927	goto out_notrans;
4928	}
4929
4930	/*
4931	* Propagate the last_unlink_trans value of the deleted dir to its
4932	* parent directory. This is to prevent an unrecoverable log tree in the
4933	* case we do something like this:
4934	* 1) create dir foo
4935	* 2) create snapshot under dir foo
4936	* 3) delete the snapshot
4937	* 4) rmdir foo
4938	* 5) mkdir foo
4939	* 6) fsync foo or some file inside foo
4940	*
4941	* This is because we can't unlink other roots when replaying the dir
4942	* deletes for directory foo.
4943	*/
4944	if (inode->last_unlink_trans >= trans->transid)
4945	btrfs_record_snapshot_destroy(trans, dir);
4946
4947	if (unlikely(btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
4948	ret = btrfs_unlink_subvol(trans, dir, dentry);
4949	goto out;
4950	}
4951
4952	ret = btrfs_orphan_add(trans, inode);
4953	if (ret)
4954	goto out;
4955
4956	btrfs_record_unlink_dir(trans, dir, inode, false);
4957
4958	/* now the directory is empty */
4959	ret = btrfs_unlink_inode(trans, dir, inode, &fname.disk_name);
4960	if (!ret)
4961	btrfs_i_size_write(inode, 0);
4962	out:
4963	btrfs_end_transaction(trans);
4964	out_notrans:
4965	btrfs_btree_balance_dirty(fs_info);
4966	fscrypt_free_filename(&fname);
4967
4968	return ret;
4969	}
4970
4971	static bool is_inside_block(u64 bytenr, u64 blockstart, u32 blocksize)
4972	{
4973	ASSERT(IS_ALIGNED(blockstart, blocksize), "blockstart=%llu blocksize=%u",
4974	blockstart, blocksize);
4975
4976	if (blockstart <= bytenr && bytenr <= blockstart + blocksize - 1)
4977	return true;
4978	return false;
4979	}
4980
4981	static int truncate_block_zero_beyond_eof(struct btrfs_inode *inode, u64 start)
4982	{
4983	const pgoff_t index = (start >> PAGE_SHIFT);
4984	struct address_space *mapping = inode->vfs_inode.i_mapping;
4985	struct folio *folio;
4986	u64 zero_start;
4987	u64 zero_end;
4988	int ret = 0;
4989
4990	again:
4991	folio = filemap_lock_folio(mapping, index);
4992	/* No folio present. */
4993	if (IS_ERR(folio))
4994	return 0;
4995
4996	if (!folio_test_uptodate(folio)) {
4997	ret = btrfs_read_folio(NULL, folio);
4998	folio_lock(folio);
4999	if (folio->mapping != mapping) {
5000	folio_unlock(folio);
5001	folio_put(folio);
5002	goto again;
5003	}
5004	if (unlikely(!folio_test_uptodate(folio))) {
5005	ret = -EIO;
5006	goto out_unlock;
5007	}
5008	}
5009	folio_wait_writeback(folio);
5010
5011	/*
5012	* We do not need to lock extents nor wait for OE, as it's already
5013	* beyond EOF.
5014	*/
5015
5016	zero_start = max_t(u64, folio_pos(folio), start);
5017	zero_end = folio_next_pos(folio);
5018	folio_zero_range(folio, zero_start - folio_pos(folio),
5019	zero_end - zero_start);
5020
5021	out_unlock:
5022	folio_unlock(folio);
5023	folio_put(folio);
5024	return ret;
5025	}
5026
5027	/*
5028	* Handle the truncation of a fs block.
5029	*
5030	* @inode - inode that we're zeroing
5031	* @offset - the file offset of the block to truncate
5032	* The value must be inside [@start, @end], and the function will do
5033	* extra checks if the block that covers @offset needs to be zeroed.
5034	* @start - the start file offset of the range we want to zero
5035	* @end - the end (inclusive) file offset of the range we want to zero.
5036	*
5037	* If the range is not block aligned, read out the folio that covers @offset,
5038	* and if needed zero blocks that are inside the folio and covered by [@start, @end).
5039	* If @start or @end + 1 lands inside a block, that block will be marked dirty
5040	* for writeback.
5041	*
5042	* This is utilized by hole punch, zero range, file expansion.
5043	*/
5044	int btrfs_truncate_block(struct btrfs_inode *inode, u64 offset, u64 start, u64 end)
5045	{
5046	struct btrfs_fs_info *fs_info = inode->root->fs_info;
5047	struct address_space *mapping = inode->vfs_inode.i_mapping;
5048	struct extent_io_tree *io_tree = &inode->io_tree;
5049	struct btrfs_ordered_extent *ordered;
5050	struct extent_state *cached_state = NULL;
5051	struct extent_changeset *data_reserved = NULL;
5052	bool only_release_metadata = false;
5053	u32 blocksize = fs_info->sectorsize;
5054	pgoff_t index = (offset >> PAGE_SHIFT);
5055	struct folio *folio;
5056	gfp_t mask = btrfs_alloc_write_mask(mapping);
5057	int ret = 0;
5058	const bool in_head_block = is_inside_block(offset, round_down(start, blocksize),
5059	blocksize);
5060	const bool in_tail_block = is_inside_block(offset, round_down(end, blocksize),
5061	blocksize);
5062	bool need_truncate_head = false;
5063	bool need_truncate_tail = false;
5064	u64 zero_start;
5065	u64 zero_end;
5066	u64 block_start;
5067	u64 block_end;
5068
5069	/* @offset should be inside the range. */
5070	ASSERT(start <= offset && offset <= end, "offset=%llu start=%llu end=%llu",
5071	offset, start, end);
5072
5073	/* The range is aligned at both ends. */
5074	if (IS_ALIGNED(start, blocksize) && IS_ALIGNED(end + 1, blocksize)) {
5075	/*
5076	* For block size < page size case, we may have polluted blocks
5077	* beyond EOF. So we also need to zero them out.
5078	*/
5079	if (end == (u64)-1 && blocksize < PAGE_SIZE)
5080	ret = truncate_block_zero_beyond_eof(inode, start);
5081	goto out;
5082	}
5083
5084	/*
5085	* @offset may not be inside the head nor tail block. In that case we
5086	* don't need to do anything.
5087	*/
5088	if (!in_head_block && !in_tail_block)
5089	goto out;
5090
5091	/*
5092	* Skip the truncation if the range in the target block is already aligned.
5093	* The seemingly complex check will also handle the same block case.
5094	*/
5095	if (in_head_block && !IS_ALIGNED(start, blocksize))
5096	need_truncate_head = true;
5097	if (in_tail_block && !IS_ALIGNED(end + 1, blocksize))
5098	need_truncate_tail = true;
5099	if (!need_truncate_head && !need_truncate_tail)
5100	goto out;
5101
5102	block_start = round_down(offset, blocksize);
5103	block_end = block_start + blocksize - 1;
5104
5105	ret = btrfs_check_data_free_space(inode, &data_reserved, block_start,
5106	blocksize, false);
5107	if (ret < 0) {
5108	size_t write_bytes = blocksize;
5109
5110	if (btrfs_check_nocow_lock(inode, block_start, &write_bytes, false) > 0) {
5111	/* For nocow case, no need to reserve data space. */
5112	ASSERT(write_bytes == blocksize, "write_bytes=%zu blocksize=%u",
5113	write_bytes, blocksize);
5114	only_release_metadata = true;
5115	} else {
5116	goto out;
5117	}
5118	}
5119	ret = btrfs_delalloc_reserve_metadata(inode, blocksize, blocksize, false);
5120	if (ret < 0) {
5121	if (!only_release_metadata)
5122	btrfs_free_reserved_data_space(inode, data_reserved,
5123	block_start, blocksize);
5124	goto out;
5125	}
5126	again:
5127	folio = __filemap_get_folio(mapping, index,
5128	FGP_LOCK \| FGP_ACCESSED \| FGP_CREAT, mask);
5129	if (IS_ERR(folio)) {
5130	if (only_release_metadata)
5131	btrfs_delalloc_release_metadata(inode, blocksize, true);
5132	else
5133	btrfs_delalloc_release_space(inode, data_reserved,
5134	block_start, blocksize, true);
5135	btrfs_delalloc_release_extents(inode, blocksize);
5136	ret = PTR_ERR(folio);
5137	goto out;
5138	}
5139
5140	if (!folio_test_uptodate(folio)) {
5141	ret = btrfs_read_folio(NULL, folio);
5142	folio_lock(folio);
5143	if (folio->mapping != mapping) {
5144	folio_unlock(folio);
5145	folio_put(folio);
5146	goto again;
5147	}
5148	if (unlikely(!folio_test_uptodate(folio))) {
5149	ret = -EIO;
5150	goto out_unlock;
5151	}
5152	}
5153
5154	/*
5155	* We unlock the page after the io is completed and then re-lock it
5156	* above. release_folio() could have come in between that and cleared
5157	* folio private, but left the page in the mapping. Set the page mapped
5158	* here to make sure it's properly set for the subpage stuff.
5159	*/
5160	ret = set_folio_extent_mapped(folio);
5161	if (ret < 0)
5162	goto out_unlock;
5163
5164	folio_wait_writeback(folio);
5165
5166	btrfs_lock_extent(io_tree, block_start, block_end, &cached_state);
5167
5168	ordered = btrfs_lookup_ordered_extent(inode, block_start);
5169	if (ordered) {
5170	btrfs_unlock_extent(io_tree, block_start, block_end, &cached_state);
5171	folio_unlock(folio);
5172	folio_put(folio);
5173	btrfs_start_ordered_extent(ordered);
5174	btrfs_put_ordered_extent(ordered);
5175	goto again;
5176	}
5177
5178	ret = btrfs_reset_extent_delalloc(inode, block_start, block_end, 0, &cached_state);
5179	if (ret) {
5180	btrfs_unlock_extent(io_tree, block_start, block_end, &cached_state);
5181	goto out_unlock;
5182	}
5183
5184	if (end == (u64)-1) {
5185	/*
5186	* We're truncating beyond EOF, the remaining blocks normally are
5187	* already holes thus no need to zero again, but it's possible for
5188	* fs block size < page size cases to have memory mapped writes
5189	* to pollute ranges beyond EOF.
5190	*
5191	* In that case although such polluted blocks beyond EOF will
5192	* not reach disk, it still affects our page caches.
5193	*/
5194	zero_start = max_t(u64, folio_pos(folio), start);
5195	zero_end = min_t(u64, folio_next_pos(folio) - 1, end);
5196	} else {
5197	zero_start = max_t(u64, block_start, start);
5198	zero_end = min_t(u64, block_end, end);
5199	}
5200	folio_zero_range(folio, zero_start - folio_pos(folio),
5201	zero_end - zero_start + 1);
5202
5203	btrfs_folio_set_dirty(fs_info, folio, block_start,
5204	block_end + 1 - block_start);
5205
5206	if (only_release_metadata)
5207	btrfs_set_extent_bit(&inode->io_tree, block_start, block_end,
5208	EXTENT_NORESERVE, &cached_state);
5209
5210	btrfs_unlock_extent(io_tree, block_start, block_end, &cached_state);
5211
5212	out_unlock:
5213	if (ret) {
5214	if (only_release_metadata)
5215	btrfs_delalloc_release_metadata(inode, blocksize, true);
5216	else
5217	btrfs_delalloc_release_space(inode, data_reserved,
5218	block_start, blocksize, true);
5219	}
5220	btrfs_delalloc_release_extents(inode, blocksize);
5221	folio_unlock(folio);
5222	folio_put(folio);
5223	out:
5224	if (only_release_metadata)
5225	btrfs_check_nocow_unlock(inode);
5226	extent_changeset_free(data_reserved);
5227	return ret;
5228	}
5229
5230	static int maybe_insert_hole(struct btrfs_inode *inode, u64 offset, u64 len)
5231	{
5232	struct btrfs_root *root = inode->root;
5233	struct btrfs_fs_info *fs_info = root->fs_info;
5234	struct btrfs_trans_handle *trans;
5235	struct btrfs_drop_extents_args drop_args = { 0 };
5236	int ret;
5237
5238	/*
5239	* If NO_HOLES is enabled, we don't need to do anything.
5240	* Later, up in the call chain, either btrfs_set_inode_last_sub_trans()
5241	* or btrfs_update_inode() will be called, which guarantee that the next
5242	* fsync will know this inode was changed and needs to be logged.
5243	*/
5244	if (btrfs_fs_incompat(fs_info, NO_HOLES))
5245	return 0;
5246
5247	/*
5248	* 1 - for the one we're dropping
5249	* 1 - for the one we're adding
5250	* 1 - for updating the inode.
5251	*/
5252	trans = btrfs_start_transaction(root, 3);
5253	if (IS_ERR(trans))
5254	return PTR_ERR(trans);
5255
5256	drop_args.start = offset;
5257	drop_args.end = offset + len;
5258	drop_args.drop_cache = true;
5259
5260	ret = btrfs_drop_extents(trans, root, inode, &drop_args);
5261	if (unlikely(ret)) {
5262	btrfs_abort_transaction(trans, ret);
5263	btrfs_end_transaction(trans);
5264	return ret;
5265	}
5266
5267	ret = btrfs_insert_hole_extent(trans, root, btrfs_ino(inode), offset, len);
5268	if (ret) {
5269	btrfs_abort_transaction(trans, ret);
5270	} else {
5271	btrfs_update_inode_bytes(inode, 0, drop_args.bytes_found);
5272	btrfs_update_inode(trans, inode);
5273	}
5274	btrfs_end_transaction(trans);
5275	return ret;
5276	}
5277
5278	/*
5279	* This function puts in dummy file extents for the area we're creating a hole
5280	* for. So if we are truncating this file to a larger size we need to insert
5281	* these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for
5282	* the range between oldsize and size
5283	*/
5284	int btrfs_cont_expand(struct btrfs_inode *inode, loff_t oldsize, loff_t size)
5285	{
5286	struct btrfs_root *root = inode->root;
5287	struct btrfs_fs_info *fs_info = root->fs_info;
5288	struct extent_io_tree *io_tree = &inode->io_tree;
5289	struct extent_map *em = NULL;
5290	struct extent_state *cached_state = NULL;
5291	u64 hole_start = ALIGN(oldsize, fs_info->sectorsize);
5292	u64 block_end = ALIGN(size, fs_info->sectorsize);
5293	u64 last_byte;
5294	u64 cur_offset;
5295	u64 hole_size;
5296	int ret = 0;
5297
5298	/*
5299	* If our size started in the middle of a block we need to zero out the
5300	* rest of the block before we expand the i_size, otherwise we could
5301	* expose stale data.
5302	*/
5303	ret = btrfs_truncate_block(inode, oldsize, oldsize, -1);
5304	if (ret)
5305	return ret;
5306
5307	if (size <= hole_start)
5308	return 0;
5309
5310	btrfs_lock_and_flush_ordered_range(inode, hole_start, block_end - 1,
5311	&cached_state);
5312	cur_offset = hole_start;
5313	while (1) {
5314	em = btrfs_get_extent(inode, NULL, cur_offset, block_end - cur_offset);
5315	if (IS_ERR(em)) {
5316	ret = PTR_ERR(em);
5317	em = NULL;
5318	break;
5319	}
5320	last_byte = min(btrfs_extent_map_end(em), block_end);
5321	last_byte = ALIGN(last_byte, fs_info->sectorsize);
5322	hole_size = last_byte - cur_offset;
5323
5324	if (!(em->flags & EXTENT_FLAG_PREALLOC)) {
5325	struct extent_map *hole_em;
5326
5327	ret = maybe_insert_hole(inode, cur_offset, hole_size);
5328	if (ret)
5329	break;
5330
5331	ret = btrfs_inode_set_file_extent_range(inode,
5332	cur_offset, hole_size);
5333	if (ret)
5334	break;
5335
5336	hole_em = btrfs_alloc_extent_map();
5337	if (!hole_em) {
5338	btrfs_drop_extent_map_range(inode, cur_offset,
5339	cur_offset + hole_size - 1,
5340	false);
5341	btrfs_set_inode_full_sync(inode);
5342	goto next;
5343	}
5344	hole_em->start = cur_offset;
5345	hole_em->len = hole_size;
5346
5347	hole_em->disk_bytenr = EXTENT_MAP_HOLE;
5348	hole_em->disk_num_bytes = 0;
5349	hole_em->ram_bytes = hole_size;
5350	hole_em->generation = btrfs_get_fs_generation(fs_info);
5351
5352	ret = btrfs_replace_extent_map_range(inode, hole_em, true);
5353	btrfs_free_extent_map(hole_em);
5354	} else {
5355	ret = btrfs_inode_set_file_extent_range(inode,
5356	cur_offset, hole_size);
5357	if (ret)
5358	break;
5359	}
5360	next:
5361	btrfs_free_extent_map(em);
5362	em = NULL;
5363	cur_offset = last_byte;
5364	if (cur_offset >= block_end)
5365	break;
5366	}
5367	btrfs_free_extent_map(em);
5368	btrfs_unlock_extent(io_tree, hole_start, block_end - 1, &cached_state);
5369	return ret;
5370	}
5371
5372	static int btrfs_setsize(struct inode inode, struct iattr attr)
5373	{
5374	struct btrfs_root *root = BTRFS_I(inode)->root;
5375	struct btrfs_trans_handle *trans;
5376	loff_t oldsize = i_size_read(inode);
5377	loff_t newsize = attr->ia_size;
5378	int mask = attr->ia_valid;
5379	int ret;
5380
5381	/*
5382	* The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a
5383	* special case where we need to update the times despite not having
5384	* these flags set. For all other operations the VFS set these flags
5385	* explicitly if it wants a timestamp update.
5386	*/
5387	if (newsize != oldsize) {
5388	inode_inc_iversion(inode);
5389	if (!(mask & (ATTR_CTIME \| ATTR_MTIME))) {
5390	inode_set_mtime_to_ts(inode,
5391	inode_set_ctime_current(inode));
5392	}
5393	}
5394
5395	if (newsize > oldsize) {
5396	/*
5397	* Don't do an expanding truncate while snapshotting is ongoing.
5398	* This is to ensure the snapshot captures a fully consistent
5399	* state of this file - if the snapshot captures this expanding
5400	* truncation, it must capture all writes that happened before
5401	* this truncation.
5402	*/
5403	btrfs_drew_write_lock(&root->snapshot_lock);
5404	ret = btrfs_cont_expand(BTRFS_I(inode), oldsize, newsize);
5405	if (ret) {
5406	btrfs_drew_write_unlock(&root->snapshot_lock);
5407	return ret;
5408	}
5409
5410	trans = btrfs_start_transaction(root, 1);
5411	if (IS_ERR(trans)) {
5412	btrfs_drew_write_unlock(&root->snapshot_lock);
5413	return PTR_ERR(trans);
5414	}
5415
5416	i_size_write(inode, newsize);
5417	btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0);
5418	pagecache_isize_extended(inode, oldsize, newsize);
5419	ret = btrfs_update_inode(trans, BTRFS_I(inode));
5420	btrfs_drew_write_unlock(&root->snapshot_lock);
5421	btrfs_end_transaction(trans);
5422	} else {
5423	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
5424
5425	if (btrfs_is_zoned(fs_info)) {
5426	ret = btrfs_wait_ordered_range(BTRFS_I(inode),
5427	ALIGN(newsize, fs_info->sectorsize),
5428	(u64)-1);
5429	if (ret)
5430	return ret;
5431	}
5432
5433	/*
5434	* We're truncating a file that used to have good data down to
5435	* zero. Make sure any new writes to the file get on disk
5436	* on close.
5437	*/
5438	if (newsize == 0 && oldsize != 0)
5439	set_bit(BTRFS_INODE_FLUSH_ON_CLOSE,
5440	&BTRFS_I(inode)->runtime_flags);
5441
5442	truncate_setsize(inode, newsize);
5443
5444	inode_dio_wait(inode);
5445
5446	ret = btrfs_truncate(BTRFS_I(inode), newsize == oldsize);
5447	if (ret && inode->i_nlink) {
5448	int ret2;
5449
5450	/*
5451	* Truncate failed, so fix up the in-memory size. We
5452	* adjusted disk_i_size down as we removed extents, so
5453	* wait for disk_i_size to be stable and then update the
5454	* in-memory size to match.
5455	*/
5456	ret2 = btrfs_wait_ordered_range(BTRFS_I(inode), 0, (u64)-1);
5457	if (ret2)
5458	return ret2;
5459	i_size_write(inode, BTRFS_I(inode)->disk_i_size);
5460	}
5461	}
5462
5463	return ret;
5464	}
5465
5466	static int btrfs_setattr(const struct mnt_idmap idmap, struct dentry dentry,
5467	struct iattr *attr)
5468	{
5469	struct inode *inode = d_inode(dentry);
5470	struct btrfs_root *root = BTRFS_I(inode)->root;
5471	int ret;
5472
5473	if (btrfs_root_readonly(root))
5474	return -EROFS;
5475
5476	ret = setattr_prepare(idmap, dentry, attr);
5477	if (ret)
5478	return ret;
5479
5480	if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
5481	ret = btrfs_setsize(inode, attr);
5482	if (ret)
5483	return ret;
5484	}
5485
5486	if (attr->ia_valid) {
5487	setattr_copy(idmap, inode, attr);
5488	inode_inc_iversion(inode);
5489	ret = btrfs_dirty_inode(BTRFS_I(inode));
5490
5491	if (!ret && attr->ia_valid & ATTR_MODE)
5492	ret = posix_acl_chmod(idmap, dentry, inode->i_mode);
5493	}
5494
5495	return ret;
5496	}
5497
5498	/*
5499	* While truncating the inode pages during eviction, we get the VFS
5500	* calling btrfs_invalidate_folio() against each folio of the inode. This
5501	* is slow because the calls to btrfs_invalidate_folio() result in a
5502	* huge amount of calls to lock_extent() and clear_extent_bit(),
5503	* which keep merging and splitting extent_state structures over and over,
5504	* wasting lots of time.
5505	*
5506	* Therefore if the inode is being evicted, let btrfs_invalidate_folio()
5507	* skip all those expensive operations on a per folio basis and do only
5508	* the ordered io finishing, while we release here the extent_map and
5509	* extent_state structures, without the excessive merging and splitting.
5510	*/
5511	static void evict_inode_truncate_pages(struct inode *inode)
5512	{
5513	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
5514	struct rb_node *node;
5515
5516	ASSERT(inode_state_read_once(inode) & I_FREEING);
5517	truncate_inode_pages_final(&inode->i_data);
5518
5519	btrfs_drop_extent_map_range(BTRFS_I(inode), 0, (u64)-1, false);
5520
5521	/*
5522	* Keep looping until we have no more ranges in the io tree.
5523	* We can have ongoing bios started by readahead that have
5524	* their endio callback (extent_io.c:end_bio_extent_readpage)
5525	* still in progress (unlocked the pages in the bio but did not yet
5526	* unlocked the ranges in the io tree). Therefore this means some
5527	* ranges can still be locked and eviction started because before
5528	* submitting those bios, which are executed by a separate task (work
5529	* queue kthread), inode references (inode->i_count) were not taken
5530	* (which would be dropped in the end io callback of each bio).
5531	* Therefore here we effectively end up waiting for those bios and
5532	* anyone else holding locked ranges without having bumped the inode's
5533	* reference count - if we don't do it, when they access the inode's
5534	* io_tree to unlock a range it may be too late, leading to an
5535	* use-after-free issue.
5536	*/
5537	spin_lock(&io_tree->lock);
5538	while (!RB_EMPTY_ROOT(&io_tree->state)) {
5539	struct extent_state *state;
5540	struct extent_state *cached_state = NULL;
5541	u64 start;
5542	u64 end;
5543	unsigned state_flags;
5544
5545	node = rb_first(&io_tree->state);
5546	state = rb_entry(node, struct extent_state, rb_node);
5547	start = state->start;
5548	end = state->end;
5549	state_flags = state->state;
5550	spin_unlock(&io_tree->lock);
5551
5552	btrfs_lock_extent(io_tree, start, end, &cached_state);
5553
5554	/*
5555	* If still has DELALLOC flag, the extent didn't reach disk,
5556	* and its reserved space won't be freed by delayed_ref.
5557	* So we need to free its reserved space here.
5558	* (Refer to comment in btrfs_invalidate_folio, case 2)
5559	*
5560	* Note, end is the bytenr of last byte, so we need + 1 here.
5561	*/
5562	if (state_flags & EXTENT_DELALLOC)
5563	btrfs_qgroup_free_data(BTRFS_I(inode), NULL, start,
5564	end - start + 1, NULL);
5565
5566	btrfs_clear_extent_bit(io_tree, start, end,
5567	EXTENT_CLEAR_ALL_BITS \| EXTENT_DO_ACCOUNTING,
5568	&cached_state);
5569
5570	cond_resched();
5571	spin_lock(&io_tree->lock);
5572	}
5573	spin_unlock(&io_tree->lock);
5574	}
5575
5576	static struct btrfs_trans_handle evict_refill_and_join(struct btrfs_root root,
5577	struct btrfs_block_rsv *rsv)
5578	{
5579	struct btrfs_fs_info *fs_info = root->fs_info;
5580	struct btrfs_trans_handle *trans;
5581	u64 delayed_refs_extra = btrfs_calc_delayed_ref_bytes(fs_info, 1);
5582	int ret;
5583
5584	/*
5585	* Eviction should be taking place at some place safe because of our
5586	* delayed iputs. However the normal flushing code will run delayed
5587	* iputs, so we cannot use FLUSH_ALL otherwise we'll deadlock.
5588	*
5589	* We reserve the delayed_refs_extra here again because we can't use
5590	* btrfs_start_transaction(root, 0) for the same deadlocky reason as
5591	* above. We reserve our extra bit here because we generate a ton of
5592	* delayed refs activity by truncating.
5593	*
5594	* BTRFS_RESERVE_FLUSH_EVICT will steal from the global_rsv if it can,
5595	* if we fail to make this reservation we can re-try without the
5596	* delayed_refs_extra so we can make some forward progress.
5597	*/
5598	ret = btrfs_block_rsv_refill(rsv, rsv->size + delayed_refs_extra,
5599	BTRFS_RESERVE_FLUSH_EVICT);
5600	if (ret) {
5601	ret = btrfs_block_rsv_refill(rsv, rsv->size,
5602	BTRFS_RESERVE_FLUSH_EVICT);
5603	if (ret) {
5604	btrfs_warn(fs_info,
5605	"could not allocate space for delete; will truncate on mount");
5606	return ERR_PTR(-ENOSPC);
5607	}
5608	delayed_refs_extra = 0;
5609	}
5610
5611	trans = btrfs_join_transaction(root);
5612	if (IS_ERR(trans))
5613	return trans;
5614
5615	if (delayed_refs_extra) {
5616	trans->block_rsv = &fs_info->trans_block_rsv;
5617	trans->bytes_reserved = delayed_refs_extra;
5618	btrfs_block_rsv_migrate(rsv, trans->block_rsv,
5619	delayed_refs_extra, true);
5620	}
5621	return trans;
5622	}
5623
5624	void btrfs_evict_inode(struct inode *inode)
5625	{
5626	struct btrfs_fs_info *fs_info;
5627	struct btrfs_trans_handle *trans;
5628	struct btrfs_root *root = BTRFS_I(inode)->root;
5629	struct btrfs_block_rsv rsv;
5630	int ret;
5631
5632	trace_btrfs_inode_evict(inode);
5633
5634	if (!root)
5635	goto clear_inode;
5636
5637	fs_info = inode_to_fs_info(inode);
5638	evict_inode_truncate_pages(inode);
5639
5640	if (inode->i_nlink &&
5641	((btrfs_root_refs(&root->root_item) != 0 &&
5642	btrfs_root_id(root) != BTRFS_ROOT_TREE_OBJECTID) \|\|
5643	btrfs_is_free_space_inode(BTRFS_I(inode))))
5644	goto out;
5645
5646	if (is_bad_inode(inode))
5647	goto out;
5648
5649	if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags))
5650	goto out;
5651
5652	if (inode->i_nlink > 0) {
5653	BUG_ON(btrfs_root_refs(&root->root_item) != 0 &&
5654	btrfs_root_id(root) != BTRFS_ROOT_TREE_OBJECTID);
5655	goto out;
5656	}
5657
5658	/*
5659	* This makes sure the inode item in tree is uptodate and the space for
5660	* the inode update is released.
5661	*/
5662	ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode));
5663	if (ret)
5664	goto out;
5665
5666	/*
5667	* This drops any pending insert or delete operations we have for this
5668	* inode. We could have a delayed dir index deletion queued up, but
5669	* we're removing the inode completely so that'll be taken care of in
5670	* the truncate.
5671	*/
5672	btrfs_kill_delayed_inode_items(BTRFS_I(inode));
5673
5674	btrfs_init_metadata_block_rsv(fs_info, &rsv, BTRFS_BLOCK_RSV_TEMP);
5675	rsv.size = btrfs_calc_metadata_size(fs_info, 1);
5676	rsv.failfast = true;
5677
5678	btrfs_i_size_write(BTRFS_I(inode), 0);
5679
5680	while (1) {
5681	struct btrfs_truncate_control control = {
5682	.inode = BTRFS_I(inode),
5683	.ino = btrfs_ino(BTRFS_I(inode)),
5684	.new_size = 0,
5685	.min_type = 0,
5686	};
5687
5688	trans = evict_refill_and_join(root, &rsv);
5689	if (IS_ERR(trans))
5690	goto out_release;
5691
5692	trans->block_rsv = &rsv;
5693
5694	ret = btrfs_truncate_inode_items(trans, root, &control);
5695	trans->block_rsv = &fs_info->trans_block_rsv;
5696	btrfs_end_transaction(trans);
5697	/*
5698	* We have not added new delayed items for our inode after we
5699	* have flushed its delayed items, so no need to throttle on
5700	* delayed items. However we have modified extent buffers.
5701	*/
5702	btrfs_btree_balance_dirty_nodelay(fs_info);
5703	if (ret && ret != -ENOSPC && ret != -EAGAIN)
5704	goto out_release;
5705	else if (!ret)
5706	break;
5707	}
5708
5709	/*
5710	* Errors here aren't a big deal, it just means we leave orphan items in
5711	* the tree. They will be cleaned up on the next mount. If the inode
5712	* number gets reused, cleanup deletes the orphan item without doing
5713	* anything, and unlink reuses the existing orphan item.
5714	*
5715	* If it turns out that we are dropping too many of these, we might want
5716	* to add a mechanism for retrying these after a commit.
5717	*/
5718	trans = evict_refill_and_join(root, &rsv);
5719	if (!IS_ERR(trans)) {
5720	trans->block_rsv = &rsv;
5721	btrfs_orphan_del(trans, BTRFS_I(inode));
5722	trans->block_rsv = &fs_info->trans_block_rsv;
5723	btrfs_end_transaction(trans);
5724	}
5725
5726	out_release:
5727	btrfs_block_rsv_release(fs_info, &rsv, (u64)-1, NULL);
5728	out:
5729	/*
5730	* If we didn't successfully delete, the orphan item will still be in
5731	* the tree and we'll retry on the next mount. Again, we might also want
5732	* to retry these periodically in the future.
5733	*/
5734	btrfs_remove_delayed_node(BTRFS_I(inode));
5735	clear_inode:
5736	clear_inode(inode);
5737	}
5738
5739	/*
5740	* Return the key found in the dir entry in the location pointer, fill @type
5741	* with BTRFS_FT_*, and return 0.
5742	*
5743	* If no dir entries were found, returns -ENOENT.
5744	* If found a corrupted location in dir entry, returns -EUCLEAN.
5745	*/
5746	static int btrfs_inode_by_name(struct btrfs_inode dir, struct dentry dentry,
5747	struct btrfs_key location, u8 type)
5748	{
5749	struct btrfs_dir_item *di;
5750	BTRFS_PATH_AUTO_FREE(path);
5751	struct btrfs_root *root = dir->root;
5752	int ret = 0;
5753	struct fscrypt_name fname;
5754
5755	path = btrfs_alloc_path();
5756	if (!path)
5757	return -ENOMEM;
5758
5759	ret = fscrypt_setup_filename(&dir->vfs_inode, &dentry->d_name, 1, &fname);
5760	if (ret < 0)
5761	return ret;
5762	/*
5763	* fscrypt_setup_filename() should never return a positive value, but
5764	* gcc on sparc/parisc thinks it can, so assert that doesn't happen.
5765	*/
5766	ASSERT(ret == 0);
5767
5768	/* This needs to handle no-key deletions later on */
5769
5770	di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(dir),
5771	&fname.disk_name, 0);
5772	if (IS_ERR_OR_NULL(di)) {
5773	ret = di ? PTR_ERR(di) : -ENOENT;
5774	goto out;
5775	}
5776
5777	btrfs_dir_item_key_to_cpu(path->nodes[0], di, location);
5778	if (unlikely(location->type != BTRFS_INODE_ITEM_KEY &&
5779	location->type != BTRFS_ROOT_ITEM_KEY)) {
5780	ret = -EUCLEAN;
5781	btrfs_warn(root->fs_info,
5782	"%s gets something invalid in DIR_ITEM (name %s, directory ino %llu, location " BTRFS_KEY_FMT ")",
5783	__func__, fname.disk_name.name, btrfs_ino(dir),
5784	BTRFS_KEY_FMT_VALUE(location));
5785	}
5786	if (!ret)
5787	*type = btrfs_dir_ftype(path->nodes[0], di);
5788	out:
5789	fscrypt_free_filename(&fname);
5790	return ret;
5791	}
5792
5793	/*
5794	* when we hit a tree root in a directory, the btrfs part of the inode
5795	* needs to be changed to reflect the root directory of the tree root. This
5796	* is kind of like crossing a mount point.
5797	*/
5798	static int fixup_tree_root_location(struct btrfs_fs_info *fs_info,
5799	struct btrfs_inode *dir,
5800	struct dentry *dentry,
5801	struct btrfs_key *location,
5802	struct btrfs_root **sub_root)
5803	{
5804	BTRFS_PATH_AUTO_FREE(path);
5805	struct btrfs_root *new_root;
5806	struct btrfs_root_ref *ref;
5807	struct extent_buffer *leaf;
5808	struct btrfs_key key;
5809	int ret;
5810	int err = 0;
5811	struct fscrypt_name fname;
5812
5813	ret = fscrypt_setup_filename(&dir->vfs_inode, &dentry->d_name, 0, &fname);
5814	if (ret)
5815	return ret;
5816
5817	path = btrfs_alloc_path();
5818	if (!path) {
5819	err = -ENOMEM;
5820	goto out;
5821	}
5822
5823	err = -ENOENT;
5824	key.objectid = btrfs_root_id(dir->root);
5825	key.type = BTRFS_ROOT_REF_KEY;
5826	key.offset = location->objectid;
5827
5828	ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
5829	if (ret) {
5830	if (ret < 0)
5831	err = ret;
5832	goto out;
5833	}
5834
5835	leaf = path->nodes[0];
5836	ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref);
5837	if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(dir) \|\|
5838	btrfs_root_ref_name_len(leaf, ref) != fname.disk_name.len)
5839	goto out;
5840
5841	ret = memcmp_extent_buffer(leaf, fname.disk_name.name,
5842	(unsigned long)(ref + 1), fname.disk_name.len);
5843	if (ret)
5844	goto out;
5845
5846	btrfs_release_path(path);
5847
5848	new_root = btrfs_get_fs_root(fs_info, location->objectid, true);
5849	if (IS_ERR(new_root)) {
5850	err = PTR_ERR(new_root);
5851	goto out;
5852	}
5853
5854	*sub_root = new_root;
5855	location->objectid = btrfs_root_dirid(&new_root->root_item);
5856	location->type = BTRFS_INODE_ITEM_KEY;
5857	location->offset = 0;
5858	err = 0;
5859	out:
5860	fscrypt_free_filename(&fname);
5861	return err;
5862	}
5863
5864
5865
5866	static void btrfs_del_inode_from_root(struct btrfs_inode *inode)
5867	{
5868	struct btrfs_root *root = inode->root;
5869	struct btrfs_inode *entry;
5870	bool empty = false;
5871
5872	xa_lock(&root->inodes);
5873	/*
5874	* This btrfs_inode is being freed and has already been unhashed at this
5875	* point. It's possible that another btrfs_inode has already been
5876	* allocated for the same inode and inserted itself into the root, so
5877	* don't delete it in that case.
5878	*
5879	* Note that this shouldn't need to allocate memory, so the gfp flags
5880	* don't really matter.
5881	*/
5882	entry = __xa_cmpxchg(&root->inodes, btrfs_ino(inode), inode, NULL,
5883	GFP_ATOMIC);
5884	if (entry == inode)
5885	empty = xa_empty(&root->inodes);
5886	xa_unlock(&root->inodes);
5887
5888	if (empty && btrfs_root_refs(&root->root_item) == 0) {
5889	xa_lock(&root->inodes);
5890	empty = xa_empty(&root->inodes);
5891	xa_unlock(&root->inodes);
5892	if (empty)
5893	btrfs_add_dead_root(root);
5894	}
5895	}
5896
5897
5898	static int btrfs_init_locked_inode(struct inode inode, void p)
5899	{
5900	struct btrfs_iget_args *args = p;
5901
5902	btrfs_set_inode_number(BTRFS_I(inode), args->ino);
5903	BTRFS_I(inode)->root = btrfs_grab_root(args->root);
5904
5905	if (args->root && args->root == args->root->fs_info->tree_root &&
5906	args->ino != BTRFS_BTREE_INODE_OBJECTID)
5907	set_bit(BTRFS_INODE_FREE_SPACE_INODE,
5908	&BTRFS_I(inode)->runtime_flags);
5909	return 0;
5910	}
5911
5912	static int btrfs_find_actor(struct inode inode, void opaque)
5913	{
5914	struct btrfs_iget_args *args = opaque;
5915
5916	return args->ino == btrfs_ino(BTRFS_I(inode)) &&
5917	args->root == BTRFS_I(inode)->root;
5918	}
5919
5920	static struct btrfs_inode btrfs_iget_locked(u64 ino, struct btrfs_root root)
5921	{
5922	struct inode *inode;
5923	struct btrfs_iget_args args;
5924	unsigned long hashval = btrfs_inode_hash(ino, root);
5925
5926	args.ino = ino;
5927	args.root = root;
5928
5929	inode = iget5_locked_rcu(root->fs_info->sb, hashval, btrfs_find_actor,
5930	btrfs_init_locked_inode,
5931	(void *)&args);
5932	if (!inode)
5933	return NULL;
5934	return BTRFS_I(inode);
5935	}
5936
5937	/*
5938	* Get an inode object given its inode number and corresponding root. Path is
5939	* preallocated to prevent recursing back to iget through allocator.
5940	*/
5941	struct btrfs_inode btrfs_iget_path(u64 ino, struct btrfs_root root,
5942	struct btrfs_path *path)
5943	{
5944	struct btrfs_inode *inode;
5945	int ret;
5946
5947	inode = btrfs_iget_locked(ino, root);
5948	if (!inode)
5949	return ERR_PTR(-ENOMEM);
5950
5951	if (!(inode_state_read_once(&inode->vfs_inode) & I_NEW))
5952	return inode;
5953
5954	ret = btrfs_read_locked_inode(inode, path);
5955	if (ret)
5956	return ERR_PTR(ret);
5957
5958	unlock_new_inode(&inode->vfs_inode);
5959	return inode;
5960	}
5961
5962	/*
5963	* Get an inode object given its inode number and corresponding root.
5964	*/
5965	struct btrfs_inode btrfs_iget(u64 ino, struct btrfs_root root)
5966	{
5967	struct btrfs_inode *inode;
5968	struct btrfs_path *path;
5969	int ret;
5970
5971	inode = btrfs_iget_locked(ino, root);
5972	if (!inode)
5973	return ERR_PTR(-ENOMEM);
5974
5975	if (!(inode_state_read_once(&inode->vfs_inode) & I_NEW))
5976	return inode;
5977
5978	path = btrfs_alloc_path();
5979	if (!path) {
5980	iget_failed(&inode->vfs_inode);
5981	return ERR_PTR(-ENOMEM);
5982	}
5983
5984	ret = btrfs_read_locked_inode(inode, path);
5985	btrfs_free_path(path);
5986	if (ret)
5987	return ERR_PTR(ret);
5988
5989	if (S_ISDIR(inode->vfs_inode.i_mode))
5990	inode->vfs_inode.i_opflags \|= IOP_FASTPERM_MAY_EXEC;
5991	unlock_new_inode(&inode->vfs_inode);
5992	return inode;
5993	}
5994
5995	static struct btrfs_inode new_simple_dir(struct inode dir,
5996	struct btrfs_key *key,
5997	struct btrfs_root *root)
5998	{
5999	struct timespec64 ts;
6000	struct inode *vfs_inode;
6001	struct btrfs_inode *inode;
6002
6003	vfs_inode = new_inode(dir->i_sb);
6004	if (!vfs_inode)
6005	return ERR_PTR(-ENOMEM);
6006
6007	inode = BTRFS_I(vfs_inode);
6008	inode->root = btrfs_grab_root(root);
6009	inode->ref_root_id = key->objectid;
6010	set_bit(BTRFS_INODE_ROOT_STUB, &inode->runtime_flags);
6011	set_bit(BTRFS_INODE_DUMMY, &inode->runtime_flags);
6012
6013	btrfs_set_inode_number(inode, BTRFS_EMPTY_SUBVOL_DIR_OBJECTID);
6014	/*
6015	* We only need lookup, the rest is read-only and there's no inode
6016	* associated with the dentry
6017	*/
6018	vfs_inode->i_op = &simple_dir_inode_operations;
6019	vfs_inode->i_opflags &= ~IOP_XATTR;
6020	vfs_inode->i_fop = &simple_dir_operations;
6021	vfs_inode->i_mode = S_IFDIR \| S_IRUGO \| S_IWUSR \| S_IXUGO;
6022
6023	ts = inode_set_ctime_current(vfs_inode);
6024	inode_set_mtime_to_ts(vfs_inode, ts);
6025	inode_set_atime_to_ts(vfs_inode, inode_get_atime(dir));
6026	inode->i_otime_sec = ts.tv_sec;
6027	inode->i_otime_nsec = ts.tv_nsec;
6028
6029	vfs_inode->i_uid = dir->i_uid;
6030	vfs_inode->i_gid = dir->i_gid;
6031
6032	return inode;
6033	}
6034
6035	static_assert(BTRFS_FT_UNKNOWN == FT_UNKNOWN);
6036	static_assert(BTRFS_FT_REG_FILE == FT_REG_FILE);
6037	static_assert(BTRFS_FT_DIR == FT_DIR);
6038	static_assert(BTRFS_FT_CHRDEV == FT_CHRDEV);
6039	static_assert(BTRFS_FT_BLKDEV == FT_BLKDEV);
6040	static_assert(BTRFS_FT_FIFO == FT_FIFO);
6041	static_assert(BTRFS_FT_SOCK == FT_SOCK);
6042	static_assert(BTRFS_FT_SYMLINK == FT_SYMLINK);
6043
6044	static inline u8 btrfs_inode_type(const struct btrfs_inode *inode)
6045	{
6046	return fs_umode_to_ftype(inode->vfs_inode.i_mode);
6047	}
6048
6049	struct inode btrfs_lookup_dentry(struct inode dir, struct dentry *dentry)
6050	{
6051	struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);
6052	struct btrfs_inode *inode;
6053	struct btrfs_root *root = BTRFS_I(dir)->root;
6054	struct btrfs_root *sub_root = root;
6055	struct btrfs_key location = { 0 };
6056	u8 di_type = 0;
6057	int ret = 0;
6058
6059	if (dentry->d_name.len > BTRFS_NAME_LEN)
6060	return ERR_PTR(-ENAMETOOLONG);
6061
6062	ret = btrfs_inode_by_name(BTRFS_I(dir), dentry, &location, &di_type);
6063	if (ret < 0)
6064	return ERR_PTR(ret);
6065
6066	if (location.type == BTRFS_INODE_ITEM_KEY) {
6067	inode = btrfs_iget(location.objectid, root);
6068	if (IS_ERR(inode))
6069	return ERR_CAST(inode);
6070
6071	/* Do extra check against inode mode with di_type */
6072	if (unlikely(btrfs_inode_type(inode) != di_type)) {
6073	btrfs_crit(fs_info,
6074	"inode mode mismatch with dir: inode mode=0%o btrfs type=%u dir type=%u",
6075	inode->vfs_inode.i_mode, btrfs_inode_type(inode),
6076	di_type);
6077	iput(&inode->vfs_inode);
6078	return ERR_PTR(-EUCLEAN);
6079	}
6080	return &inode->vfs_inode;
6081	}
6082
6083	ret = fixup_tree_root_location(fs_info, BTRFS_I(dir), dentry,
6084	&location, &sub_root);
6085	if (ret < 0) {
6086	if (ret != -ENOENT)
6087	inode = ERR_PTR(ret);
6088	else
6089	inode = new_simple_dir(dir, &location, root);
6090	} else {
6091	inode = btrfs_iget(location.objectid, sub_root);
6092	btrfs_put_root(sub_root);
6093
6094	if (IS_ERR(inode))
6095	return ERR_CAST(inode);
6096
6097	down_read(&fs_info->cleanup_work_sem);
6098	if (!sb_rdonly(inode->vfs_inode.i_sb))
6099	ret = btrfs_orphan_cleanup(sub_root);
6100	up_read(&fs_info->cleanup_work_sem);
6101	if (ret) {
6102	iput(&inode->vfs_inode);
6103	inode = ERR_PTR(ret);
6104	}
6105	}
6106
6107	if (IS_ERR(inode))
6108	return ERR_CAST(inode);
6109
6110	return &inode->vfs_inode;
6111	}
6112
6113	static int btrfs_dentry_delete(const struct dentry *dentry)
6114	{
6115	struct btrfs_root *root;
6116	struct inode *inode = d_inode(dentry);
6117
6118	if (!inode && !IS_ROOT(dentry))
6119	inode = d_inode(dentry->d_parent);
6120
6121	if (inode) {
6122	root = BTRFS_I(inode)->root;
6123	if (btrfs_root_refs(&root->root_item) == 0)
6124	return 1;
6125
6126	if (btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
6127	return 1;
6128	}
6129	return 0;
6130	}
6131
6132	static struct dentry btrfs_lookup(struct inode dir, struct dentry *dentry,
6133	unsigned int flags)
6134	{
6135	struct inode *inode = btrfs_lookup_dentry(dir, dentry);
6136
6137	if (inode == ERR_PTR(-ENOENT))
6138	inode = NULL;
6139	return d_splice_alias(inode, dentry);
6140	}
6141
6142	/*
6143	* Find the highest existing sequence number in a directory and then set the
6144	* in-memory index_cnt variable to the first free sequence number.
6145	*/
6146	static int btrfs_set_inode_index_count(struct btrfs_inode *inode)
6147	{
6148	struct btrfs_root *root = inode->root;
6149	struct btrfs_key key, found_key;
6150	BTRFS_PATH_AUTO_FREE(path);
6151	struct extent_buffer *leaf;
6152	int ret;
6153
6154	key.objectid = btrfs_ino(inode);
6155	key.type = BTRFS_DIR_INDEX_KEY;
6156	key.offset = (u64)-1;
6157
6158	path = btrfs_alloc_path();
6159	if (!path)
6160	return -ENOMEM;
6161
6162	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
6163	if (ret < 0)
6164	return ret;
6165
6166	if (unlikely(ret == 0)) {
6167	/*
6168	* Key with offset -1 found, there would have to exist a dir
6169	* index item with such offset, but this is out of the valid
6170	* range.
6171	*/
6172	btrfs_err(root->fs_info,
6173	"unexpected exact match for DIR_INDEX key, inode %llu",
6174	btrfs_ino(inode));
6175	return -EUCLEAN;
6176	}
6177
6178	if (path->slots[0] == 0) {
6179	inode->index_cnt = BTRFS_DIR_START_INDEX;
6180	return 0;
6181	}
6182
6183	path->slots[0]--;
6184
6185	leaf = path->nodes[0];
6186	btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
6187
6188	if (found_key.objectid != btrfs_ino(inode) \|\|
6189	found_key.type != BTRFS_DIR_INDEX_KEY) {
6190	inode->index_cnt = BTRFS_DIR_START_INDEX;
6191	return 0;
6192	}
6193
6194	inode->index_cnt = found_key.offset + 1;
6195
6196	return 0;
6197	}
6198
6199	static int btrfs_get_dir_last_index(struct btrfs_inode dir, u64 index)
6200	{
6201	int ret = 0;
6202
6203	btrfs_inode_lock(dir, 0);
6204	if (dir->index_cnt == (u64)-1) {
6205	ret = btrfs_inode_delayed_dir_index_count(dir);
6206	if (ret) {
6207	ret = btrfs_set_inode_index_count(dir);
6208	if (ret)
6209	goto out;
6210	}
6211	}
6212
6213	/* index_cnt is the index number of next new entry, so decrement it. */
6214	*index = dir->index_cnt - 1;
6215	out:
6216	btrfs_inode_unlock(dir, 0);
6217
6218	return ret;
6219	}
6220
6221	/*
6222	* All this infrastructure exists because dir_emit can fault, and we are holding
6223	* the tree lock when doing readdir. For now just allocate a buffer and copy
6224	* our information into that, and then dir_emit from the buffer. This is
6225	* similar to what NFS does, only we don't keep the buffer around in pagecache
6226	* because I'm afraid I'll mess that up. Long term we need to make filldir do
6227	* copy_to_user_inatomic so we don't have to worry about page faulting under the
6228	* tree lock.
6229	*/
6230	static int btrfs_opendir(struct inode inode, struct file file)
6231	{
6232	struct btrfs_file_private *private;
6233	u64 last_index;
6234	int ret;
6235
6236	ret = btrfs_get_dir_last_index(BTRFS_I(inode), &last_index);
6237	if (ret)
6238	return ret;
6239
6240	private = kzalloc_obj(struct btrfs_file_private);
6241	if (!private)
6242	return -ENOMEM;
6243	private->last_index = last_index;
6244	private->filldir_buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
6245	if (!private->filldir_buf) {
6246	kfree(private);
6247	return -ENOMEM;
6248	}
6249	file->private_data = private;
6250	return 0;
6251	}
6252
6253	static loff_t btrfs_dir_llseek(struct file *file, loff_t offset, int whence)
6254	{
6255	struct btrfs_file_private *private = file->private_data;
6256	int ret;
6257
6258	ret = btrfs_get_dir_last_index(BTRFS_I(file_inode(file)),
6259	&private->last_index);
6260	if (ret)
6261	return ret;
6262
6263	return generic_file_llseek(file, offset, whence);
6264	}
6265
6266	struct dir_entry {
6267	u64 ino;
6268	u64 offset;
6269	unsigned type;
6270	int name_len;
6271	};
6272
6273	static int btrfs_filldir(void addr, int entries, struct dir_context ctx)
6274	{
6275	while (entries--) {
6276	struct dir_entry *entry = addr;
6277	char name = (char )(entry + 1);
6278
6279	ctx->pos = get_unaligned(&entry->offset);
6280	if (!dir_emit(ctx, name, get_unaligned(&entry->name_len),
6281	get_unaligned(&entry->ino),
6282	get_unaligned(&entry->type)))
6283	return 1;
6284	addr += sizeof(struct dir_entry) +
6285	get_unaligned(&entry->name_len);
6286	ctx->pos++;
6287	}
6288	return 0;
6289	}
6290
6291	static int btrfs_real_readdir(struct file file, struct dir_context ctx)
6292	{
6293	struct inode *inode = file_inode(file);
6294	struct btrfs_root *root = BTRFS_I(inode)->root;
6295	struct btrfs_file_private *private = file->private_data;
6296	struct btrfs_dir_item *di;
6297	struct btrfs_key key;
6298	struct btrfs_key found_key;
6299	BTRFS_PATH_AUTO_FREE(path);
6300	void *addr;
6301	LIST_HEAD(ins_list);
6302	LIST_HEAD(del_list);
6303	int ret;
6304	char *name_ptr;
6305	int name_len;
6306	int entries = 0;
6307	int total_len = 0;
6308	bool put = false;
6309	struct btrfs_key location;
6310
6311	if (!dir_emit_dots(file, ctx))
6312	return 0;
6313
6314	path = btrfs_alloc_path();
6315	if (!path)
6316	return -ENOMEM;
6317
6318	addr = private->filldir_buf;
6319	path->reada = READA_FORWARD;
6320
6321	put = btrfs_readdir_get_delayed_items(BTRFS_I(inode), private->last_index,
6322	&ins_list, &del_list);
6323
6324	again:
6325	key.type = BTRFS_DIR_INDEX_KEY;
6326	key.offset = ctx->pos;
6327	key.objectid = btrfs_ino(BTRFS_I(inode));
6328
6329	btrfs_for_each_slot(root, &key, &found_key, path, ret) {
6330	struct dir_entry *entry;
6331	struct extent_buffer *leaf = path->nodes[0];
6332	u8 ftype;
6333
6334	if (found_key.objectid != key.objectid)
6335	break;
6336	if (found_key.type != BTRFS_DIR_INDEX_KEY)
6337	break;
6338	if (found_key.offset < ctx->pos)
6339	continue;
6340	if (found_key.offset > private->last_index)
6341	break;
6342	if (btrfs_should_delete_dir_index(&del_list, found_key.offset))
6343	continue;
6344	di = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dir_item);
6345	name_len = btrfs_dir_name_len(leaf, di);
6346	if ((total_len + sizeof(struct dir_entry) + name_len) >=
6347	PAGE_SIZE) {
6348	btrfs_release_path(path);
6349	ret = btrfs_filldir(private->filldir_buf, entries, ctx);
6350	if (ret)
6351	goto nopos;
6352	addr = private->filldir_buf;
6353	entries = 0;
6354	total_len = 0;
6355	goto again;
6356	}
6357
6358	ftype = btrfs_dir_flags_to_ftype(btrfs_dir_flags(leaf, di));
6359	entry = addr;
6360	name_ptr = (char *)(entry + 1);
6361	read_extent_buffer(leaf, name_ptr,
6362	(unsigned long)(di + 1), name_len);
6363	put_unaligned(name_len, &entry->name_len);
6364	put_unaligned(fs_ftype_to_dtype(ftype), &entry->type);
6365	btrfs_dir_item_key_to_cpu(leaf, di, &location);
6366	put_unaligned(location.objectid, &entry->ino);
6367	put_unaligned(found_key.offset, &entry->offset);
6368	entries++;
6369	addr += sizeof(struct dir_entry) + name_len;
6370	total_len += sizeof(struct dir_entry) + name_len;
6371	}
6372	/* Catch error encountered during iteration */
6373	if (ret < 0)
6374	goto err;
6375
6376	btrfs_release_path(path);
6377
6378	ret = btrfs_filldir(private->filldir_buf, entries, ctx);
6379	if (ret)
6380	goto nopos;
6381
6382	if (btrfs_readdir_delayed_dir_index(ctx, &ins_list))
6383	goto nopos;
6384
6385	/*
6386	* Stop new entries from being returned after we return the last
6387	* entry.
6388	*
6389	* New directory entries are assigned a strictly increasing
6390	* offset. This means that new entries created during readdir
6391	* are guaranteed to be seen in the future by that readdir.
6392	* This has broken buggy programs which operate on names as
6393	* they're returned by readdir. Until we reuse freed offsets
6394	* we have this hack to stop new entries from being returned
6395	* under the assumption that they'll never reach this huge
6396	* offset.
6397	*
6398	* This is being careful not to overflow 32bit loff_t unless the
6399	* last entry requires it because doing so has broken 32bit apps
6400	* in the past.
6401	*/
6402	if (ctx->pos >= INT_MAX)
6403	ctx->pos = LLONG_MAX;
6404	else
6405	ctx->pos = INT_MAX;
6406	nopos:
6407	ret = 0;
6408	err:
6409	if (put)
6410	btrfs_readdir_put_delayed_items(BTRFS_I(inode), &ins_list, &del_list);
6411	return ret;
6412	}
6413
6414	/*
6415	* This is somewhat expensive, updating the tree every time the
6416	* inode changes. But, it is most likely to find the inode in cache.
6417	* FIXME, needs more benchmarking...there are no reasons other than performance
6418	* to keep or drop this code.
6419	*/
6420	static int btrfs_dirty_inode(struct btrfs_inode *inode)
6421	{
6422	struct btrfs_root *root = inode->root;
6423	struct btrfs_fs_info *fs_info = root->fs_info;
6424	struct btrfs_trans_handle *trans;
6425	int ret;
6426
6427	if (test_bit(BTRFS_INODE_DUMMY, &inode->runtime_flags))
6428	return 0;
6429
6430	trans = btrfs_join_transaction(root);
6431	if (IS_ERR(trans))
6432	return PTR_ERR(trans);
6433
6434	ret = btrfs_update_inode(trans, inode);
6435	if (ret == -ENOSPC \|\| ret == -EDQUOT) {
6436	/* whoops, lets try again with the full transaction */
6437	btrfs_end_transaction(trans);
6438	trans = btrfs_start_transaction(root, 1);
6439	if (IS_ERR(trans))
6440	return PTR_ERR(trans);
6441
6442	ret = btrfs_update_inode(trans, inode);
6443	}
6444	btrfs_end_transaction(trans);
6445	if (inode->delayed_node)
6446	btrfs_balance_delayed_items(fs_info);
6447
6448	return ret;
6449	}
6450
6451	/*
6452	* We need our own ->update_time so that we can return error on ENOSPC for
6453	* updating the inode in the case of file write and mmap writes.
6454	*/
6455	static int btrfs_update_time(struct inode *inode, enum fs_update_time type,
6456	unsigned int flags)
6457	{
6458	struct btrfs_root *root = BTRFS_I(inode)->root;
6459	int dirty;
6460
6461	if (btrfs_root_readonly(root))
6462	return -EROFS;
6463	if (flags & IOCB_NOWAIT)
6464	return -EAGAIN;
6465
6466	dirty = inode_update_time(inode, type, flags);
6467	if (dirty <= 0)
6468	return dirty;
6469	return btrfs_dirty_inode(BTRFS_I(inode));
6470	}
6471
6472	/*
6473	* helper to find a free sequence number in a given directory. This current
6474	* code is very simple, later versions will do smarter things in the btree
6475	*/
6476	int btrfs_set_inode_index(struct btrfs_inode dir, u64 index)
6477	{
6478	int ret = 0;
6479
6480	if (dir->index_cnt == (u64)-1) {
6481	ret = btrfs_inode_delayed_dir_index_count(dir);
6482	if (ret) {
6483	ret = btrfs_set_inode_index_count(dir);
6484	if (ret)
6485	return ret;
6486	}
6487	}
6488
6489	*index = dir->index_cnt;
6490	dir->index_cnt++;
6491
6492	return ret;
6493	}
6494
6495	static int btrfs_insert_inode_locked(struct inode *inode)
6496	{
6497	struct btrfs_iget_args args;
6498
6499	args.ino = btrfs_ino(BTRFS_I(inode));
6500	args.root = BTRFS_I(inode)->root;
6501
6502	return insert_inode_locked4(inode,
6503	btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root),
6504	btrfs_find_actor, &args);
6505	}
6506
6507	int btrfs_new_inode_prepare(struct btrfs_new_inode_args *args,
6508	unsigned int *trans_num_items)
6509	{
6510	struct inode *dir = args->dir;
6511	struct inode *inode = args->inode;
6512	int ret;
6513
6514	if (!args->orphan) {
6515	ret = fscrypt_setup_filename(dir, &args->dentry->d_name, 0,
6516	&args->fname);
6517	if (ret)
6518	return ret;
6519	}
6520
6521	ret = posix_acl_create(dir, &inode->i_mode, &args->default_acl, &args->acl);
6522	if (ret) {
6523	fscrypt_free_filename(&args->fname);
6524	return ret;
6525	}
6526
6527	/* 1 to add inode item */
6528	*trans_num_items = 1;
6529	/* 1 to add compression property */
6530	if (BTRFS_I(dir)->prop_compress)
6531	(*trans_num_items)++;
6532	/* 1 to add default ACL xattr */
6533	if (args->default_acl)
6534	(*trans_num_items)++;
6535	/* 1 to add access ACL xattr */
6536	if (args->acl)
6537	(*trans_num_items)++;
6538	#ifdef CONFIG_SECURITY
6539	/* 1 to add LSM xattr */
6540	if (dir->i_security)
6541	(*trans_num_items)++;
6542	#endif
6543	if (args->orphan) {
6544	/* 1 to add orphan item */
6545	(*trans_num_items)++;
6546	} else {
6547	/*
6548	* 1 to add dir item
6549	* 1 to add dir index
6550	* 1 to update parent inode item
6551	*
6552	* No need for 1 unit for the inode ref item because it is
6553	* inserted in a batch together with the inode item at
6554	* btrfs_create_new_inode().
6555	*/
6556	*trans_num_items += 3;
6557	}
6558	return 0;
6559	}
6560
6561	void btrfs_new_inode_args_destroy(struct btrfs_new_inode_args *args)
6562	{
6563	posix_acl_release(args->acl);
6564	posix_acl_release(args->default_acl);
6565	fscrypt_free_filename(&args->fname);
6566	}
6567
6568	/*
6569	* Inherit flags from the parent inode.
6570	*
6571	* Currently only the compression flags and the cow flags are inherited.
6572	*/
6573	static void btrfs_inherit_iflags(struct btrfs_inode inode, struct btrfs_inode dir)
6574	{
6575	unsigned int flags;
6576
6577	flags = dir->flags;
6578
6579	if (flags & BTRFS_INODE_NOCOMPRESS) {
6580	inode->flags &= ~BTRFS_INODE_COMPRESS;
6581	inode->flags \|= BTRFS_INODE_NOCOMPRESS;
6582	} else if (flags & BTRFS_INODE_COMPRESS) {
6583	inode->flags &= ~BTRFS_INODE_NOCOMPRESS;
6584	inode->flags \|= BTRFS_INODE_COMPRESS;
6585	}
6586
6587	if (flags & BTRFS_INODE_NODATACOW) {
6588	inode->flags \|= BTRFS_INODE_NODATACOW;
6589	if (S_ISREG(inode->vfs_inode.i_mode))
6590	inode->flags \|= BTRFS_INODE_NODATASUM;
6591	}
6592
6593	btrfs_sync_inode_flags_to_i_flags(inode);
6594	}
6595
6596	int btrfs_create_new_inode(struct btrfs_trans_handle *trans,
6597	struct btrfs_new_inode_args *args)
6598	{
6599	struct timespec64 ts;
6600	struct inode *dir = args->dir;
6601	struct inode *inode = args->inode;
6602	const struct fscrypt_str *name = args->orphan ? NULL : &args->fname.disk_name;
6603	struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);
6604	struct btrfs_root *root;
6605	struct btrfs_inode_item *inode_item;
6606	struct btrfs_path *path;
6607	u64 objectid;
6608	struct btrfs_inode_ref *ref;
6609	struct btrfs_key key[2];
6610	u32 sizes[2];
6611	struct btrfs_item_batch batch;
6612	unsigned long ptr;
6613	int ret;
6614	bool xa_reserved = false;
6615
6616	if (!args->orphan && !args->subvol) {
6617	/*
6618	* Before anything else, check if we can add the name to the
6619	* parent directory. We want to avoid a dir item overflow in
6620	* case we have an existing dir item due to existing name
6621	* hash collisions. We do this check here before we call
6622	* btrfs_add_link() down below so that we can avoid a
6623	* transaction abort (which could be exploited by malicious
6624	* users).
6625	*
6626	* For subvolumes we already do this in btrfs_mksubvol().
6627	*/
6628	ret = btrfs_check_dir_item_collision(BTRFS_I(dir)->root,
6629	btrfs_ino(BTRFS_I(dir)),
6630	name);
6631	if (ret < 0)
6632	return ret;
6633	}
6634
6635	path = btrfs_alloc_path();
6636	if (!path)
6637	return -ENOMEM;
6638
6639	if (!args->subvol)
6640	BTRFS_I(inode)->root = btrfs_grab_root(BTRFS_I(dir)->root);
6641	root = BTRFS_I(inode)->root;
6642
6643	ret = btrfs_init_file_extent_tree(BTRFS_I(inode));
6644	if (ret)
6645	goto out;
6646
6647	ret = btrfs_get_free_objectid(root, &objectid);
6648	if (ret)
6649	goto out;
6650	btrfs_set_inode_number(BTRFS_I(inode), objectid);
6651
6652	ret = xa_reserve(&root->inodes, objectid, GFP_NOFS);
6653	if (ret)
6654	goto out;
6655	xa_reserved = true;
6656
6657	if (args->orphan) {
6658	/*
6659	* O_TMPFILE, set link count to 0, so that after this point, we
6660	* fill in an inode item with the correct link count.
6661	*/
6662	set_nlink(inode, 0);
6663	} else {
6664	trace_btrfs_inode_request(dir);
6665
6666	ret = btrfs_set_inode_index(BTRFS_I(dir), &BTRFS_I(inode)->dir_index);
6667	if (ret)
6668	goto out;
6669	}
6670
6671	if (S_ISDIR(inode->i_mode))
6672	BTRFS_I(inode)->index_cnt = BTRFS_DIR_START_INDEX;
6673
6674	BTRFS_I(inode)->generation = trans->transid;
6675	inode->i_generation = BTRFS_I(inode)->generation;
6676
6677	/*
6678	* We don't have any capability xattrs set here yet, shortcut any
6679	* queries for the xattrs here. If we add them later via the inode
6680	* security init path or any other path this flag will be cleared.
6681	*/
6682	set_bit(BTRFS_INODE_NO_CAP_XATTR, &BTRFS_I(inode)->runtime_flags);
6683
6684	/*
6685	* Subvolumes don't inherit flags from their parent directory.
6686	* Originally this was probably by accident, but we probably can't
6687	* change it now without compatibility issues.
6688	*/
6689	if (!args->subvol)
6690	btrfs_inherit_iflags(BTRFS_I(inode), BTRFS_I(dir));
6691
6692	btrfs_set_inode_mapping_order(BTRFS_I(inode));
6693	if (S_ISREG(inode->i_mode)) {
6694	if (btrfs_test_opt(fs_info, NODATASUM))
6695	BTRFS_I(inode)->flags \|= BTRFS_INODE_NODATASUM;
6696	if (btrfs_test_opt(fs_info, NODATACOW))
6697	BTRFS_I(inode)->flags \|= BTRFS_INODE_NODATACOW \|
6698	BTRFS_INODE_NODATASUM;
6699	btrfs_update_inode_mapping_flags(BTRFS_I(inode));
6700	}
6701
6702	ret = btrfs_insert_inode_locked(inode);
6703	if (ret < 0) {
6704	if (!args->orphan)
6705	BTRFS_I(dir)->index_cnt--;
6706	goto out;
6707	}
6708
6709	/*
6710	* We could have gotten an inode number from somebody who was fsynced
6711	* and then removed in this same transaction, so let's just set full
6712	* sync since it will be a full sync anyway and this will blow away the
6713	* old info in the log.
6714	*/
6715	btrfs_set_inode_full_sync(BTRFS_I(inode));
6716
6717	key[0].objectid = objectid;
6718	key[0].type = BTRFS_INODE_ITEM_KEY;
6719	key[0].offset = 0;
6720
6721	sizes[0] = sizeof(struct btrfs_inode_item);
6722
6723	if (!args->orphan) {
6724	/*
6725	* Start new inodes with an inode_ref. This is slightly more
6726	* efficient for small numbers of hard links since they will
6727	* be packed into one item. Extended refs will kick in if we
6728	* add more hard links than can fit in the ref item.
6729	*/
6730	key[1].objectid = objectid;
6731	key[1].type = BTRFS_INODE_REF_KEY;
6732	if (args->subvol) {
6733	key[1].offset = objectid;
6734	sizes[1] = 2 + sizeof(*ref);
6735	} else {
6736	key[1].offset = btrfs_ino(BTRFS_I(dir));
6737	sizes[1] = name->len + sizeof(*ref);
6738	}
6739	}
6740
6741	batch.keys = &key[0];
6742	batch.data_sizes = &sizes[0];
6743	batch.total_data_size = sizes[0] + (args->orphan ? 0 : sizes[1]);
6744	batch.nr = args->orphan ? 1 : 2;
6745	ret = btrfs_insert_empty_items(trans, root, path, &batch);
6746	if (unlikely(ret != 0)) {
6747	btrfs_abort_transaction(trans, ret);
6748	goto discard;
6749	}
6750
6751	ts = simple_inode_init_ts(inode);
6752	BTRFS_I(inode)->i_otime_sec = ts.tv_sec;
6753	BTRFS_I(inode)->i_otime_nsec = ts.tv_nsec;
6754
6755	/*
6756	* We're going to fill the inode item now, so at this point the inode
6757	* must be fully initialized.
6758	*/
6759
6760	inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0],
6761	struct btrfs_inode_item);
6762	memzero_extent_buffer(path->nodes[0], (unsigned long)inode_item,
6763	sizeof(*inode_item));
6764	fill_inode_item(trans, path->nodes[0], inode_item, inode);
6765
6766	if (!args->orphan) {
6767	ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1,
6768	struct btrfs_inode_ref);
6769	ptr = (unsigned long)(ref + 1);
6770	if (args->subvol) {
6771	btrfs_set_inode_ref_name_len(path->nodes[0], ref, 2);
6772	btrfs_set_inode_ref_index(path->nodes[0], ref, 0);
6773	write_extent_buffer(path->nodes[0], "..", ptr, 2);
6774	} else {
6775	btrfs_set_inode_ref_name_len(path->nodes[0], ref,
6776	name->len);
6777	btrfs_set_inode_ref_index(path->nodes[0], ref,
6778	BTRFS_I(inode)->dir_index);
6779	write_extent_buffer(path->nodes[0], name->name, ptr,
6780	name->len);
6781	}
6782	}
6783
6784	/*
6785	* We don't need the path anymore, plus inheriting properties, adding
6786	* ACLs, security xattrs, orphan item or adding the link, will result in
6787	* allocating yet another path. So just free our path.
6788	*/
6789	btrfs_free_path(path);
6790	path = NULL;
6791
6792	if (args->subvol) {
6793	struct btrfs_inode *parent;
6794
6795	/*
6796	* Subvolumes inherit properties from their parent subvolume,
6797	* not the directory they were created in.
6798	*/
6799	parent = btrfs_iget(BTRFS_FIRST_FREE_OBJECTID, BTRFS_I(dir)->root);
6800	if (IS_ERR(parent)) {
6801	ret = PTR_ERR(parent);
6802	} else {
6803	ret = btrfs_inode_inherit_props(trans, BTRFS_I(inode),
6804	parent);
6805	iput(&parent->vfs_inode);
6806	}
6807	} else {
6808	ret = btrfs_inode_inherit_props(trans, BTRFS_I(inode),
6809	BTRFS_I(dir));
6810	}
6811	if (ret) {
6812	btrfs_err(fs_info,
6813	"error inheriting props for ino %llu (root %llu): %pe",
6814	btrfs_ino(BTRFS_I(inode)), btrfs_root_id(root), ERR_PTR(ret));
6815	}
6816
6817	/*
6818	* Subvolumes don't inherit ACLs or get passed to the LSM. This is
6819	* probably a bug.
6820	*/
6821	if (!args->subvol) {
6822	ret = btrfs_init_inode_security(trans, args);
6823	if (unlikely(ret)) {
6824	btrfs_abort_transaction(trans, ret);
6825	goto discard;
6826	}
6827	}
6828
6829	ret = btrfs_add_inode_to_root(BTRFS_I(inode), false);
6830	if (WARN_ON(ret)) {
6831	/* Shouldn't happen, we used xa_reserve() before. */
6832	btrfs_abort_transaction(trans, ret);
6833	goto discard;
6834	}
6835
6836	trace_btrfs_inode_new(inode);
6837	btrfs_set_inode_last_trans(trans, BTRFS_I(inode));
6838
6839	btrfs_update_root_times(trans, root);
6840
6841	if (args->orphan) {
6842	ret = btrfs_orphan_add(trans, BTRFS_I(inode));
6843	if (unlikely(ret)) {
6844	btrfs_abort_transaction(trans, ret);
6845	goto discard;
6846	}
6847	} else {
6848	ret = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), name,
6849	false, BTRFS_I(inode)->dir_index, NULL);
6850	if (ret == -ENOMEM) {
6851	/*
6852	* Orphan the new inode instead of aborting. The inode
6853	* item was already written with nlink 1, and discard's
6854	* eviction won't delete a bad inode, so nlink 0 must be
6855	* persisted here or orphan cleanup would see nlink > 0,
6856	* drop the orphan item, and leak the inode.
6857	*/
6858	clear_nlink(inode);
6859	/* btrfs_orphan_add() aborts the transaction on failure. */
6860	ret = btrfs_orphan_add(trans, BTRFS_I(inode));
6861	if (ret)
6862	goto discard;
6863	ret = btrfs_update_inode(trans, BTRFS_I(inode));
6864	if (ret) {
6865	btrfs_abort_transaction(trans, ret);
6866	goto discard;
6867	}
6868	ret = -ENOMEM;
6869	goto discard;
6870	} else if (unlikely(ret)) {
6871	btrfs_abort_transaction(trans, ret);
6872	goto discard;
6873	}
6874	}
6875
6876	return 0;
6877
6878	discard:
6879	/*
6880	* discard_new_inode() calls iput(), but the caller owns the reference
6881	* to the inode.
6882	*/
6883	ihold(inode);
6884	discard_new_inode(inode);
6885	out:
6886	if (xa_reserved)
6887	xa_release(&root->inodes, objectid);
6888
6889	btrfs_free_path(path);
6890	return ret;
6891	}
6892
6893	/*
6894	* utility function to add 'inode' into 'parent_inode' with
6895	* a give name and a given sequence number.
6896	* if 'add_backref' is true, also insert a backref from the
6897	* inode to the parent directory.
6898	*/
6899	int btrfs_add_link(struct btrfs_trans_handle *trans,
6900	struct btrfs_inode parent_inode, struct btrfs_inode inode,
6901	const struct fscrypt_str *name, bool add_backref, u64 index,
6902	struct btrfs_dir_index_prealloc *prealloc)
6903	{
6904	int ret = 0;
6905	struct btrfs_key key;
6906	struct btrfs_root *root = parent_inode->root;
6907	u64 ino = btrfs_ino(inode);
6908	u64 parent_ino = btrfs_ino(parent_inode);
6909
6910	if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6911	memcpy(&key, &inode->root->root_key, sizeof(key));
6912	} else {
6913	key.objectid = ino;
6914	key.type = BTRFS_INODE_ITEM_KEY;
6915	key.offset = 0;
6916	}
6917
6918	if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6919	ret = btrfs_add_root_ref(trans, key.objectid,
6920	btrfs_root_id(root), parent_ino,
6921	index, name);
6922	} else if (add_backref) {
6923	ret = btrfs_insert_inode_ref(trans, root, name,
6924	ino, parent_ino, index);
6925	}
6926
6927	/* Nothing to clean up yet */
6928	if (ret) {
6929	btrfs_free_delayed_dir_index_prealloc(trans, prealloc);
6930	return ret;
6931	}
6932
6933	ret = btrfs_insert_dir_item(trans, name, parent_inode, &key,
6934	btrfs_inode_type(inode), index, prealloc);
6935	if (ret == -EEXIST \|\| ret == -EOVERFLOW \|\| ret == -ENOMEM)
6936	goto fail_dir_item;
6937	else if (unlikely(ret)) {
6938	btrfs_abort_transaction(trans, ret);
6939	return ret;
6940	}
6941
6942	btrfs_i_size_write(parent_inode, parent_inode->vfs_inode.i_size +
6943	name->len * 2);
6944	inode_inc_iversion(&parent_inode->vfs_inode);
6945	update_time_after_link_or_unlink(parent_inode);
6946
6947	ret = btrfs_update_inode(trans, parent_inode);
6948	if (ret)
6949	btrfs_abort_transaction(trans, ret);
6950	return ret;
6951
6952	fail_dir_item:
6953	if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6954	u64 local_index;
6955	int ret2;
6956
6957	ret2 = btrfs_del_root_ref(trans, key.objectid, btrfs_root_id(root),
6958	parent_ino, &local_index, name);
6959	if (ret2)
6960	btrfs_abort_transaction(trans, ret2);
6961	} else if (add_backref) {
6962	int ret2;
6963
6964	ret2 = btrfs_del_inode_ref(trans, root, name, ino, parent_ino, NULL);
6965	if (ret2)
6966	btrfs_abort_transaction(trans, ret2);
6967	}
6968
6969	/* Return the original error code */
6970	return ret;
6971	}
6972
6973	static int btrfs_create_common(struct inode dir, struct dentry dentry,
6974	struct inode *inode)
6975	{
6976	struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);
6977	struct btrfs_root *root = BTRFS_I(dir)->root;
6978	struct btrfs_new_inode_args new_inode_args = {
6979	.dir = dir,
6980	.dentry = dentry,
6981	.inode = inode,
6982	};
6983	unsigned int trans_num_items;
6984	struct btrfs_trans_handle *trans;
6985	int ret;
6986
6987	ret = btrfs_new_inode_prepare(&new_inode_args, &trans_num_items);
6988	if (ret)
6989	goto out_inode;
6990
6991	trans = btrfs_start_transaction(root, trans_num_items);
6992	if (IS_ERR(trans)) {
6993	ret = PTR_ERR(trans);
6994	goto out_new_inode_args;
6995	}
6996
6997	ret = btrfs_create_new_inode(trans, &new_inode_args);
6998	if (!ret) {
6999	if (S_ISDIR(inode->i_mode))
7000	inode->i_opflags \|= IOP_FASTPERM_MAY_EXEC;
7001	d_instantiate_new(dentry, inode);
7002	}
7003
7004	btrfs_end_transaction(trans);
7005	btrfs_btree_balance_dirty(fs_info);
7006	out_new_inode_args:
7007	btrfs_new_inode_args_destroy(&new_inode_args);
7008	out_inode:
7009	if (ret)
7010	iput(inode);
7011	return ret;
7012	}
7013
7014	static int btrfs_mknod(const struct mnt_idmap idmap, struct inode dir,
7015	struct dentry *dentry, umode_t mode, dev_t rdev)
7016	{
7017	struct inode *inode;
7018
7019	inode = new_inode(dir->i_sb);
7020	if (!inode)
7021	return -ENOMEM;
7022	inode_init_owner(idmap, inode, dir, mode);
7023	inode->i_op = &btrfs_special_inode_operations;
7024	init_special_inode(inode, inode->i_mode, rdev);
7025	return btrfs_create_common(dir, dentry, inode);
7026	}
7027
7028	static int btrfs_create(const struct mnt_idmap idmap, struct inode dir,
7029	struct dentry *dentry, umode_t mode)
7030	{
7031	struct inode *inode;
7032
7033	inode = new_inode(dir->i_sb);
7034	if (!inode)
7035	return -ENOMEM;
7036	inode_init_owner(idmap, inode, dir, mode);
7037	inode->i_fop = &btrfs_file_operations;
7038	inode->i_op = &btrfs_file_inode_operations;
7039	inode->i_mapping->a_ops = &btrfs_aops;
7040	return btrfs_create_common(dir, dentry, inode);
7041	}
7042
7043	static int btrfs_link(struct dentry old_dentry, struct inode dir,
7044	struct dentry *dentry)
7045	{
7046	struct btrfs_trans_handle *trans = NULL;
7047	struct btrfs_root *root = BTRFS_I(dir)->root;
7048	struct inode *inode = d_inode(old_dentry);
7049	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
7050	struct fscrypt_name fname;
7051	u64 index;
7052	int ret;
7053
7054	/* do not allow sys_link's with other subvols of the same device */
7055	if (btrfs_root_id(root) != btrfs_root_id(BTRFS_I(inode)->root))
7056	return -EXDEV;
7057
7058	if (inode->i_nlink >= BTRFS_LINK_MAX)
7059	return -EMLINK;
7060
7061	ret = fscrypt_setup_filename(dir, &dentry->d_name, 0, &fname);
7062	if (ret)
7063	goto fail;
7064
7065	ret = btrfs_set_inode_index(BTRFS_I(dir), &index);
7066	if (ret)
7067	goto fail;
7068
7069	/*
7070	* 2 items for inode and inode ref
7071	* 2 items for dir items
7072	* 1 item for parent inode
7073	* 1 item for orphan item deletion if O_TMPFILE
7074	*/
7075	trans = btrfs_start_transaction(root, inode->i_nlink ? 5 : 6);
7076	if (IS_ERR(trans)) {
7077	ret = PTR_ERR(trans);
7078	trans = NULL;
7079	goto fail;
7080	}
7081
7082	/* There are several dir indexes for this inode, clear the cache. */
7083	BTRFS_I(inode)->dir_index = 0ULL;
7084	inode_inc_iversion(inode);
7085	inode_set_ctime_current(inode);
7086
7087	ret = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode),
7088	&fname.disk_name, true, index, NULL);
7089	if (ret)
7090	goto fail;
7091
7092	/* Link added now we update the inode item with the new link count. */
7093	inc_nlink(inode);
7094	ret = btrfs_update_inode(trans, BTRFS_I(inode));
7095	if (unlikely(ret)) {
7096	btrfs_abort_transaction(trans, ret);
7097	goto fail;
7098	}
7099
7100	if (inode->i_nlink == 1) {
7101	/*
7102	* If the new hard link count is 1, it's a file created with the
7103	* open(2) O_TMPFILE flag.
7104	*/
7105	ret = btrfs_orphan_del(trans, BTRFS_I(inode));
7106	if (unlikely(ret)) {
7107	btrfs_abort_transaction(trans, ret);
7108	goto fail;
7109	}
7110	}
7111
7112	/* Grab reference for the new dentry passed to d_instantiate(). */
7113	ihold(inode);
7114	d_instantiate(dentry, inode);
7115	btrfs_log_new_name(trans, old_dentry, NULL, 0, dentry->d_parent);
7116
7117	fail:
7118	fscrypt_free_filename(&fname);
7119	if (trans)
7120	btrfs_end_transaction(trans);
7121	btrfs_btree_balance_dirty(fs_info);
7122	return ret;
7123	}
7124
7125	static struct dentry btrfs_mkdir(const struct mnt_idmap idmap, struct inode *dir,
7126	struct dentry *dentry, umode_t mode)
7127	{
7128	struct inode *inode;
7129
7130	inode = new_inode(dir->i_sb);
7131	if (!inode)
7132	return ERR_PTR(-ENOMEM);
7133	inode_init_owner(idmap, inode, dir, mode);
7134	inode->i_op = &btrfs_dir_inode_operations;
7135	inode->i_fop = &btrfs_dir_file_operations;
7136	return ERR_PTR(btrfs_create_common(dir, dentry, inode));
7137	}
7138
7139	static noinline int uncompress_inline(struct btrfs_path *path,
7140	struct folio *folio,
7141	struct btrfs_file_extent_item *item)
7142	{
7143	int ret;
7144	struct extent_buffer *leaf = path->nodes[0];
7145	const u32 blocksize = leaf->fs_info->sectorsize;
7146	char *tmp;
7147	size_t max_size;
7148	unsigned long inline_size;
7149	unsigned long ptr;
7150	int compress_type;
7151
7152	compress_type = btrfs_file_extent_compression(leaf, item);
7153	max_size = btrfs_file_extent_ram_bytes(leaf, item);
7154	inline_size = btrfs_file_extent_inline_item_len(leaf, path->slots[0]);
7155	tmp = kvmalloc(inline_size, GFP_NOFS);
7156	if (!tmp)
7157	return -ENOMEM;
7158	ptr = btrfs_file_extent_inline_start(item);
7159
7160	read_extent_buffer(leaf, tmp, ptr, inline_size);
7161
7162	max_size = min_t(unsigned long, blocksize, max_size);
7163	ret = btrfs_decompress(compress_type, tmp, folio, 0, inline_size,
7164	max_size);
7165
7166	/*
7167	* decompression code contains a memset to fill in any space between the end
7168	* of the uncompressed data and the end of max_size in case the decompressed
7169	* data ends up shorter than ram_bytes. That doesn't cover the hole between
7170	* the end of an inline extent and the beginning of the next block, so we
7171	* cover that region here.
7172	*/
7173
7174	if (max_size < blocksize)
7175	folio_zero_range(folio, max_size, blocksize - max_size);
7176	kvfree(tmp);
7177	return ret;
7178	}
7179
7180	static int read_inline_extent(struct btrfs_path path, struct folio folio)
7181	{
7182	const u32 blocksize = path->nodes[0]->fs_info->sectorsize;
7183	struct btrfs_file_extent_item *fi;
7184	void *kaddr;
7185	size_t copy_size;
7186
7187	if (!folio \|\| folio_test_uptodate(folio))
7188	return 0;
7189
7190	ASSERT(folio_pos(folio) == 0);
7191
7192	fi = btrfs_item_ptr(path->nodes[0], path->slots[0],
7193	struct btrfs_file_extent_item);
7194	if (btrfs_file_extent_compression(path->nodes[0], fi) != BTRFS_COMPRESS_NONE)
7195	return uncompress_inline(path, folio, fi);
7196
7197	copy_size = min_t(u64, blocksize,
7198	btrfs_file_extent_ram_bytes(path->nodes[0], fi));
7199	kaddr = kmap_local_folio(folio, 0);
7200	read_extent_buffer(path->nodes[0], kaddr,
7201	btrfs_file_extent_inline_start(fi), copy_size);
7202	kunmap_local(kaddr);
7203	if (copy_size < blocksize)
7204	folio_zero_range(folio, copy_size, blocksize - copy_size);
7205	return 0;
7206	}
7207
7208	/*
7209	* Lookup the first extent overlapping a range in a file.
7210	*
7211	* @inode: file to search in
7212	* @page: page to read extent data into if the extent is inline
7213	* @start: file offset
7214	* @len: length of range starting at @start
7215	*
7216	* Return the first &struct extent_map which overlaps the given range, reading
7217	* it from the B-tree and caching it if necessary. Note that there may be more
7218	* extents which overlap the given range after the returned extent_map.
7219	*
7220	* If @page is not NULL and the extent is inline, this also reads the extent
7221	* data directly into the page and marks the extent up to date in the io_tree.
7222	*
7223	* Return: ERR_PTR on error, non-NULL extent_map on success.
7224	*/
7225	struct extent_map btrfs_get_extent(struct btrfs_inode inode,
7226	struct folio *folio, u64 start, u64 len)
7227	{
7228	struct btrfs_fs_info *fs_info = inode->root->fs_info;
7229	int ret = 0;
7230	u64 extent_start = 0;
7231	u64 extent_end = 0;
7232	u64 objectid = btrfs_ino(inode);
7233	int extent_type = -1;
7234	struct btrfs_path *path = NULL;
7235	struct btrfs_root *root = inode->root;
7236	struct btrfs_file_extent_item *item;
7237	struct extent_buffer *leaf;
7238	struct btrfs_key found_key;
7239	struct extent_map *em = NULL;
7240	struct extent_map_tree *em_tree = &inode->extent_tree;
7241
7242	read_lock(&em_tree->lock);
7243	em = btrfs_lookup_extent_mapping(em_tree, start, len);
7244	read_unlock(&em_tree->lock);
7245
7246	if (em) {
7247	if (em->start > start \|\| btrfs_extent_map_end(em) <= start)
7248	btrfs_free_extent_map(em);
7249	else if (em->disk_bytenr == EXTENT_MAP_INLINE && folio)
7250	btrfs_free_extent_map(em);
7251	else
7252	goto out;
7253	}
7254	em = btrfs_alloc_extent_map();
7255	if (!em) {
7256	ret = -ENOMEM;
7257	goto out;
7258	}
7259	em->start = EXTENT_MAP_HOLE;
7260	em->disk_bytenr = EXTENT_MAP_HOLE;
7261	em->len = (u64)-1;
7262
7263	path = btrfs_alloc_path();
7264	if (!path) {
7265	ret = -ENOMEM;
7266	goto out;
7267	}
7268
7269	/* Chances are we'll be called again, so go ahead and do readahead */
7270	path->reada = READA_FORWARD;
7271
7272	/*
7273	* The same explanation in load_free_space_cache applies here as well,
7274	* we only read when we're loading the free space cache, and at that
7275	* point the commit_root has everything we need.
7276	*/
7277	if (btrfs_is_free_space_inode(inode)) {
7278	path->search_commit_root = true;
7279	path->skip_locking = true;
7280	}
7281
7282	ret = btrfs_lookup_file_extent(NULL, root, path, objectid, start, 0);
7283	if (ret < 0) {
7284	goto out;
7285	} else if (ret > 0) {
7286	if (path->slots[0] == 0)
7287	goto not_found;
7288	path->slots[0]--;
7289	ret = 0;
7290	}
7291
7292	leaf = path->nodes[0];
7293	item = btrfs_item_ptr(leaf, path->slots[0],
7294	struct btrfs_file_extent_item);
7295	btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
7296	if (found_key.objectid != objectid \|\|
7297	found_key.type != BTRFS_EXTENT_DATA_KEY) {
7298	/*
7299	* If we backup past the first extent we want to move forward
7300	* and see if there is an extent in front of us, otherwise we'll
7301	* say there is a hole for our whole search range which can
7302	* cause problems.
7303	*/
7304	extent_end = start;
7305	goto next;
7306	}
7307
7308	extent_type = btrfs_file_extent_type(leaf, item);
7309	extent_start = found_key.offset;
7310	extent_end = btrfs_file_extent_end(path);
7311	if (extent_type == BTRFS_FILE_EXTENT_REG \|\|
7312	extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
7313	/* Only regular file could have regular/prealloc extent */
7314	if (unlikely(!S_ISREG(inode->vfs_inode.i_mode))) {
7315	ret = -EUCLEAN;
7316	btrfs_crit(fs_info,
7317	"regular/prealloc extent found for non-regular inode %llu",
7318	btrfs_ino(inode));
7319	goto out;
7320	}
7321	trace_btrfs_get_extent_show_fi_regular(inode, leaf, item,
7322	extent_start);
7323	} else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
7324	trace_btrfs_get_extent_show_fi_inline(inode, leaf, item,
7325	path->slots[0],
7326	extent_start);
7327	}
7328	next:
7329	if (start >= extent_end) {
7330	path->slots[0]++;
7331	if (path->slots[0] >= btrfs_header_nritems(leaf)) {
7332	ret = btrfs_next_leaf(root, path);
7333	if (ret < 0)
7334	goto out;
7335	else if (ret > 0)
7336	goto not_found;
7337
7338	leaf = path->nodes[0];
7339	}
7340	btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
7341	if (found_key.objectid != objectid \|\|
7342	found_key.type != BTRFS_EXTENT_DATA_KEY)
7343	goto not_found;
7344	if (start + len <= found_key.offset)
7345	goto not_found;
7346	if (start > found_key.offset)
7347	goto next;
7348
7349	/* New extent overlaps with existing one */
7350	em->start = start;
7351	em->len = found_key.offset - start;
7352	em->disk_bytenr = EXTENT_MAP_HOLE;
7353	goto insert;
7354	}
7355
7356	btrfs_extent_item_to_extent_map(inode, path, item, em);
7357
7358	if (extent_type == BTRFS_FILE_EXTENT_REG \|\|
7359	extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
7360	goto insert;
7361	} else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
7362	/*
7363	* Inline extent can only exist at file offset 0. This is
7364	* ensured by tree-checker and inline extent creation path.
7365	* Thus all members representing file offsets should be zero.
7366	*/
7367	ASSERT(extent_start == 0);
7368	ASSERT(em->start == 0);
7369
7370	/*
7371	* btrfs_extent_item_to_extent_map() should have properly
7372	* initialized em members already.
7373	*
7374	* Other members are not utilized for inline extents.
7375	*/
7376	ASSERT(em->disk_bytenr == EXTENT_MAP_INLINE);
7377	ASSERT(em->len == fs_info->sectorsize);
7378
7379	ret = read_inline_extent(path, folio);
7380	if (ret < 0)
7381	goto out;
7382	goto insert;
7383	}
7384	not_found:
7385	em->start = start;
7386	em->len = len;
7387	em->disk_bytenr = EXTENT_MAP_HOLE;
7388	insert:
7389	ret = 0;
7390	btrfs_release_path(path);
7391	if (unlikely(em->start > start \|\| btrfs_extent_map_end(em) <= start)) {
7392	btrfs_err(fs_info,
7393	"bad extent! em: [%llu %llu] passed [%llu %llu]",
7394	em->start, em->len, start, len);
7395	ret = -EIO;
7396	goto out;
7397	}
7398
7399	write_lock(&em_tree->lock);
7400	ret = btrfs_add_extent_mapping(inode, &em, start, len);
7401	write_unlock(&em_tree->lock);
7402	out:
7403	btrfs_free_path(path);
7404
7405	trace_btrfs_get_extent(root, inode, em);
7406
7407	if (ret) {
7408	btrfs_free_extent_map(em);
7409	return ERR_PTR(ret);
7410	}
7411	return em;
7412	}
7413
7414	static bool btrfs_extent_readonly(struct btrfs_fs_info *fs_info, u64 bytenr)
7415	{
7416	struct btrfs_block_group *block_group;
7417	bool readonly = false;
7418
7419	block_group = btrfs_lookup_block_group(fs_info, bytenr);
7420	if (!block_group \|\| block_group->ro)
7421	readonly = true;
7422	if (block_group)
7423	btrfs_put_block_group(block_group);
7424	return readonly;
7425	}
7426
7427	/*
7428	* Check if we can do nocow write into the range [@offset, @offset + @len)
7429	*
7430	* @offset: File offset
7431	* @len: The length to write, will be updated to the nocow writeable
7432	* range
7433	* @orig_start: (optional) Return the original file offset of the file extent
7434	* @orig_len: (optional) Return the original on-disk length of the file extent
7435	* @ram_bytes: (optional) Return the ram_bytes of the file extent
7436	*
7437	* Return:
7438	* >0 and update @len if we can do nocow write
7439	* 0 if we can't do nocow write
7440	* <0 if error happened
7441	*
7442	* NOTE: This only checks the file extents, caller is responsible to wait for
7443	* any ordered extents.
7444	*/
7445	noinline int can_nocow_extent(struct btrfs_inode inode, u64 offset, u64 len,
7446	struct btrfs_file_extent *file_extent,
7447	bool nowait)
7448	{
7449	struct btrfs_root *root = inode->root;
7450	struct btrfs_fs_info *fs_info = root->fs_info;
7451	struct can_nocow_file_extent_args nocow_args = { 0 };
7452	BTRFS_PATH_AUTO_FREE(path);
7453	int ret;
7454	struct extent_buffer *leaf;
7455	struct extent_io_tree *io_tree = &inode->io_tree;
7456	struct btrfs_file_extent_item *fi;
7457	struct btrfs_key key;
7458	int found_type;
7459
7460	path = btrfs_alloc_path();
7461	if (!path)
7462	return -ENOMEM;
7463	path->nowait = nowait;
7464
7465	ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode),
7466	offset, 0);
7467	if (ret < 0)
7468	return ret;
7469
7470	if (ret == 1) {
7471	if (path->slots[0] == 0) {
7472	/* Can't find the item, must COW. */
7473	return 0;
7474	}
7475	path->slots[0]--;
7476	}
7477	ret = 0;
7478	leaf = path->nodes[0];
7479	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
7480	if (key.objectid != btrfs_ino(inode) \|\|
7481	key.type != BTRFS_EXTENT_DATA_KEY) {
7482	/* Not our file or wrong item type, must COW. */
7483	return 0;
7484	}
7485
7486	if (key.offset > offset) {
7487	/* Wrong offset, must COW. */
7488	return 0;
7489	}
7490
7491	if (btrfs_file_extent_end(path) <= offset)
7492	return 0;
7493
7494	fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
7495	found_type = btrfs_file_extent_type(leaf, fi);
7496
7497	nocow_args.start = offset;
7498	nocow_args.end = offset + *len - 1;
7499	nocow_args.free_path = true;
7500
7501	ret = can_nocow_file_extent(path, &key, inode, &nocow_args);
7502	/* can_nocow_file_extent() has freed the path. */
7503	path = NULL;
7504
7505	if (ret != 1) {
7506	/* Treat errors as not being able to NOCOW. */
7507	return 0;
7508	}
7509
7510	if (btrfs_extent_readonly(fs_info,
7511	nocow_args.file_extent.disk_bytenr +
7512	nocow_args.file_extent.offset))
7513	return 0;
7514
7515	if (!(inode->flags & BTRFS_INODE_NODATACOW) &&
7516	found_type == BTRFS_FILE_EXTENT_PREALLOC) {
7517	u64 range_end;
7518
7519	range_end = round_up(offset + nocow_args.file_extent.num_bytes,
7520	root->fs_info->sectorsize) - 1;
7521	ret = btrfs_test_range_bit_exists(io_tree, offset, range_end,
7522	EXTENT_DELALLOC);
7523	if (ret)
7524	return -EAGAIN;
7525	}
7526
7527	if (file_extent)
7528	memcpy(file_extent, &nocow_args.file_extent, sizeof(*file_extent));
7529
7530	*len = nocow_args.file_extent.num_bytes;
7531
7532	return 1;
7533	}
7534
7535	/* The callers of this must take lock_extent() */
7536	struct extent_map btrfs_create_io_em(struct btrfs_inode inode, u64 start,
7537	const struct btrfs_file_extent *file_extent,
7538	int type)
7539	{
7540	struct extent_map *em;
7541	int ret;
7542
7543	/*
7544	* Note the missing NOCOW type.
7545	*
7546	* For pure NOCOW writes, we should not create an io extent map, but
7547	* just reusing the existing one.
7548	* Only PREALLOC writes (NOCOW write into preallocated range) can
7549	* create an io extent map.
7550	*/
7551	ASSERT(type == BTRFS_ORDERED_PREALLOC \|\|
7552	type == BTRFS_ORDERED_COMPRESSED \|\|
7553	type == BTRFS_ORDERED_REGULAR);
7554
7555	switch (type) {
7556	case BTRFS_ORDERED_PREALLOC:
7557	/* We're only referring part of a larger preallocated extent. */
7558	ASSERT(file_extent->num_bytes <= file_extent->ram_bytes);
7559	break;
7560	case BTRFS_ORDERED_REGULAR:
7561	/* COW results a new extent matching our file extent size. */
7562	ASSERT(file_extent->disk_num_bytes == file_extent->num_bytes);
7563	ASSERT(file_extent->ram_bytes == file_extent->num_bytes);
7564
7565	/* Since it's a new extent, we should not have any offset. */
7566	ASSERT(file_extent->offset == 0);
7567	break;
7568	case BTRFS_ORDERED_COMPRESSED:
7569	/* Must be compressed. */
7570	ASSERT(file_extent->compression != BTRFS_COMPRESS_NONE);
7571
7572	/*
7573	* Encoded write can make us to refer to part of the
7574	* uncompressed extent.
7575	*/
7576	ASSERT(file_extent->num_bytes <= file_extent->ram_bytes);
7577	break;
7578	}
7579
7580	em = btrfs_alloc_extent_map();
7581	if (!em)
7582	return ERR_PTR(-ENOMEM);
7583
7584	em->start = start;
7585	em->len = file_extent->num_bytes;
7586	em->disk_bytenr = file_extent->disk_bytenr;
7587	em->disk_num_bytes = file_extent->disk_num_bytes;
7588	em->ram_bytes = file_extent->ram_bytes;
7589	em->generation = -1;
7590	em->offset = file_extent->offset;
7591	em->flags \|= EXTENT_FLAG_PINNED;
7592	if (type == BTRFS_ORDERED_COMPRESSED)
7593	btrfs_extent_map_set_compression(em, file_extent->compression);
7594
7595	ret = btrfs_replace_extent_map_range(inode, em, true);
7596	if (ret) {
7597	btrfs_free_extent_map(em);
7598	return ERR_PTR(ret);
7599	}
7600
7601	/* em got 2 refs now, callers needs to do btrfs_free_extent_map once. */
7602	return em;
7603	}
7604
7605	/*
7606	* For release_folio() and invalidate_folio() we have a race window where
7607	* folio_end_writeback() is called but the subpage spinlock is not yet released.
7608	* If we continue to release/invalidate the page, we could cause use-after-free
7609	* for subpage spinlock. So this function is to spin and wait for subpage
7610	* spinlock.
7611	*/
7612	static void wait_subpage_spinlock(struct folio *folio)
7613	{
7614	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
7615	struct btrfs_folio_state *bfs;
7616
7617	if (!btrfs_is_subpage(fs_info, folio))
7618	return;
7619
7620	ASSERT(folio_test_private(folio) && folio_get_private(folio));
7621	bfs = folio_get_private(folio);
7622
7623	/*
7624	* This may look insane as we just acquire the spinlock and release it,
7625	* without doing anything. But we just want to make sure no one is
7626	* still holding the subpage spinlock.
7627	* And since the page is not dirty nor writeback, and we have page
7628	* locked, the only possible way to hold a spinlock is from the endio
7629	* function to clear page writeback.
7630	*
7631	* Here we just acquire the spinlock so that all existing callers
7632	* should exit and we're safe to release/invalidate the page.
7633	*/
7634	spin_lock_irq(&bfs->lock);
7635	spin_unlock_irq(&bfs->lock);
7636	}
7637
7638	static int btrfs_launder_folio(struct folio *folio)
7639	{
7640	return btrfs_qgroup_free_data(folio_to_inode(folio), NULL, folio_pos(folio),
7641	folio_size(folio), NULL);
7642	}
7643
7644	static bool __btrfs_release_folio(struct folio *folio, gfp_t gfp_flags)
7645	{
7646	if (try_release_extent_mapping(folio, gfp_flags)) {
7647	wait_subpage_spinlock(folio);
7648	clear_folio_extent_mapped(folio);
7649	return true;
7650	}
7651	return false;
7652	}
7653
7654	static bool btrfs_release_folio(struct folio *folio, gfp_t gfp_flags)
7655	{
7656	if (folio_test_writeback(folio) \|\| folio_test_dirty(folio))
7657	return false;
7658	return __btrfs_release_folio(folio, gfp_flags);
7659	}
7660
7661	#ifdef CONFIG_MIGRATION
7662	static int btrfs_migrate_folio(struct address_space *mapping,
7663	struct folio dst, struct folio src,
7664	enum migrate_mode mode)
7665	{
7666	int ret = filemap_migrate_folio(mapping, dst, src, mode);
7667
7668	if (ret)
7669	return ret;
7670	return 0;
7671	}
7672	#else
7673	#define btrfs_migrate_folio NULL
7674	#endif
7675
7676	static void btrfs_invalidate_folio(struct folio *folio, size_t offset,
7677	size_t length)
7678	{
7679	struct btrfs_inode *inode = folio_to_inode(folio);
7680	struct btrfs_fs_info *fs_info = inode->root->fs_info;
7681	struct extent_io_tree *tree = &inode->io_tree;
7682	struct extent_state *cached_state = NULL;
7683	u64 page_start = folio_pos(folio);
7684	u64 page_end = page_start + folio_size(folio) - 1;
7685	u64 cur;
7686	int inode_evicting = inode_state_read_once(&inode->vfs_inode) & I_FREEING;
7687
7688	/*
7689	* We have folio locked so no new ordered extent can be created on this
7690	* page, nor bio can be submitted for this folio.
7691	*
7692	* But already submitted bio can still be finished on this folio.
7693	* Furthermore, endio function won't skip folio which has Ordered
7694	* already cleared, so it's possible for endio and
7695	* invalidate_folio to do the same ordered extent accounting twice
7696	* on one folio.
7697	*
7698	* So here we wait for any submitted bios to finish, so that we won't
7699	* do double ordered extent accounting on the same folio.
7700	*/
7701	folio_wait_writeback(folio);
7702	wait_subpage_spinlock(folio);
7703
7704	/*
7705	* The invalidated blocks are going away; drop any fixup blocks among
7706	* them, data included, as they have no space reservation.
7707	*/
7708	btrfs_folio_clear_fixup_dirty(fs_info, folio, page_start + offset, length);
7709
7710	/*
7711	* For subpage case, we have call sites like
7712	* btrfs_punch_hole_lock_range() which passes range not aligned to
7713	* sectorsize.
7714	* If the range doesn't cover the full folio, we don't need to and
7715	* shouldn't clear page extent mapped, as folio->private can still
7716	* record subpage dirty bits for other part of the range.
7717	*
7718	* For cases that invalidate the full folio even the range doesn't
7719	* cover the full folio, like invalidating the last folio, we're
7720	* still safe to wait for ordered extent to finish.
7721	*/
7722	if (!(offset == 0 && length == folio_size(folio))) {
7723	btrfs_release_folio(folio, GFP_NOFS);
7724	return;
7725	}
7726
7727	if (!inode_evicting)
7728	btrfs_lock_extent(tree, page_start, page_end, &cached_state);
7729
7730	cur = page_start;
7731	while (cur < page_end) {
7732	struct btrfs_ordered_extent *ordered;
7733	u64 range_end;
7734	u32 range_len;
7735	u32 extra_flags = 0;
7736
7737	ordered = btrfs_lookup_first_ordered_range(inode, cur,
7738	page_end + 1 - cur);
7739	if (!ordered) {
7740	range_end = page_end;
7741	/*
7742	* No ordered extent covering this range, we are safe
7743	* to delete all extent states in the range.
7744	*/
7745	extra_flags = EXTENT_CLEAR_ALL_BITS;
7746	goto next;
7747	}
7748	if (ordered->file_offset > cur) {
7749	/*
7750	* There is a range between [cur, oe->file_offset) not
7751	* covered by any ordered extent.
7752	* We are safe to delete all extent states, and handle
7753	* the ordered extent in the next iteration.
7754	*/
7755	range_end = ordered->file_offset - 1;
7756	extra_flags = EXTENT_CLEAR_ALL_BITS;
7757	goto next;
7758	}
7759
7760	range_end = min(ordered->file_offset + ordered->num_bytes - 1,
7761	page_end);
7762	ASSERT(range_end + 1 - cur < U32_MAX);
7763	range_len = range_end + 1 - cur;
7764	/*
7765	* If the range is not dirty, the range has been submitted and
7766	* since we have waited for the writeback, endio has been
7767	* executed, thus we must skip the range to avoid double
7768	* accounting for the ordered extent.
7769	*/
7770	if (!btrfs_folio_test_dirty(fs_info, folio, cur, range_len))
7771	goto next;
7772
7773	/*
7774	* The range is dirty meaning it has not been submitted.
7775	* Here we need to truncate the OE range as the range will never
7776	* be submitted.
7777	*
7778	* IO on this page will never be started, so we need to account
7779	* for any ordered extents now. Don't clear EXTENT_DELALLOC_NEW
7780	* here, must leave that up for the ordered extent completion.
7781	*
7782	* This will also unlock the range for incoming
7783	* btrfs_finish_ordered_io().
7784	*/
7785	if (!inode_evicting)
7786	btrfs_clear_extent_bit(tree, cur, range_end,
7787	EXTENT_DELALLOC \|
7788	EXTENT_LOCKED \| EXTENT_DO_ACCOUNTING \|
7789	EXTENT_DEFRAG, &cached_state);
7790
7791	btrfs_mark_ordered_extent_truncated(ordered, cur - ordered->file_offset);
7792
7793	/*
7794	* If the ordered extent has finished, we're safe to delete all
7795	* the extent states of the range, otherwise
7796	* btrfs_finish_ordered_io() will get executed by endio for
7797	* other pages, so we can't delete extent states.
7798	*/
7799	if (btrfs_dec_test_ordered_pending(inode, &ordered,
7800	cur, range_end + 1 - cur)) {
7801	btrfs_finish_ordered_io(ordered);
7802	/*
7803	* The ordered extent has finished, now we're again
7804	* safe to delete all extent states of the range.
7805	*/
7806	extra_flags = EXTENT_CLEAR_ALL_BITS;
7807	}
7808	next:
7809	if (ordered)
7810	btrfs_put_ordered_extent(ordered);
7811	/*
7812	* Qgroup reserved space handler
7813	* Sector(s) here will be either:
7814	*
7815	* 1) Already written to disk or bio already finished
7816	* Then its QGROUP_RESERVED bit in io_tree is already cleared.
7817	* Qgroup will be handled by its qgroup_record then.
7818	* btrfs_qgroup_free_data() call will do nothing here.
7819	*
7820	* 2) Not written to disk yet
7821	* Then btrfs_qgroup_free_data() call will clear the
7822	* QGROUP_RESERVED bit of its io_tree, and free the qgroup
7823	* reserved data space.
7824	* Since the IO will never happen for this page.
7825	*/
7826	btrfs_qgroup_free_data(inode, NULL, cur, range_end + 1 - cur, NULL);
7827	if (!inode_evicting)
7828	btrfs_clear_extent_bit(tree, cur, range_end, EXTENT_LOCKED \|
7829	EXTENT_DELALLOC \| EXTENT_DO_ACCOUNTING \|
7830	EXTENT_DEFRAG \| extra_flags,
7831	&cached_state);
7832	cur = range_end + 1;
7833	}
7834	btrfs_folio_clear_dirty(fs_info, folio, page_start, folio_size(folio));
7835	btrfs_clear_folio_dirty_tag(folio);
7836	if (!inode_evicting)
7837	__btrfs_release_folio(folio, GFP_NOFS);
7838	clear_folio_extent_mapped(folio);
7839	}
7840
7841	static int btrfs_truncate(struct btrfs_inode *inode, bool skip_writeback)
7842	{
7843	struct btrfs_truncate_control control = {
7844	.inode = inode,
7845	.ino = btrfs_ino(inode),
7846	.min_type = BTRFS_EXTENT_DATA_KEY,
7847	.clear_extent_range = true,
7848	.new_size = inode->vfs_inode.i_size,
7849	};
7850	struct btrfs_root *root = inode->root;
7851	struct btrfs_fs_info *fs_info = root->fs_info;
7852	struct btrfs_block_rsv rsv;
7853	int ret;
7854	struct btrfs_trans_handle *trans;
7855	const u64 min_size = btrfs_calc_metadata_size(fs_info, 1);
7856	const u64 lock_start = round_down(inode->vfs_inode.i_size, fs_info->sectorsize);
7857	const u64 i_size_up = round_up(inode->vfs_inode.i_size, fs_info->sectorsize);
7858
7859	/* Our inode is locked and the i_size can't be changed concurrently. */
7860	btrfs_assert_inode_locked(inode);
7861
7862	if (!skip_writeback) {
7863	ret = btrfs_wait_ordered_range(inode, lock_start, (u64)-1);
7864	if (ret)
7865	return ret;
7866	}
7867
7868	/*
7869	* Yes ladies and gentlemen, this is indeed ugly. We have a couple of
7870	* things going on here:
7871	*
7872	* 1) We need to reserve space to update our inode.
7873	*
7874	* 2) We need to have something to cache all the space that is going to
7875	* be free'd up by the truncate operation, but also have some slack
7876	* space reserved in case it uses space during the truncate (thank you
7877	* very much snapshotting).
7878	*
7879	* And we need these to be separate. The fact is we can use a lot of
7880	* space doing the truncate, and we have no earthly idea how much space
7881	* we will use, so we need the truncate reservation to be separate so it
7882	* doesn't end up using space reserved for updating the inode. We also
7883	* need to be able to stop the transaction and start a new one, which
7884	* means we need to be able to update the inode several times, and we
7885	* have no idea of knowing how many times that will be, so we can't just
7886	* reserve 1 item for the entirety of the operation, so that has to be
7887	* done separately as well.
7888	*
7889	* So that leaves us with
7890	*
7891	* 1) rsv - for the truncate reservation, which we will steal from the
7892	* transaction reservation.
7893	* 2) fs_info->trans_block_rsv - this will have 1 items worth left for
7894	* updating the inode.
7895	*/
7896	btrfs_init_metadata_block_rsv(fs_info, &rsv, BTRFS_BLOCK_RSV_TEMP);
7897	rsv.size = min_size;
7898	rsv.failfast = true;
7899
7900	/*
7901	* 1 for the truncate slack space
7902	* 1 for updating the inode.
7903	*/
7904	trans = btrfs_start_transaction(root, 2);
7905	if (IS_ERR(trans)) {
7906	ret = PTR_ERR(trans);
7907	goto out;
7908	}
7909
7910	/* Migrate the slack space for the truncate to our reserve */
7911	ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, &rsv,
7912	min_size, false);
7913	/*
7914	* We have reserved 2 metadata units when we started the transaction and
7915	* min_size matches 1 unit, so this should never fail, but if it does,
7916	* it's not critical we just fail truncation.
7917	*/
7918	if (WARN_ON(ret)) {
7919	btrfs_end_transaction(trans);
7920	goto out;
7921	}
7922
7923	trans->block_rsv = &rsv;
7924
7925	while (1) {
7926	struct extent_state *cached_state = NULL;
7927
7928	btrfs_lock_extent(&inode->io_tree, lock_start, (u64)-1, &cached_state);
7929	/*
7930	* We want to drop from the next block forward in case this new
7931	* size is not block aligned since we will be keeping the last
7932	* block of the extent just the way it is.
7933	*/
7934	btrfs_drop_extent_map_range(inode, i_size_up, (u64)-1, false);
7935
7936	ret = btrfs_truncate_inode_items(trans, root, &control);
7937
7938	inode_sub_bytes(&inode->vfs_inode, control.sub_bytes);
7939	btrfs_inode_safe_disk_i_size_write(inode, control.last_size);
7940
7941	btrfs_unlock_extent(&inode->io_tree, lock_start, (u64)-1, &cached_state);
7942
7943	trans->block_rsv = &fs_info->trans_block_rsv;
7944	if (ret != -ENOSPC && ret != -EAGAIN)
7945	break;
7946
7947	ret = btrfs_update_inode(trans, inode);
7948	if (ret)
7949	break;
7950
7951	btrfs_end_transaction(trans);
7952	btrfs_btree_balance_dirty(fs_info);
7953
7954	trans = btrfs_start_transaction(root, 2);
7955	if (IS_ERR(trans)) {
7956	ret = PTR_ERR(trans);
7957	trans = NULL;
7958	break;
7959	}
7960
7961	btrfs_block_rsv_release(fs_info, &rsv, -1, NULL);
7962	ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv,
7963	&rsv, min_size, false);
7964	/*
7965	* We have reserved 2 metadata units when we started the
7966	* transaction and min_size matches 1 unit, so this should never
7967	* fail, but if it does, it's not critical we just fail truncation.
7968	*/
7969	if (WARN_ON(ret))
7970	break;
7971
7972	trans->block_rsv = &rsv;
7973	}
7974
7975	/*
7976	* We can't call btrfs_truncate_block inside a trans handle as we could
7977	* deadlock with freeze, if we got BTRFS_NEED_TRUNCATE_BLOCK then we
7978	* know we've truncated everything except the last little bit, and can
7979	* do btrfs_truncate_block and then update the disk_i_size.
7980	*/
7981	if (ret == BTRFS_NEED_TRUNCATE_BLOCK) {
7982	btrfs_end_transaction(trans);
7983	btrfs_btree_balance_dirty(fs_info);
7984
7985	ret = btrfs_truncate_block(inode, inode->vfs_inode.i_size,
7986	inode->vfs_inode.i_size, (u64)-1);
7987	if (ret)
7988	goto out;
7989	trans = btrfs_start_transaction(root, 1);
7990	if (IS_ERR(trans)) {
7991	ret = PTR_ERR(trans);
7992	goto out;
7993	}
7994	btrfs_inode_safe_disk_i_size_write(inode, 0);
7995	}
7996
7997	if (trans) {
7998	int ret2;
7999
8000	trans->block_rsv = &fs_info->trans_block_rsv;
8001	ret2 = btrfs_update_inode(trans, inode);
8002	if (ret2 && !ret)
8003	ret = ret2;
8004
8005	ret2 = btrfs_end_transaction(trans);
8006	if (ret2 && !ret)
8007	ret = ret2;
8008	btrfs_btree_balance_dirty(fs_info);
8009	}
8010	out:
8011	btrfs_block_rsv_release(fs_info, &rsv, (u64)-1, NULL);
8012	/*
8013	* So if we truncate and then write and fsync we normally would just
8014	* write the extents that changed, which is a problem if we need to
8015	* first truncate that entire inode. So set this flag so we write out
8016	* all of the extents in the inode to the sync log so we're completely
8017	* safe.
8018	*
8019	* If no extents were dropped or trimmed we don't need to force the next
8020	* fsync to truncate all the inode's items from the log and re-log them
8021	* all. This means the truncate operation did not change the file size,
8022	* or changed it to a smaller size but there was only an implicit hole
8023	* between the old i_size and the new i_size, and there were no prealloc
8024	* extents beyond i_size to drop.
8025	*/
8026	if (control.extents_found > 0)
8027	btrfs_set_inode_full_sync(inode);
8028
8029	return ret;
8030	}
8031
8032	struct inode btrfs_new_subvol_inode(const struct mnt_idmap idmap,
8033	struct inode *dir)
8034	{
8035	struct inode *inode;
8036
8037	inode = new_inode(dir->i_sb);
8038	if (inode) {
8039	/*
8040	* Subvolumes don't inherit the sgid bit or the parent's gid if
8041	* the parent's sgid bit is set. This is probably a bug.
8042	*/
8043	inode_init_owner(idmap, inode, NULL,
8044	S_IFDIR \| (~current_umask() & S_IRWXUGO));
8045	inode->i_op = &btrfs_dir_inode_operations;
8046	inode->i_fop = &btrfs_dir_file_operations;
8047	}
8048	return inode;
8049	}
8050
8051	struct inode btrfs_alloc_inode(struct super_block sb)
8052	{
8053	struct btrfs_fs_info *fs_info = btrfs_sb(sb);
8054	struct btrfs_inode *ei;
8055	struct inode *inode;
8056
8057	ei = alloc_inode_sb(sb, btrfs_inode_cachep, GFP_KERNEL);
8058	if (!ei)
8059	return NULL;
8060
8061	ei->root = NULL;
8062	ei->generation = 0;
8063	ei->last_trans = 0;
8064	ei->last_sub_trans = 0;
8065	ei->logged_trans = 0;
8066	ei->delalloc_bytes = 0;
8067	/* new_delalloc_bytes and last_dir_index_offset are in a union. */
8068	ei->new_delalloc_bytes = 0;
8069	ei->defrag_bytes = 0;
8070	ei->disk_i_size = 0;
8071	ei->flags = 0;
8072	ei->ro_flags = 0;
8073	/*
8074	* ->index_cnt will be properly initialized later when creating a new
8075	* inode (btrfs_create_new_inode()) or when reading an existing inode
8076	* from disk (btrfs_read_locked_inode()).
8077	*/
8078	ei->csum_bytes = 0;
8079	ei->dir_index = 0;
8080	ei->last_unlink_trans = 0;
8081	ei->last_reflink_trans = 0;
8082	ei->last_log_commit = 0;
8083
8084	spin_lock_init(&ei->lock);
8085	ei->outstanding_extents = 0;
8086	if (sb->s_magic != BTRFS_TEST_MAGIC)
8087	btrfs_init_metadata_block_rsv(fs_info, &ei->block_rsv,
8088	BTRFS_BLOCK_RSV_DELALLOC);
8089	ei->runtime_flags = 0;
8090	ei->prop_compress = BTRFS_COMPRESS_NONE;
8091	ei->defrag_compress = BTRFS_COMPRESS_NONE;
8092
8093	ei->delayed_node = NULL;
8094
8095	ei->i_otime_sec = 0;
8096	ei->i_otime_nsec = 0;
8097
8098	inode = &ei->vfs_inode;
8099	btrfs_extent_map_tree_init(&ei->extent_tree);
8100
8101	/* This io tree sets the valid inode. */
8102	btrfs_extent_io_tree_init(fs_info, &ei->io_tree, IO_TREE_INODE_IO);
8103	ei->io_tree.inode = ei;
8104
8105	ei->file_extent_tree = NULL;
8106
8107	mutex_init(&ei->log_mutex);
8108	spin_lock_init(&ei->ordered_tree_lock);
8109	ei->ordered_tree = RB_ROOT;
8110	ei->ordered_tree_last = NULL;
8111	INIT_LIST_HEAD(&ei->delalloc_inodes);
8112	INIT_LIST_HEAD(&ei->delayed_iput);
8113	init_rwsem(&ei->i_mmap_lock);
8114
8115	return inode;
8116	}
8117
8118	#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
8119	void btrfs_test_destroy_inode(struct inode *inode)
8120	{
8121	btrfs_drop_extent_map_range(BTRFS_I(inode), 0, (u64)-1, false);
8122	kfree(BTRFS_I(inode)->file_extent_tree);
8123	kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
8124	}
8125	#endif
8126
8127	void btrfs_free_inode(struct inode *inode)
8128	{
8129	kfree(BTRFS_I(inode)->file_extent_tree);
8130	kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
8131	}
8132
8133	void btrfs_destroy_inode(struct inode *vfs_inode)
8134	{
8135	struct btrfs_ordered_extent *ordered;
8136	struct btrfs_inode *inode = BTRFS_I(vfs_inode);
8137	struct btrfs_root *root = inode->root;
8138	bool freespace_inode;
8139
8140	WARN_ON(!hlist_empty(&vfs_inode->i_dentry));
8141	WARN_ON(vfs_inode->i_data.nrpages);
8142	WARN_ON(inode->block_rsv.reserved);
8143	WARN_ON(inode->block_rsv.size);
8144	WARN_ON(inode->outstanding_extents);
8145	if (!S_ISDIR(vfs_inode->i_mode)) {
8146	WARN_ON(inode->delalloc_bytes);
8147	WARN_ON(inode->new_delalloc_bytes);
8148	WARN_ON(inode->csum_bytes);
8149	}
8150	if (!root \|\| !btrfs_is_data_reloc_root(root))
8151	WARN_ON(inode->defrag_bytes);
8152
8153	/*
8154	* This can happen where we create an inode, but somebody else also
8155	* created the same inode and we need to destroy the one we already
8156	* created.
8157	*/
8158	if (!root)
8159	return;
8160
8161	/*
8162	* If this is a free space inode do not take the ordered extents lockdep
8163	* map.
8164	*/
8165	freespace_inode = btrfs_is_free_space_inode(inode);
8166
8167	while (1) {
8168	ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1);
8169	if (!ordered)
8170	break;
8171	else {
8172	btrfs_err(root->fs_info,
8173	"found ordered extent %llu %llu on inode cleanup",
8174	ordered->file_offset, ordered->num_bytes);
8175
8176	if (!freespace_inode)
8177	btrfs_lockdep_acquire(root->fs_info, btrfs_ordered_extent);
8178
8179	btrfs_remove_ordered_extent(ordered);
8180	btrfs_put_ordered_extent(ordered);
8181	btrfs_put_ordered_extent(ordered);
8182	}
8183	}
8184	btrfs_qgroup_check_reserved_leak(inode);
8185	btrfs_del_inode_from_root(inode);
8186	btrfs_drop_extent_map_range(inode, 0, (u64)-1, false);
8187	btrfs_inode_clear_file_extent_range(inode, 0, (u64)-1);
8188	btrfs_put_root(inode->root);
8189	}
8190
8191	int btrfs_drop_inode(struct inode *inode)
8192	{
8193	struct btrfs_root *root = BTRFS_I(inode)->root;
8194
8195	if (root == NULL)
8196	return 1;
8197
8198	/* the snap/subvol tree is on deleting */
8199	if (btrfs_root_refs(&root->root_item) == 0)
8200	return 1;
8201	else
8202	return inode_generic_drop(inode);
8203	}
8204
8205	static void init_once(void *foo)
8206	{
8207	struct btrfs_inode *ei = foo;
8208
8209	inode_init_once(&ei->vfs_inode);
8210	}
8211
8212	void __cold btrfs_destroy_cachep(void)
8213	{
8214	/*
8215	* Make sure all delayed rcu free inodes are flushed before we
8216	* destroy cache.
8217	*/
8218	rcu_barrier();
8219	kmem_cache_destroy(btrfs_inode_cachep);
8220	}
8221
8222	int __init btrfs_init_cachep(void)
8223	{
8224	btrfs_inode_cachep = kmem_cache_create("btrfs_inode",
8225	sizeof(struct btrfs_inode), 0,
8226	SLAB_RECLAIM_ACCOUNT \| SLAB_ACCOUNT,
8227	init_once);
8228	if (!btrfs_inode_cachep)
8229	return -ENOMEM;
8230
8231	return 0;
8232	}
8233
8234	static int btrfs_getattr(const struct mnt_idmap *idmap,
8235	const struct path path, struct kstat stat,
8236	u32 request_mask, unsigned int flags)
8237	{
8238	u64 delalloc_bytes;
8239	u64 inode_bytes;
8240	struct inode *inode = d_inode(path->dentry);
8241	u32 blocksize = btrfs_sb(inode->i_sb)->sectorsize;
8242	u32 bi_flags = BTRFS_I(inode)->flags;
8243
8244	stat->result_mask \|= STATX_BTIME;
8245	stat->btime.tv_sec = BTRFS_I(inode)->i_otime_sec;
8246	stat->btime.tv_nsec = BTRFS_I(inode)->i_otime_nsec;
8247	if (bi_flags & BTRFS_INODE_APPEND)
8248	stat->attributes \|= STATX_ATTR_APPEND;
8249	if (bi_flags & BTRFS_INODE_COMPRESS)
8250	stat->attributes \|= STATX_ATTR_COMPRESSED;
8251	if (bi_flags & BTRFS_INODE_IMMUTABLE)
8252	stat->attributes \|= STATX_ATTR_IMMUTABLE;
8253	if (bi_flags & BTRFS_INODE_NODUMP)
8254	stat->attributes \|= STATX_ATTR_NODUMP;
8255
8256	stat->attributes_mask \|= (STATX_ATTR_APPEND \|
8257	STATX_ATTR_COMPRESSED \|
8258	STATX_ATTR_IMMUTABLE \|
8259	STATX_ATTR_NODUMP);
8260
8261	generic_fillattr(idmap, request_mask, inode, stat);
8262	stat->dev = BTRFS_I(inode)->root->anon_dev;
8263
8264	stat->subvol = btrfs_root_id(BTRFS_I(inode)->root);
8265	stat->result_mask \|= STATX_SUBVOL;
8266
8267	spin_lock(&BTRFS_I(inode)->lock);
8268	delalloc_bytes = S_ISREG(inode->i_mode) ?
8269	BTRFS_I(inode)->new_delalloc_bytes : 0;
8270	inode_bytes = inode_get_bytes(inode);
8271	spin_unlock(&BTRFS_I(inode)->lock);
8272	stat->blocks = (ALIGN(inode_bytes, blocksize) +
8273	ALIGN(delalloc_bytes, blocksize)) >> SECTOR_SHIFT;
8274	return 0;
8275	}
8276
8277	static int btrfs_rename_exchange(struct inode *old_dir,
8278	struct dentry *old_dentry,
8279	struct inode *new_dir,
8280	struct dentry *new_dentry)
8281	{
8282	struct btrfs_fs_info *fs_info = inode_to_fs_info(old_dir);
8283	struct btrfs_trans_handle *trans;
8284	unsigned int trans_num_items;
8285	struct btrfs_root *root = BTRFS_I(old_dir)->root;
8286	struct btrfs_root *dest = BTRFS_I(new_dir)->root;
8287	struct inode *new_inode = new_dentry->d_inode;
8288	struct inode *old_inode = old_dentry->d_inode;
8289	struct btrfs_rename_ctx old_rename_ctx;
8290	struct btrfs_rename_ctx new_rename_ctx;
8291	u64 old_ino = btrfs_ino(BTRFS_I(old_inode));
8292	u64 new_ino = btrfs_ino(BTRFS_I(new_inode));
8293	u64 old_idx = 0;
8294	u64 new_idx = 0;
8295	int ret;
8296	int ret2;
8297	bool need_abort = false;
8298	bool logs_pinned = false;
8299	struct fscrypt_name old_fname, new_fname;
8300	struct fscrypt_str old_name, new_name;
8301
8302	/*
8303	* For non-subvolumes allow exchange only within one subvolume, in the
8304	* same inode namespace. Two subvolumes (represented as directory) can
8305	* be exchanged as they're a logical link and have a fixed inode number.
8306	*/
8307	if (root != dest &&
8308	(old_ino != BTRFS_FIRST_FREE_OBJECTID \|\|
8309	new_ino != BTRFS_FIRST_FREE_OBJECTID))
8310	return -EXDEV;
8311
8312	ret = fscrypt_setup_filename(old_dir, &old_dentry->d_name, 0, &old_fname);
8313	if (ret)
8314	return ret;
8315
8316	ret = fscrypt_setup_filename(new_dir, &new_dentry->d_name, 0, &new_fname);
8317	if (ret) {
8318	fscrypt_free_filename(&old_fname);
8319	return ret;
8320	}
8321
8322	old_name = &old_fname.disk_name;
8323	new_name = &new_fname.disk_name;
8324
8325	/* close the race window with snapshot create/destroy ioctl */
8326	if (old_ino == BTRFS_FIRST_FREE_OBJECTID \|\|
8327	new_ino == BTRFS_FIRST_FREE_OBJECTID)
8328	down_read(&fs_info->subvol_sem);
8329
8330	/*
8331	* For each inode:
8332	* 1 to remove old dir item
8333	* 1 to remove old dir index
8334	* 1 to add new dir item
8335	* 1 to add new dir index
8336	* 1 to update parent inode
8337	*
8338	* If the parents are the same, we only need to account for one
8339	*/
8340	trans_num_items = (old_dir == new_dir ? 9 : 10);
8341	if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
8342	/*
8343	* 1 to remove old root ref
8344	* 1 to remove old root backref
8345	* 1 to add new root ref
8346	* 1 to add new root backref
8347	*/
8348	trans_num_items += 4;
8349	} else {
8350	/*
8351	* 1 to update inode item
8352	* 1 to remove old inode ref
8353	* 1 to add new inode ref
8354	*/
8355	trans_num_items += 3;
8356	}
8357	if (new_ino == BTRFS_FIRST_FREE_OBJECTID)
8358	trans_num_items += 4;
8359	else
8360	trans_num_items += 3;
8361	trans = btrfs_start_transaction(root, trans_num_items);
8362	if (IS_ERR(trans)) {
8363	ret = PTR_ERR(trans);
8364	goto out_notrans;
8365	}
8366
8367	if (dest != root) {
8368	ret = btrfs_record_root_in_trans(trans, dest);
8369	if (ret)
8370	goto out_fail;
8371	}
8372
8373	/*
8374	* We need to find a free sequence number both in the source and
8375	* in the destination directory for the exchange.
8376	*/
8377	ret = btrfs_set_inode_index(BTRFS_I(new_dir), &old_idx);
8378	if (ret)
8379	goto out_fail;
8380	ret = btrfs_set_inode_index(BTRFS_I(old_dir), &new_idx);
8381	if (ret)
8382	goto out_fail;
8383
8384	BTRFS_I(old_inode)->dir_index = 0ULL;
8385	BTRFS_I(new_inode)->dir_index = 0ULL;
8386
8387	/* Reference for the source. */
8388	if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
8389	/* force full log commit if subvolume involved. */
8390	btrfs_set_log_full_commit(trans);
8391	} else {
8392	ret = btrfs_insert_inode_ref(trans, dest, new_name, old_ino,
8393	btrfs_ino(BTRFS_I(new_dir)),
8394	old_idx);
8395	if (ret)
8396	goto out_fail;
8397	need_abort = true;
8398	}
8399
8400	/* And now for the dest. */
8401	if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
8402	/* force full log commit if subvolume involved. */
8403	btrfs_set_log_full_commit(trans);
8404	} else {
8405	ret = btrfs_insert_inode_ref(trans, root, old_name, new_ino,
8406	btrfs_ino(BTRFS_I(old_dir)),
8407	new_idx);
8408	if (ret) {
8409	if (unlikely(need_abort))
8410	btrfs_abort_transaction(trans, ret);
8411	goto out_fail;
8412	}
8413	}
8414
8415	/* Update inode version and ctime/mtime. */
8416	inode_inc_iversion(old_dir);
8417	inode_inc_iversion(new_dir);
8418	inode_inc_iversion(old_inode);
8419	inode_inc_iversion(new_inode);
8420	simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry);
8421
8422	if (old_ino != BTRFS_FIRST_FREE_OBJECTID &&
8423	new_ino != BTRFS_FIRST_FREE_OBJECTID) {
8424	/*
8425	* If we are renaming in the same directory (and it's not for
8426	* root entries) pin the log early to prevent any concurrent
8427	* task from logging the directory after we removed the old
8428	* entries and before we add the new entries, otherwise that
8429	* task can sync a log without any entry for the inodes we are
8430	* renaming and therefore replaying that log, if a power failure
8431	* happens after syncing the log, would result in deleting the
8432	* inodes.
8433	*
8434	* If the rename affects two different directories, we want to
8435	* make sure the that there's no log commit that contains
8436	* updates for only one of the directories but not for the
8437	* other.
8438	*
8439	* If we are renaming an entry for a root, we don't care about
8440	* log updates since we called btrfs_set_log_full_commit().
8441	*/
8442	btrfs_pin_log_trans(root);
8443	btrfs_pin_log_trans(dest);
8444	logs_pinned = true;
8445	}
8446
8447	if (old_dentry->d_parent != new_dentry->d_parent) {
8448	btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),
8449	BTRFS_I(old_inode), true);
8450	btrfs_record_unlink_dir(trans, BTRFS_I(new_dir),
8451	BTRFS_I(new_inode), true);
8452	}
8453
8454	/* src is a subvolume */
8455	if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
8456	ret = btrfs_unlink_subvol(trans, BTRFS_I(old_dir), old_dentry);
8457	if (unlikely(ret)) {
8458	btrfs_abort_transaction(trans, ret);
8459	goto out_fail;
8460	}
8461	} else { /* src is an inode */
8462	ret = __btrfs_unlink_inode(trans, BTRFS_I(old_dir),
8463	BTRFS_I(old_dentry->d_inode),
8464	old_name, &old_rename_ctx);
8465	if (unlikely(ret)) {
8466	btrfs_abort_transaction(trans, ret);
8467	goto out_fail;
8468	}
8469	ret = btrfs_update_inode(trans, BTRFS_I(old_inode));
8470	if (unlikely(ret)) {
8471	btrfs_abort_transaction(trans, ret);
8472	goto out_fail;
8473	}
8474	}
8475
8476	/* dest is a subvolume */
8477	if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
8478	ret = btrfs_unlink_subvol(trans, BTRFS_I(new_dir), new_dentry);
8479	if (unlikely(ret)) {
8480	btrfs_abort_transaction(trans, ret);
8481	goto out_fail;
8482	}
8483	} else { /* dest is an inode */
8484	ret = __btrfs_unlink_inode(trans, BTRFS_I(new_dir),
8485	BTRFS_I(new_dentry->d_inode),
8486	new_name, &new_rename_ctx);
8487	if (unlikely(ret)) {
8488	btrfs_abort_transaction(trans, ret);
8489	goto out_fail;
8490	}
8491	ret = btrfs_update_inode(trans, BTRFS_I(new_inode));
8492	if (unlikely(ret)) {
8493	btrfs_abort_transaction(trans, ret);
8494	goto out_fail;
8495	}
8496	}
8497
8498	ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
8499	new_name, false, old_idx, NULL);
8500	if (unlikely(ret)) {
8501	btrfs_abort_transaction(trans, ret);
8502	goto out_fail;
8503	}
8504
8505	ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode),
8506	old_name, false, new_idx, NULL);
8507	if (unlikely(ret)) {
8508	btrfs_abort_transaction(trans, ret);
8509	goto out_fail;
8510	}
8511
8512	if (old_inode->i_nlink == 1)
8513	BTRFS_I(old_inode)->dir_index = old_idx;
8514	if (new_inode->i_nlink == 1)
8515	BTRFS_I(new_inode)->dir_index = new_idx;
8516
8517	/*
8518	* Do the log updates for all inodes.
8519	*
8520	* If either entry is for a root we don't need to update the logs since
8521	* we've called btrfs_set_log_full_commit() before.
8522	*/
8523	if (logs_pinned) {
8524	btrfs_log_new_name(trans, old_dentry, BTRFS_I(old_dir),
8525	old_rename_ctx.index, new_dentry->d_parent);
8526	btrfs_log_new_name(trans, new_dentry, BTRFS_I(new_dir),
8527	new_rename_ctx.index, old_dentry->d_parent);
8528	}
8529
8530	out_fail:
8531	if (logs_pinned) {
8532	btrfs_end_log_trans(root);
8533	btrfs_end_log_trans(dest);
8534	}
8535	ret2 = btrfs_end_transaction(trans);
8536	ret = ret ? ret : ret2;
8537	out_notrans:
8538	if (new_ino == BTRFS_FIRST_FREE_OBJECTID \|\|
8539	old_ino == BTRFS_FIRST_FREE_OBJECTID)
8540	up_read(&fs_info->subvol_sem);
8541
8542	fscrypt_free_filename(&new_fname);
8543	fscrypt_free_filename(&old_fname);
8544	return ret;
8545	}
8546
8547	static struct inode new_whiteout_inode(const struct mnt_idmap idmap,
8548	struct inode *dir)
8549	{
8550	struct inode *inode;
8551
8552	inode = new_inode(dir->i_sb);
8553	if (inode) {
8554	inode_init_owner(idmap, inode, dir,
8555	S_IFCHR \| WHITEOUT_MODE);
8556	inode->i_op = &btrfs_special_inode_operations;
8557	init_special_inode(inode, inode->i_mode, WHITEOUT_DEV);
8558	}
8559	return inode;
8560	}
8561
8562	static int btrfs_rename(const struct mnt_idmap *idmap,
8563	struct inode old_dir, struct dentry old_dentry,
8564	struct inode new_dir, struct dentry new_dentry,
8565	unsigned int flags)
8566	{
8567	struct btrfs_fs_info *fs_info = inode_to_fs_info(old_dir);
8568	struct btrfs_new_inode_args whiteout_args = {
8569	.dir = old_dir,
8570	.dentry = old_dentry,
8571	};
8572	struct btrfs_trans_handle *trans;
8573	unsigned int trans_num_items;
8574	struct btrfs_root *root = BTRFS_I(old_dir)->root;
8575	struct btrfs_root *dest = BTRFS_I(new_dir)->root;
8576	struct inode *new_inode = d_inode(new_dentry);
8577	struct inode *old_inode = d_inode(old_dentry);
8578	struct btrfs_rename_ctx rename_ctx;
8579	struct btrfs_dir_index_prealloc *prealloc = NULL;
8580	u64 index = 0;
8581	int ret;
8582	int ret2;
8583	u64 old_ino = btrfs_ino(BTRFS_I(old_inode));
8584	struct fscrypt_name old_fname, new_fname;
8585	bool logs_pinned = false;
8586
8587	if (btrfs_ino(BTRFS_I(new_dir)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
8588	return -EPERM;
8589
8590	/* we only allow rename subvolume link between subvolumes */
8591	if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest)
8592	return -EXDEV;
8593
8594	if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID \|\|
8595	(new_inode && btrfs_ino(BTRFS_I(new_inode)) == BTRFS_FIRST_FREE_OBJECTID))
8596	return -ENOTEMPTY;
8597
8598	if (S_ISDIR(old_inode->i_mode) && new_inode &&
8599	new_inode->i_size > BTRFS_EMPTY_DIR_SIZE)
8600	return -ENOTEMPTY;
8601
8602	ret = fscrypt_setup_filename(old_dir, &old_dentry->d_name, 0, &old_fname);
8603	if (ret)
8604	return ret;
8605
8606	ret = fscrypt_setup_filename(new_dir, &new_dentry->d_name, 0, &new_fname);
8607	if (ret) {
8608	fscrypt_free_filename(&old_fname);
8609	return ret;
8610	}
8611
8612	/* check for collisions, even if the name isn't there */
8613	ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino, &new_fname.disk_name);
8614	if (ret) {
8615	if (ret == -EEXIST) {
8616	/* we shouldn't get
8617	* eexist without a new_inode */
8618	if (WARN_ON(!new_inode)) {
8619	goto out_fscrypt_names;
8620	}
8621	} else {
8622	/* maybe -EOVERFLOW */
8623	goto out_fscrypt_names;
8624	}
8625	}
8626	ret = 0;
8627
8628	/*
8629	* we're using rename to replace one file with another. Start IO on it
8630	* now so we don't add too much work to the end of the transaction
8631	*/
8632	if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size)
8633	filemap_flush(old_inode->i_mapping);
8634
8635	if (flags & RENAME_WHITEOUT) {
8636	whiteout_args.inode = new_whiteout_inode(idmap, old_dir);
8637	if (!whiteout_args.inode) {
8638	ret = -ENOMEM;
8639	goto out_fscrypt_names;
8640	}
8641	ret = btrfs_new_inode_prepare(&whiteout_args, &trans_num_items);
8642	if (ret)
8643	goto out_whiteout_inode;
8644	} else {
8645	/* 1 to update the old parent inode. */
8646	trans_num_items = 1;
8647	}
8648
8649	if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
8650	/* Close the race window with snapshot create/destroy ioctl */
8651	down_read(&fs_info->subvol_sem);
8652	/*
8653	* 1 to remove old root ref
8654	* 1 to remove old root backref
8655	* 1 to add new root ref
8656	* 1 to add new root backref
8657	*/
8658	trans_num_items += 4;
8659	} else {
8660	/*
8661	* 1 to update inode
8662	* 1 to remove old inode ref
8663	* 1 to add new inode ref
8664	*/
8665	trans_num_items += 3;
8666	}
8667	/*
8668	* 1 to remove old dir item
8669	* 1 to remove old dir index
8670	* 1 to add new dir item
8671	* 1 to add new dir index
8672	*/
8673	trans_num_items += 4;
8674	/* 1 to update new parent inode if it's not the same as the old parent */
8675	if (new_dir != old_dir)
8676	trans_num_items++;
8677	if (new_inode) {
8678	/*
8679	* 1 to update inode
8680	* 1 to remove inode ref
8681	* 1 to remove dir item
8682	* 1 to remove dir index
8683	* 1 to possibly add orphan item
8684	*/
8685	trans_num_items += 5;
8686	}
8687	trans = btrfs_start_transaction(root, trans_num_items);
8688	if (IS_ERR(trans)) {
8689	ret = PTR_ERR(trans);
8690	goto out_notrans;
8691	}
8692
8693	if (dest != root) {
8694	ret = btrfs_record_root_in_trans(trans, dest);
8695	if (ret)
8696	goto out_fail;
8697	}
8698
8699	ret = btrfs_set_inode_index(BTRFS_I(new_dir), &index);
8700	if (ret)
8701	goto out_fail;
8702
8703	/*
8704	* When not overwriting an existing entry, pre-allocate the delayed dir
8705	* index now so that ENOMEM is returned before any btree modifications.
8706	* For the overwrite case, too many btree changes have already happened
8707	* by the time btrfs_add_link() is called.
8708	*/
8709	if (!new_inode) {
8710	prealloc = btrfs_prealloc_delayed_dir_index(BTRFS_I(new_dir),
8711	new_fname.disk_name.name,
8712	new_fname.disk_name.len);
8713	if (IS_ERR(prealloc)) {
8714	ret = PTR_ERR(prealloc);
8715	prealloc = NULL;
8716	goto out_fail;
8717	}
8718	}
8719
8720	BTRFS_I(old_inode)->dir_index = 0ULL;
8721	if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
8722	/* force full log commit if subvolume involved. */
8723	btrfs_set_log_full_commit(trans);
8724	} else {
8725	ret = btrfs_insert_inode_ref(trans, dest, &new_fname.disk_name,
8726	old_ino, btrfs_ino(BTRFS_I(new_dir)),
8727	index);
8728	if (ret)
8729	goto out_fail;
8730	}
8731
8732	inode_inc_iversion(old_dir);
8733	inode_inc_iversion(new_dir);
8734	inode_inc_iversion(old_inode);
8735	simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry);
8736
8737	if (old_ino != BTRFS_FIRST_FREE_OBJECTID) {
8738	/*
8739	* If we are renaming in the same directory (and it's not a
8740	* root entry) pin the log to prevent any concurrent task from
8741	* logging the directory after we removed the old entry and
8742	* before we add the new entry, otherwise that task can sync
8743	* a log without any entry for the inode we are renaming and
8744	* therefore replaying that log, if a power failure happens
8745	* after syncing the log, would result in deleting the inode.
8746	*
8747	* If the rename affects two different directories, we want to
8748	* make sure the that there's no log commit that contains
8749	* updates for only one of the directories but not for the
8750	* other.
8751	*
8752	* If we are renaming an entry for a root, we don't care about
8753	* log updates since we called btrfs_set_log_full_commit().
8754	*/
8755	btrfs_pin_log_trans(root);
8756	btrfs_pin_log_trans(dest);
8757	logs_pinned = true;
8758	}
8759
8760	if (old_dentry->d_parent != new_dentry->d_parent)
8761	btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),
8762	BTRFS_I(old_inode), true);
8763
8764	if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
8765	ret = btrfs_unlink_subvol(trans, BTRFS_I(old_dir), old_dentry);
8766	if (unlikely(ret)) {
8767	btrfs_abort_transaction(trans, ret);
8768	goto out_fail;
8769	}
8770	} else {
8771	ret = __btrfs_unlink_inode(trans, BTRFS_I(old_dir),
8772	BTRFS_I(d_inode(old_dentry)),
8773	&old_fname.disk_name, &rename_ctx);
8774	if (unlikely(ret)) {
8775	btrfs_abort_transaction(trans, ret);
8776	goto out_fail;
8777	}
8778	ret = btrfs_update_inode(trans, BTRFS_I(old_inode));
8779	if (unlikely(ret)) {
8780	btrfs_abort_transaction(trans, ret);
8781	goto out_fail;
8782	}
8783	}
8784
8785	if (new_inode) {
8786	inode_inc_iversion(new_inode);
8787	if (unlikely(btrfs_ino(BTRFS_I(new_inode)) ==
8788	BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
8789	ret = btrfs_unlink_subvol(trans, BTRFS_I(new_dir), new_dentry);
8790	if (unlikely(ret)) {
8791	btrfs_abort_transaction(trans, ret);
8792	goto out_fail;
8793	}
8794	BUG_ON(new_inode->i_nlink == 0);
8795	} else {
8796	ret = btrfs_unlink_inode(trans, BTRFS_I(new_dir),
8797	BTRFS_I(d_inode(new_dentry)),
8798	&new_fname.disk_name);
8799	if (unlikely(ret)) {
8800	btrfs_abort_transaction(trans, ret);
8801	goto out_fail;
8802	}
8803	}
8804	if (new_inode->i_nlink == 0) {
8805	ret = btrfs_orphan_add(trans,
8806	BTRFS_I(d_inode(new_dentry)));
8807	if (unlikely(ret)) {
8808	btrfs_abort_transaction(trans, ret);
8809	goto out_fail;
8810	}
8811	}
8812	}
8813
8814	ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
8815	&new_fname.disk_name, false, index, prealloc);
8816	prealloc = NULL;
8817	if (unlikely(ret)) {
8818	btrfs_abort_transaction(trans, ret);
8819	goto out_fail;
8820	}
8821
8822	if (old_inode->i_nlink == 1)
8823	BTRFS_I(old_inode)->dir_index = index;
8824
8825	if (logs_pinned)
8826	btrfs_log_new_name(trans, old_dentry, BTRFS_I(old_dir),
8827	rename_ctx.index, new_dentry->d_parent);
8828
8829	if (flags & RENAME_WHITEOUT) {
8830	ret = btrfs_create_new_inode(trans, &whiteout_args);
8831	if (unlikely(ret)) {
8832	btrfs_abort_transaction(trans, ret);
8833	goto out_fail;
8834	} else {
8835	unlock_new_inode(whiteout_args.inode);
8836	iput(whiteout_args.inode);
8837	whiteout_args.inode = NULL;
8838	}
8839	}
8840	out_fail:
8841	btrfs_free_delayed_dir_index_prealloc(trans, prealloc);
8842	if (logs_pinned) {
8843	btrfs_end_log_trans(root);
8844	btrfs_end_log_trans(dest);
8845	}
8846	ret2 = btrfs_end_transaction(trans);
8847	ret = ret ? ret : ret2;
8848	out_notrans:
8849	if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
8850	up_read(&fs_info->subvol_sem);
8851	if (flags & RENAME_WHITEOUT)
8852	btrfs_new_inode_args_destroy(&whiteout_args);
8853	out_whiteout_inode:
8854	if (flags & RENAME_WHITEOUT)
8855	iput(whiteout_args.inode);
8856	out_fscrypt_names:
8857	fscrypt_free_filename(&old_fname);
8858	fscrypt_free_filename(&new_fname);
8859	return ret;
8860	}
8861
8862	static int btrfs_rename2(const struct mnt_idmap idmap, struct inode old_dir,
8863	struct dentry old_dentry, struct inode new_dir,
8864	struct dentry *new_dentry, unsigned int flags)
8865	{
8866	int ret;
8867
8868	if (flags & ~(RENAME_NOREPLACE \| RENAME_EXCHANGE \| RENAME_WHITEOUT))
8869	return -EINVAL;
8870
8871	if (flags & RENAME_EXCHANGE)
8872	ret = btrfs_rename_exchange(old_dir, old_dentry, new_dir,
8873	new_dentry);
8874	else
8875	ret = btrfs_rename(idmap, old_dir, old_dentry, new_dir,
8876	new_dentry, flags);
8877
8878	btrfs_btree_balance_dirty(BTRFS_I(new_dir)->root->fs_info);
8879
8880	return ret;
8881	}
8882
8883	struct btrfs_delalloc_work {
8884	struct inode *inode;
8885	struct completion completion;
8886	struct list_head list;
8887	struct btrfs_work work;
8888	};
8889
8890	static void btrfs_run_delalloc_work(struct btrfs_work *work)
8891	{
8892	struct btrfs_delalloc_work *delalloc_work;
8893	struct inode *inode;
8894
8895	delalloc_work = container_of(work, struct btrfs_delalloc_work,
8896	work);
8897	inode = delalloc_work->inode;
8898	filemap_flush(inode->i_mapping);
8899	if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
8900	&BTRFS_I(inode)->runtime_flags))
8901	filemap_flush(inode->i_mapping);
8902
8903	iput(inode);
8904	complete(&delalloc_work->completion);
8905	}
8906
8907	static struct btrfs_delalloc_work btrfs_alloc_delalloc_work(struct inode inode)
8908	{
8909	struct btrfs_delalloc_work *work;
8910
8911	work = kmalloc_obj(*work, GFP_NOFS);
8912	if (!work)
8913	return NULL;
8914
8915	init_completion(&work->completion);
8916	INIT_LIST_HEAD(&work->list);
8917	work->inode = inode;
8918	btrfs_init_work(&work->work, btrfs_run_delalloc_work, NULL);
8919
8920	return work;
8921	}
8922
8923	/*
8924	* some fairly slow code that needs optimization. This walks the list
8925	* of all the inodes with pending delalloc and forces them to disk.
8926	*/
8927	static int start_delalloc_inodes(struct btrfs_root root, long nr_to_write,
8928	bool snapshot, bool in_reclaim_context)
8929	{
8930	struct btrfs_delalloc_work work, next;
8931	LIST_HEAD(works);
8932	LIST_HEAD(splice);
8933	int ret = 0;
8934
8935	mutex_lock(&root->delalloc_mutex);
8936	spin_lock(&root->delalloc_lock);
8937	list_splice_init(&root->delalloc_inodes, &splice);
8938	while (!list_empty(&splice)) {
8939	struct btrfs_inode *inode;
8940	struct inode *tmp_inode;
8941
8942	inode = list_first_entry(&splice, struct btrfs_inode, delalloc_inodes);
8943
8944	list_move_tail(&inode->delalloc_inodes, &root->delalloc_inodes);
8945
8946	if (in_reclaim_context &&
8947	test_bit(BTRFS_INODE_NO_DELALLOC_FLUSH, &inode->runtime_flags))
8948	continue;
8949
8950	tmp_inode = igrab(&inode->vfs_inode);
8951	if (!tmp_inode) {
8952	cond_resched_lock(&root->delalloc_lock);
8953	continue;
8954	}
8955	spin_unlock(&root->delalloc_lock);
8956
8957	if (snapshot)
8958	set_bit(BTRFS_INODE_SNAPSHOT_FLUSH, &inode->runtime_flags);
8959	if (nr_to_write == NULL) {
8960	work = btrfs_alloc_delalloc_work(tmp_inode);
8961	if (!work) {
8962	iput(tmp_inode);
8963	ret = -ENOMEM;
8964	goto out;
8965	}
8966	list_add_tail(&work->list, &works);
8967	btrfs_queue_work(root->fs_info->flush_workers,
8968	&work->work);
8969	} else {
8970	ret = filemap_flush_nr(tmp_inode->i_mapping,
8971	nr_to_write);
8972	btrfs_add_delayed_iput(inode);
8973
8974	if (ret \|\| *nr_to_write <= 0)
8975	goto out;
8976	}
8977	cond_resched();
8978	spin_lock(&root->delalloc_lock);
8979	}
8980	spin_unlock(&root->delalloc_lock);
8981
8982	out:
8983	list_for_each_entry_safe(work, next, &works, list) {
8984	list_del_init(&work->list);
8985	wait_for_completion(&work->completion);
8986	kfree(work);
8987	}
8988
8989	if (!list_empty(&splice)) {
8990	spin_lock(&root->delalloc_lock);
8991	list_splice_tail(&splice, &root->delalloc_inodes);
8992	spin_unlock(&root->delalloc_lock);
8993	}
8994	mutex_unlock(&root->delalloc_mutex);
8995	return ret;
8996	}
8997
8998	int btrfs_start_delalloc_snapshot(struct btrfs_root *root, bool in_reclaim_context)
8999	{
9000	struct btrfs_fs_info *fs_info = root->fs_info;
9001
9002	if (unlikely(BTRFS_FS_ERROR(fs_info)))
9003	return -EROFS;
9004	return start_delalloc_inodes(root, NULL, true, in_reclaim_context);
9005	}
9006
9007	int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, long nr,
9008	bool in_reclaim_context)
9009	{
9010	long *nr_to_write = nr == LONG_MAX ? NULL : &nr;
9011	struct btrfs_root *root;
9012	LIST_HEAD(splice);
9013	int ret;
9014
9015	if (unlikely(BTRFS_FS_ERROR(fs_info)))
9016	return -EROFS;
9017
9018	mutex_lock(&fs_info->delalloc_root_mutex);
9019	spin_lock(&fs_info->delalloc_root_lock);
9020	list_splice_init(&fs_info->delalloc_roots, &splice);
9021	while (!list_empty(&splice)) {
9022	root = list_first_entry(&splice, struct btrfs_root,
9023	delalloc_root);
9024	root = btrfs_grab_root(root);
9025	BUG_ON(!root);
9026	list_move_tail(&root->delalloc_root,
9027	&fs_info->delalloc_roots);
9028	spin_unlock(&fs_info->delalloc_root_lock);
9029
9030	ret = start_delalloc_inodes(root, nr_to_write, false,
9031	in_reclaim_context);
9032	btrfs_put_root(root);
9033	if (ret < 0 \|\| nr <= 0)
9034	goto out;
9035	spin_lock(&fs_info->delalloc_root_lock);
9036	}
9037	spin_unlock(&fs_info->delalloc_root_lock);
9038
9039	ret = 0;
9040	out:
9041	if (!list_empty(&splice)) {
9042	spin_lock(&fs_info->delalloc_root_lock);
9043	list_splice_tail(&splice, &fs_info->delalloc_roots);
9044	spin_unlock(&fs_info->delalloc_root_lock);
9045	}
9046	mutex_unlock(&fs_info->delalloc_root_mutex);
9047	return ret;
9048	}
9049
9050	static int btrfs_symlink(const struct mnt_idmap idmap, struct inode dir,
9051	struct dentry dentry, const char symname)
9052	{
9053	struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);
9054	struct btrfs_trans_handle *trans;
9055	struct btrfs_root *root = BTRFS_I(dir)->root;
9056	struct btrfs_path *path;
9057	struct btrfs_key key;
9058	struct inode *inode;
9059	struct btrfs_new_inode_args new_inode_args = {
9060	.dir = dir,
9061	.dentry = dentry,
9062	};
9063	unsigned int trans_num_items;
9064	int ret;
9065	int name_len;
9066	int datasize;
9067	unsigned long ptr;
9068	struct btrfs_file_extent_item *ei;
9069	struct extent_buffer *leaf;
9070
9071	name_len = strlen(symname);
9072	/*
9073	* Symlinks utilize uncompressed inline extent data, which should not
9074	* reach block size.
9075	*/
9076	if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info) \|\|
9077	name_len >= fs_info->sectorsize)
9078	return -ENAMETOOLONG;
9079
9080	inode = new_inode(dir->i_sb);
9081	if (!inode)
9082	return -ENOMEM;
9083	inode_init_owner(idmap, inode, dir, S_IFLNK \| S_IRWXUGO);
9084	inode->i_op = &btrfs_symlink_inode_operations;
9085	inode_nohighmem(inode);
9086	inode->i_mapping->a_ops = &btrfs_aops;
9087	btrfs_i_size_write(BTRFS_I(inode), name_len);
9088	inode_set_bytes(inode, name_len);
9089
9090	new_inode_args.inode = inode;
9091	ret = btrfs_new_inode_prepare(&new_inode_args, &trans_num_items);
9092	if (ret)
9093	goto out_inode;
9094	/* 1 additional item for the inline extent */
9095	trans_num_items++;
9096
9097	trans = btrfs_start_transaction(root, trans_num_items);
9098	if (IS_ERR(trans)) {
9099	ret = PTR_ERR(trans);
9100	goto out_new_inode_args;
9101	}
9102
9103	ret = btrfs_create_new_inode(trans, &new_inode_args);
9104	if (ret)
9105	goto out;
9106
9107	path = btrfs_alloc_path();
9108	if (unlikely(!path)) {
9109	ret = -ENOMEM;
9110	btrfs_abort_transaction(trans, ret);
9111	discard_new_inode(inode);
9112	inode = NULL;
9113	goto out;
9114	}
9115	key.objectid = btrfs_ino(BTRFS_I(inode));
9116	key.type = BTRFS_EXTENT_DATA_KEY;
9117	key.offset = 0;
9118	datasize = btrfs_file_extent_calc_inline_size(name_len);
9119	ret = btrfs_insert_empty_item(trans, root, path, &key, datasize);
9120	if (unlikely(ret)) {
9121	btrfs_abort_transaction(trans, ret);
9122	btrfs_free_path(path);
9123	discard_new_inode(inode);
9124	inode = NULL;
9125	goto out;
9126	}
9127	leaf = path->nodes[0];
9128	ei = btrfs_item_ptr(leaf, path->slots[0],
9129	struct btrfs_file_extent_item);
9130	btrfs_set_file_extent_generation(leaf, ei, trans->transid);
9131	btrfs_set_file_extent_type(leaf, ei,
9132	BTRFS_FILE_EXTENT_INLINE);
9133	btrfs_set_file_extent_encryption(leaf, ei, 0);
9134	btrfs_set_file_extent_compression(leaf, ei, 0);
9135	btrfs_set_file_extent_other_encoding(leaf, ei, 0);
9136	btrfs_set_file_extent_ram_bytes(leaf, ei, name_len);
9137
9138	ptr = btrfs_file_extent_inline_start(ei);
9139	write_extent_buffer(leaf, symname, ptr, name_len);
9140	btrfs_free_path(path);
9141
9142	d_instantiate_new(dentry, inode);
9143	ret = 0;
9144	out:
9145	btrfs_end_transaction(trans);
9146	btrfs_btree_balance_dirty(fs_info);
9147	out_new_inode_args:
9148	btrfs_new_inode_args_destroy(&new_inode_args);
9149	out_inode:
9150	if (ret)
9151	iput(inode);
9152	return ret;
9153	}
9154
9155	static struct btrfs_trans_handle *insert_prealloc_file_extent(
9156	struct btrfs_trans_handle *trans_in,
9157	struct btrfs_inode *inode,
9158	struct btrfs_key *ins,
9159	u64 file_offset)
9160	{
9161	struct btrfs_file_extent_item stack_fi;
9162	struct btrfs_replace_extent_info extent_info;
9163	struct btrfs_trans_handle *trans = trans_in;
9164	struct btrfs_path *path;
9165	u64 start = ins->objectid;
9166	u64 len = ins->offset;
9167	u64 qgroup_released = 0;
9168	int ret;
9169
9170	memset(&stack_fi, 0, sizeof(stack_fi));
9171
9172	btrfs_set_stack_file_extent_type(&stack_fi, BTRFS_FILE_EXTENT_PREALLOC);
9173	btrfs_set_stack_file_extent_disk_bytenr(&stack_fi, start);
9174	btrfs_set_stack_file_extent_disk_num_bytes(&stack_fi, len);
9175	btrfs_set_stack_file_extent_num_bytes(&stack_fi, len);
9176	btrfs_set_stack_file_extent_ram_bytes(&stack_fi, len);
9177	btrfs_set_stack_file_extent_compression(&stack_fi, BTRFS_COMPRESS_NONE);
9178	/* Encryption and other encoding is reserved and all 0 */
9179
9180	ret = btrfs_qgroup_release_data(inode, file_offset, len, &qgroup_released);
9181	if (ret < 0)
9182	return ERR_PTR(ret);
9183
9184	if (trans) {
9185	ret = insert_reserved_file_extent(trans, inode,
9186	file_offset, &stack_fi,
9187	true, qgroup_released);
9188	if (ret)
9189	goto free_qgroup;
9190	return trans;
9191	}
9192
9193	extent_info.disk_offset = start;
9194	extent_info.disk_len = len;
9195	extent_info.data_offset = 0;
9196	extent_info.data_len = len;
9197	extent_info.file_offset = file_offset;
9198	extent_info.extent_buf = (char *)&stack_fi;
9199	extent_info.is_new_extent = true;
9200	extent_info.update_times = true;
9201	extent_info.qgroup_reserved = qgroup_released;
9202	extent_info.insertions = 0;
9203
9204	path = btrfs_alloc_path();
9205	if (!path) {
9206	ret = -ENOMEM;
9207	goto free_qgroup;
9208	}
9209
9210	ret = btrfs_replace_file_extents(inode, path, file_offset,
9211	file_offset + len - 1, &extent_info,
9212	&trans);
9213	btrfs_free_path(path);
9214	if (ret)
9215	goto free_qgroup;
9216	return trans;
9217
9218	free_qgroup:
9219	/*
9220	* We have released qgroup data range at the beginning of the function,
9221	* and normally qgroup_released bytes will be freed when committing
9222	* transaction.
9223	* But if we error out early, we have to free what we have released
9224	* or we leak qgroup data reservation.
9225	*/
9226	btrfs_qgroup_free_refroot(inode->root->fs_info,
9227	btrfs_root_id(inode->root), qgroup_released,
9228	BTRFS_QGROUP_RSV_DATA);
9229	return ERR_PTR(ret);
9230	}
9231
9232	static int __btrfs_prealloc_file_range(struct inode *inode, int mode,
9233	u64 start, u64 num_bytes, u64 min_size,
9234	loff_t actual_len, u64 *alloc_hint,
9235	struct btrfs_trans_handle *trans)
9236	{
9237	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
9238	struct extent_map *em;
9239	struct btrfs_root *root = BTRFS_I(inode)->root;
9240	struct btrfs_key ins;
9241	u64 cur_offset = start;
9242	u64 clear_offset = start;
9243	u64 i_size;
9244	u64 cur_bytes;
9245	u64 last_alloc = (u64)-1;
9246	int ret = 0;
9247	bool own_trans = true;
9248	u64 end = start + num_bytes - 1;
9249
9250	if (trans)
9251	own_trans = false;
9252	while (num_bytes > 0) {
9253	cur_bytes = min_t(u64, num_bytes, SZ_256M);
9254	cur_bytes = max(cur_bytes, min_size);
9255	/*
9256	* If we are severely fragmented we could end up with really
9257	* small allocations, so if the allocator is returning small
9258	* chunks lets make its job easier by only searching for those
9259	* sized chunks.
9260	*/
9261	cur_bytes = min(cur_bytes, last_alloc);
9262	ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes,
9263	min_size, 0, *alloc_hint, &ins, true, false);
9264	if (ret)
9265	break;
9266
9267	/*
9268	* We've reserved this space, and thus converted it from
9269	* ->bytes_may_use to ->bytes_reserved. Any error that happens
9270	* from here on out we will only need to clear our reservation
9271	* for the remaining unreserved area, so advance our
9272	* clear_offset by our extent size.
9273	*/
9274	clear_offset += ins.offset;
9275
9276	last_alloc = ins.offset;
9277	trans = insert_prealloc_file_extent(trans, BTRFS_I(inode),
9278	&ins, cur_offset);
9279	/*
9280	* Now that we inserted the prealloc extent we can finally
9281	* decrement the number of reservations in the block group.
9282	* If we did it before, we could race with relocation and have
9283	* relocation miss the reserved extent, making it fail later.
9284	*/
9285	btrfs_dec_block_group_reservations(fs_info, ins.objectid);
9286	if (IS_ERR(trans)) {
9287	ret = PTR_ERR(trans);
9288	btrfs_free_reserved_extent(fs_info, ins.objectid,
9289	ins.offset, false);
9290	break;
9291	}
9292
9293	em = btrfs_alloc_extent_map();
9294	if (!em) {
9295	btrfs_drop_extent_map_range(BTRFS_I(inode), cur_offset,
9296	cur_offset + ins.offset - 1, false);
9297	btrfs_set_inode_full_sync(BTRFS_I(inode));
9298	goto next;
9299	}
9300
9301	em->start = cur_offset;
9302	em->len = ins.offset;
9303	em->disk_bytenr = ins.objectid;
9304	em->offset = 0;
9305	em->disk_num_bytes = ins.offset;
9306	em->ram_bytes = ins.offset;
9307	em->flags \|= EXTENT_FLAG_PREALLOC;
9308	em->generation = trans->transid;
9309
9310	ret = btrfs_replace_extent_map_range(BTRFS_I(inode), em, true);
9311	btrfs_free_extent_map(em);
9312	next:
9313	num_bytes -= ins.offset;
9314	cur_offset += ins.offset;
9315	*alloc_hint = ins.objectid + ins.offset;
9316
9317	inode_inc_iversion(inode);
9318	inode_set_ctime_current(inode);
9319	BTRFS_I(inode)->flags \|= BTRFS_INODE_PREALLOC;
9320	if (!(mode & FALLOC_FL_KEEP_SIZE) &&
9321	(actual_len > inode->i_size) &&
9322	(cur_offset > inode->i_size)) {
9323	u64 range_start;
9324	u64 range_end;
9325
9326	if (cur_offset > actual_len)
9327	i_size = actual_len;
9328	else
9329	i_size = cur_offset;
9330
9331	/*
9332	* Make sure the file_extent_tree covers the entire
9333	* range [old_i_size, new_i_size) before we update
9334	* disk_i_size. Without this, a previous KEEP_SIZE
9335	* prealloc that extended past i_size (and was lost
9336	* across umount/mount because file_extent_tree is
9337	* only populated up to round_up(i_size) on inode
9338	* load) can leave a gap inside this range. That gap
9339	* would cause btrfs_inode_safe_disk_i_size_write()
9340	* (via find_contiguous_extent_bit() starting at 0)
9341	* to truncate disk_i_size to the start of the gap,
9342	* making the persisted size smaller than i_size.
9343	*/
9344	range_start = round_down(inode->i_size, fs_info->sectorsize);
9345	range_end = round_up(i_size, fs_info->sectorsize);
9346	ret = btrfs_inode_set_file_extent_range(BTRFS_I(inode),
9347	range_start, range_end - range_start);
9348	if (ret) {
9349	btrfs_abort_transaction(trans, ret);
9350	if (own_trans)
9351	btrfs_end_transaction(trans);
9352	break;
9353	}
9354
9355	i_size_write(inode, i_size);
9356	btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0);
9357	}
9358
9359	ret = btrfs_update_inode(trans, BTRFS_I(inode));
9360
9361	if (unlikely(ret)) {
9362	btrfs_abort_transaction(trans, ret);
9363	if (own_trans)
9364	btrfs_end_transaction(trans);
9365	break;
9366	}
9367
9368	if (own_trans) {
9369	btrfs_end_transaction(trans);
9370	trans = NULL;
9371	}
9372	}
9373	if (clear_offset < end)
9374	btrfs_free_reserved_data_space(BTRFS_I(inode), NULL, clear_offset,
9375	end - clear_offset + 1);
9376	return ret;
9377	}
9378
9379	int btrfs_prealloc_file_range(struct inode *inode, int mode,
9380	u64 start, u64 num_bytes, u64 min_size,
9381	loff_t actual_len, u64 *alloc_hint)
9382	{
9383	return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
9384	min_size, actual_len, alloc_hint,
9385	NULL);
9386	}
9387
9388	int btrfs_prealloc_file_range_trans(struct inode *inode,
9389	struct btrfs_trans_handle *trans, int mode,
9390	u64 start, u64 num_bytes, u64 min_size,
9391	loff_t actual_len, u64 *alloc_hint)
9392	{
9393	return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
9394	min_size, actual_len, alloc_hint, trans);
9395	}
9396
9397	/*
9398	* NOTE: in case you are adding MAY_EXEC check for directories:
9399	* we are marking them with IOP_FASTPERM_MAY_EXEC, allowing path lookup to
9400	* elide calls here.
9401	*/
9402	static int btrfs_permission(const struct mnt_idmap *idmap,
9403	struct inode *inode, int mask)
9404	{
9405	struct btrfs_root *root = BTRFS_I(inode)->root;
9406	umode_t mode = inode->i_mode;
9407
9408	if (mask & MAY_WRITE &&
9409	(S_ISREG(mode) \|\| S_ISDIR(mode) \|\| S_ISLNK(mode))) {
9410	if (btrfs_root_readonly(root))
9411	return -EROFS;
9412	if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY)
9413	return -EACCES;
9414	}
9415	return generic_permission(idmap, inode, mask);
9416	}
9417
9418	static int btrfs_tmpfile(const struct mnt_idmap idmap, struct inode dir,
9419	struct file *file, umode_t mode)
9420	{
9421	struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);
9422	struct btrfs_trans_handle *trans;
9423	struct btrfs_root *root = BTRFS_I(dir)->root;
9424	struct inode *inode;
9425	struct btrfs_new_inode_args new_inode_args = {
9426	.dir = dir,
9427	.dentry = file->f_path.dentry,
9428	.orphan = true,
9429	};
9430	unsigned int trans_num_items;
9431	int ret;
9432
9433	inode = new_inode(dir->i_sb);
9434	if (!inode)
9435	return -ENOMEM;
9436	inode_init_owner(idmap, inode, dir, mode);
9437	inode->i_fop = &btrfs_file_operations;
9438	inode->i_op = &btrfs_file_inode_operations;
9439	inode->i_mapping->a_ops = &btrfs_aops;
9440
9441	new_inode_args.inode = inode;
9442	ret = btrfs_new_inode_prepare(&new_inode_args, &trans_num_items);
9443	if (ret)
9444	goto out_inode;
9445
9446	trans = btrfs_start_transaction(root, trans_num_items);
9447	if (IS_ERR(trans)) {
9448	ret = PTR_ERR(trans);
9449	goto out_new_inode_args;
9450	}
9451
9452	ret = btrfs_create_new_inode(trans, &new_inode_args);
9453
9454	/*
9455	* We set number of links to 0 in btrfs_create_new_inode(), and here we
9456	* set it to 1 because d_tmpfile() will issue a warning if the count is
9457	* 0, through:
9458	*
9459	* d_tmpfile() -> inode_dec_link_count() -> drop_nlink()
9460	*/
9461	set_nlink(inode, 1);
9462
9463	if (!ret) {
9464	d_tmpfile(file, inode);
9465	unlock_new_inode(inode);
9466	mark_inode_dirty(inode);
9467	}
9468
9469	btrfs_end_transaction(trans);
9470	btrfs_btree_balance_dirty(fs_info);
9471	out_new_inode_args:
9472	btrfs_new_inode_args_destroy(&new_inode_args);
9473	out_inode:
9474	if (ret)
9475	iput(inode);
9476	return finish_open_simple(file, ret);
9477	}
9478
9479	int btrfs_encoded_io_compression_from_extent(struct btrfs_fs_info *fs_info,
9480	int compress_type)
9481	{
9482	switch (compress_type) {
9483	case BTRFS_COMPRESS_NONE:
9484	return BTRFS_ENCODED_IO_COMPRESSION_NONE;
9485	case BTRFS_COMPRESS_ZLIB:
9486	return BTRFS_ENCODED_IO_COMPRESSION_ZLIB;
9487	case BTRFS_COMPRESS_LZO:
9488	/*
9489	* The LZO format depends on the sector size. 64K is the maximum
9490	* sector size that we support.
9491	*/
9492	if (fs_info->sectorsize < SZ_4K \|\| fs_info->sectorsize > SZ_64K)
9493	return -EINVAL;
9494	return BTRFS_ENCODED_IO_COMPRESSION_LZO_4K +
9495	(fs_info->sectorsize_bits - 12);
9496	case BTRFS_COMPRESS_ZSTD:
9497	return BTRFS_ENCODED_IO_COMPRESSION_ZSTD;
9498	default:
9499	return -EUCLEAN;
9500	}
9501	}
9502
9503	static ssize_t btrfs_encoded_read_inline(
9504	struct kiocb *iocb,
9505	struct iov_iter *iter, u64 start,
9506	u64 lockend,
9507	struct extent_state **cached_state,
9508	u64 extent_start, size_t count,
9509	struct btrfs_ioctl_encoded_io_args *encoded,
9510	bool *unlocked)
9511	{
9512	struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp));
9513	struct btrfs_root *root = inode->root;
9514	struct btrfs_fs_info *fs_info = root->fs_info;
9515	struct extent_io_tree *io_tree = &inode->io_tree;
9516	BTRFS_PATH_AUTO_FREE(path);
9517	struct extent_buffer *leaf;
9518	struct btrfs_file_extent_item *item;
9519	u64 ram_bytes;
9520	unsigned long ptr;
9521	void *tmp;
9522	ssize_t ret;
9523	const bool nowait = (iocb->ki_flags & IOCB_NOWAIT);
9524
9525	path = btrfs_alloc_path();
9526	if (!path)
9527	return -ENOMEM;
9528
9529	path->nowait = nowait;
9530
9531	ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode),
9532	extent_start, 0);
9533	if (ret) {
9534	if (unlikely(ret > 0)) {
9535	/* The extent item disappeared? */
9536	return -EIO;
9537	}
9538	return ret;
9539	}
9540	leaf = path->nodes[0];
9541	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
9542
9543	ram_bytes = btrfs_file_extent_ram_bytes(leaf, item);
9544	ptr = btrfs_file_extent_inline_start(item);
9545
9546	encoded->len = min_t(u64, extent_start + ram_bytes,
9547	inode->vfs_inode.i_size) - iocb->ki_pos;
9548	ret = btrfs_encoded_io_compression_from_extent(fs_info,
9549	btrfs_file_extent_compression(leaf, item));
9550	if (ret < 0)
9551	return ret;
9552	encoded->compression = ret;
9553	if (encoded->compression) {
9554	size_t inline_size;
9555
9556	inline_size = btrfs_file_extent_inline_item_len(leaf,
9557	path->slots[0]);
9558	if (inline_size > count)
9559	return -ENOBUFS;
9560
9561	count = inline_size;
9562	encoded->unencoded_len = ram_bytes;
9563	encoded->unencoded_offset = iocb->ki_pos - extent_start;
9564	} else {
9565	count = min_t(u64, count, encoded->len);
9566	encoded->len = count;
9567	encoded->unencoded_len = count;
9568	ptr += iocb->ki_pos - extent_start;
9569	}
9570
9571	tmp = kmalloc(count, GFP_NOFS);
9572	if (!tmp)
9573	return -ENOMEM;
9574
9575	read_extent_buffer(leaf, tmp, ptr, count);
9576	btrfs_release_path(path);
9577	btrfs_unlock_extent(io_tree, start, lockend, cached_state);
9578	btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
9579	*unlocked = true;
9580
9581	ret = copy_to_iter(tmp, count, iter);
9582	if (ret != count)
9583	ret = -EFAULT;
9584	kfree(tmp);
9585
9586	return ret;
9587	}
9588
9589	struct btrfs_encoded_read_private {
9590	struct completion *sync_reads;
9591	void *uring_ctx;
9592	refcount_t pending_refs;
9593	blk_status_t status;
9594	};
9595
9596	static void btrfs_encoded_read_endio(struct btrfs_bio *bbio)
9597	{
9598	struct btrfs_encoded_read_private *priv = bbio->private;
9599
9600	if (bbio->bio.bi_status) {
9601	/*
9602	* The memory barrier implied by the refcount_dec_and_test() here
9603	* pairs with the memory barrier implied by the refcount_dec_and_test()
9604	* in btrfs_encoded_read_regular_fill_pages() to ensure that
9605	* this write is observed before the load of status in
9606	* btrfs_encoded_read_regular_fill_pages().
9607	*/
9608	WRITE_ONCE(priv->status, bbio->bio.bi_status);
9609	}
9610	if (refcount_dec_and_test(&priv->pending_refs)) {
9611	int err = blk_status_to_errno(READ_ONCE(priv->status));
9612
9613	if (priv->uring_ctx) {
9614	btrfs_uring_read_extent_endio(priv->uring_ctx, err);
9615	kfree(priv);
9616	} else {
9617	complete(priv->sync_reads);
9618	}
9619	}
9620	bio_put(&bbio->bio);
9621	}
9622
9623	int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode,
9624	u64 disk_bytenr, u64 disk_io_size,
9625	struct page *pages, void uring_ctx)
9626	{
9627	struct btrfs_encoded_read_private *priv, sync_priv;
9628	struct completion sync_reads;
9629	unsigned long i = 0;
9630	struct btrfs_bio *bbio;
9631
9632	/*
9633	* Fast path for synchronous reads which completes in this call, io_uring
9634	* needs longer time span.
9635	*/
9636	if (uring_ctx) {
9637	priv = kmalloc_obj(struct btrfs_encoded_read_private, GFP_NOFS);
9638	if (!priv)
9639	return -ENOMEM;
9640	} else {
9641	priv = &sync_priv;
9642	init_completion(&sync_reads);
9643	priv->sync_reads = &sync_reads;
9644	}
9645
9646	refcount_set(&priv->pending_refs, 1);
9647	priv->status = 0;
9648	priv->uring_ctx = uring_ctx;
9649
9650	bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, inode, 0,
9651	btrfs_encoded_read_endio, priv);
9652	bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;
9653
9654	do {
9655	size_t bytes = min_t(u64, disk_io_size, PAGE_SIZE);
9656
9657	if (bio_add_page(&bbio->bio, pages[i], bytes, 0) < bytes) {
9658	refcount_inc(&priv->pending_refs);
9659	btrfs_submit_bbio(bbio, 0);
9660
9661	bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, inode, 0,
9662	btrfs_encoded_read_endio, priv);
9663	bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;
9664	continue;
9665	}
9666
9667	i++;
9668	disk_bytenr += bytes;
9669	disk_io_size -= bytes;
9670	} while (disk_io_size);
9671
9672	refcount_inc(&priv->pending_refs);
9673	btrfs_submit_bbio(bbio, 0);
9674
9675	if (uring_ctx) {
9676	if (refcount_dec_and_test(&priv->pending_refs)) {
9677	int error = blk_status_to_errno(READ_ONCE(priv->status));
9678
9679	btrfs_uring_read_extent_endio(uring_ctx, error);
9680	kfree(priv);
9681	}
9682
9683	return -EIOCBQUEUED;
9684	} else {
9685	if (!refcount_dec_and_test(&priv->pending_refs))
9686	wait_for_completion_io(&sync_reads);
9687	/* See btrfs_encoded_read_endio() for ordering. */
9688	return blk_status_to_errno(READ_ONCE(priv->status));
9689	}
9690	}
9691
9692	ssize_t btrfs_encoded_read_regular(struct kiocb iocb, struct iov_iter iter,
9693	u64 start, u64 lockend,
9694	struct extent_state **cached_state,
9695	u64 disk_bytenr, u64 disk_io_size,
9696	size_t count, bool compressed, bool *unlocked)
9697	{
9698	struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp));
9699	struct extent_io_tree *io_tree = &inode->io_tree;
9700	struct page **pages;
9701	unsigned long nr_pages, i;
9702	u64 cur;
9703	size_t page_offset;
9704	ssize_t ret;
9705
9706	nr_pages = DIV_ROUND_UP(disk_io_size, PAGE_SIZE);
9707	pages = kzalloc_objs(struct page *, nr_pages, GFP_NOFS);
9708	if (!pages)
9709	return -ENOMEM;
9710	ret = btrfs_alloc_page_array(nr_pages, pages, GFP_NOFS);
9711	if (ret) {
9712	ret = -ENOMEM;
9713	goto out;
9714	}
9715
9716	ret = btrfs_encoded_read_regular_fill_pages(inode, disk_bytenr,
9717	disk_io_size, pages, NULL);
9718	if (ret)
9719	goto out;
9720
9721	btrfs_unlock_extent(io_tree, start, lockend, cached_state);
9722	btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
9723	*unlocked = true;
9724
9725	if (compressed) {
9726	i = 0;
9727	page_offset = 0;
9728	} else {
9729	i = (iocb->ki_pos - start) >> PAGE_SHIFT;
9730	page_offset = (iocb->ki_pos - start) & (PAGE_SIZE - 1);
9731	}
9732	cur = 0;
9733	while (cur < count) {
9734	size_t bytes = min_t(size_t, count - cur,
9735	PAGE_SIZE - page_offset);
9736
9737	if (copy_page_to_iter(pages[i], page_offset, bytes,
9738	iter) != bytes) {
9739	ret = -EFAULT;
9740	goto out;
9741	}
9742	i++;
9743	cur += bytes;
9744	page_offset = 0;
9745	}
9746	ret = count;
9747	out:
9748	for (i = 0; i < nr_pages; i++) {
9749	if (pages[i])
9750	__free_page(pages[i]);
9751	}
9752	kfree(pages);
9753	return ret;
9754	}
9755
9756	ssize_t btrfs_encoded_read(struct kiocb iocb, struct iov_iter iter,
9757	struct btrfs_ioctl_encoded_io_args *encoded,
9758	struct extent_state **cached_state,
9759	u64 disk_bytenr, u64 disk_io_size)
9760	{
9761	struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp));
9762	struct btrfs_fs_info *fs_info = inode->root->fs_info;
9763	struct extent_io_tree *io_tree = &inode->io_tree;
9764	ssize_t ret;
9765	size_t count = iov_iter_count(iter);
9766	u64 start, lockend;
9767	struct extent_map *em;
9768	const bool nowait = (iocb->ki_flags & IOCB_NOWAIT);
9769	bool unlocked = false;
9770
9771	file_accessed(iocb->ki_filp);
9772
9773	ret = btrfs_inode_lock(inode,
9774	BTRFS_ILOCK_SHARED \| (nowait ? BTRFS_ILOCK_TRY : 0));
9775	if (ret)
9776	return ret;
9777
9778	if (iocb->ki_pos >= inode->vfs_inode.i_size) {
9779	btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
9780	return 0;
9781	}
9782	start = ALIGN_DOWN(iocb->ki_pos, fs_info->sectorsize);
9783	/*
9784	* We don't know how long the extent containing iocb->ki_pos is, but if
9785	* it's compressed we know that it won't be longer than this.
9786	*/
9787	lockend = start + BTRFS_MAX_UNCOMPRESSED - 1;
9788
9789	if (nowait) {
9790	struct btrfs_ordered_extent *ordered;
9791
9792	if (filemap_range_needs_writeback(inode->vfs_inode.i_mapping,
9793	start, lockend)) {
9794	ret = -EAGAIN;
9795	goto out_unlock_inode;
9796	}
9797
9798	if (!btrfs_try_lock_extent(io_tree, start, lockend, cached_state)) {
9799	ret = -EAGAIN;
9800	goto out_unlock_inode;
9801	}
9802
9803	ordered = btrfs_lookup_ordered_range(inode, start,
9804	lockend - start + 1);
9805	if (ordered) {
9806	btrfs_put_ordered_extent(ordered);
9807	btrfs_unlock_extent(io_tree, start, lockend, cached_state);
9808	ret = -EAGAIN;
9809	goto out_unlock_inode;
9810	}
9811	} else {
9812	for (;;) {
9813	struct btrfs_ordered_extent *ordered;
9814
9815	ret = btrfs_wait_ordered_range(inode, start,
9816	lockend - start + 1);
9817	if (ret)
9818	goto out_unlock_inode;
9819
9820	btrfs_lock_extent(io_tree, start, lockend, cached_state);
9821	ordered = btrfs_lookup_ordered_range(inode, start,
9822	lockend - start + 1);
9823	if (!ordered)
9824	break;
9825	btrfs_put_ordered_extent(ordered);
9826	btrfs_unlock_extent(io_tree, start, lockend, cached_state);
9827	cond_resched();
9828	}
9829	}
9830
9831	em = btrfs_get_extent(inode, NULL, start, lockend - start + 1);
9832	if (IS_ERR(em)) {
9833	ret = PTR_ERR(em);
9834	goto out_unlock_extent;
9835	}
9836
9837	if (em->disk_bytenr == EXTENT_MAP_INLINE) {
9838	u64 extent_start = em->start;
9839
9840	/*
9841	* For inline extents we get everything we need out of the
9842	* extent item.
9843	*/
9844	btrfs_free_extent_map(em);
9845	em = NULL;
9846	ret = btrfs_encoded_read_inline(iocb, iter, start, lockend,
9847	cached_state, extent_start,
9848	count, encoded, &unlocked);
9849	goto out_unlock_extent;
9850	}
9851
9852	/*
9853	* We only want to return up to EOF even if the extent extends beyond
9854	* that.
9855	*/
9856	encoded->len = min_t(u64, btrfs_extent_map_end(em),
9857	inode->vfs_inode.i_size) - iocb->ki_pos;
9858	if (em->disk_bytenr == EXTENT_MAP_HOLE \|\|
9859	(em->flags & EXTENT_FLAG_PREALLOC)) {
9860	*disk_bytenr = EXTENT_MAP_HOLE;
9861	count = min_t(u64, count, encoded->len);
9862	encoded->len = count;
9863	encoded->unencoded_len = count;
9864	} else if (btrfs_extent_map_is_compressed(em)) {
9865	*disk_bytenr = em->disk_bytenr;
9866	/*
9867	* Bail if the buffer isn't large enough to return the whole
9868	* compressed extent.
9869	*/
9870	if (em->disk_num_bytes > count) {
9871	ret = -ENOBUFS;
9872	goto out_em;
9873	}
9874	*disk_io_size = em->disk_num_bytes;
9875	count = em->disk_num_bytes;
9876	encoded->unencoded_len = em->ram_bytes;
9877	encoded->unencoded_offset = iocb->ki_pos - (em->start - em->offset);
9878	ret = btrfs_encoded_io_compression_from_extent(fs_info,
9879	btrfs_extent_map_compression(em));
9880	if (ret < 0)
9881	goto out_em;
9882	encoded->compression = ret;
9883	} else {
9884	*disk_bytenr = btrfs_extent_map_block_start(em) + (start - em->start);
9885	if (encoded->len > count)
9886	encoded->len = count;
9887	/*
9888	* Don't read beyond what we locked. This also limits the page
9889	* allocations that we'll do.
9890	*/
9891	*disk_io_size = min(lockend + 1, iocb->ki_pos + encoded->len) - start;
9892	count = start + *disk_io_size - iocb->ki_pos;
9893	encoded->len = count;
9894	encoded->unencoded_len = count;
9895	disk_io_size = ALIGN(disk_io_size, fs_info->sectorsize);
9896	}
9897	btrfs_free_extent_map(em);
9898	em = NULL;
9899
9900	if (*disk_bytenr == EXTENT_MAP_HOLE) {
9901	btrfs_unlock_extent(io_tree, start, lockend, cached_state);
9902	btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
9903	unlocked = true;
9904	ret = iov_iter_zero(count, iter);
9905	if (ret != count)
9906	ret = -EFAULT;
9907	} else {
9908	ret = -EIOCBQUEUED;
9909	goto out_unlock_extent;
9910	}
9911
9912	out_em:
9913	btrfs_free_extent_map(em);
9914	out_unlock_extent:
9915	/* Leave inode and extent locked if we need to do a read. */
9916	if (!unlocked && ret != -EIOCBQUEUED)
9917	btrfs_unlock_extent(io_tree, start, lockend, cached_state);
9918	out_unlock_inode:
9919	if (!unlocked && ret != -EIOCBQUEUED)
9920	btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
9921	return ret;
9922	}
9923
9924	ssize_t btrfs_do_encoded_write(struct kiocb iocb, struct iov_iter from,
9925	const struct btrfs_ioctl_encoded_io_args *encoded)
9926	{
9927	struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp));
9928	struct btrfs_root *root = inode->root;
9929	struct btrfs_fs_info *fs_info = root->fs_info;
9930	struct extent_io_tree *io_tree = &inode->io_tree;
9931	struct extent_changeset *data_reserved = NULL;
9932	struct extent_state *cached_state = NULL;
9933	struct btrfs_ordered_extent *ordered;
9934	struct btrfs_file_extent file_extent;
9935	struct compressed_bio *cb = NULL;
9936	int compression;
9937	size_t orig_count;
9938	const u32 min_folio_size = btrfs_min_folio_size(fs_info);
9939	const u32 blocksize = fs_info->sectorsize;
9940	u64 start, end;
9941	u64 num_bytes, ram_bytes, disk_num_bytes;
9942	struct btrfs_key ins;
9943	bool extent_reserved = false;
9944	struct extent_map *em;
9945	ssize_t ret;
9946
9947	switch (encoded->compression) {
9948	case BTRFS_ENCODED_IO_COMPRESSION_ZLIB:
9949	compression = BTRFS_COMPRESS_ZLIB;
9950	break;
9951	case BTRFS_ENCODED_IO_COMPRESSION_ZSTD:
9952	compression = BTRFS_COMPRESS_ZSTD;
9953	break;
9954	case BTRFS_ENCODED_IO_COMPRESSION_LZO_4K:
9955	case BTRFS_ENCODED_IO_COMPRESSION_LZO_8K:
9956	case BTRFS_ENCODED_IO_COMPRESSION_LZO_16K:
9957	case BTRFS_ENCODED_IO_COMPRESSION_LZO_32K:
9958	case BTRFS_ENCODED_IO_COMPRESSION_LZO_64K:
9959	/* The sector size must match for LZO. */
9960	if (encoded->compression -
9961	BTRFS_ENCODED_IO_COMPRESSION_LZO_4K + 12 !=
9962	fs_info->sectorsize_bits)
9963	return -EINVAL;
9964	compression = BTRFS_COMPRESS_LZO;
9965	break;
9966	default:
9967	return -EINVAL;
9968	}
9969	if (encoded->encryption != BTRFS_ENCODED_IO_ENCRYPTION_NONE)
9970	return -EINVAL;
9971
9972	/*
9973	* Compressed extents should always have checksums, so error out if we
9974	* have a NOCOW file or inode was created while mounted with NODATASUM.
9975	*/
9976	if (inode->flags & BTRFS_INODE_NODATASUM)
9977	return -EINVAL;
9978
9979	orig_count = iov_iter_count(from);
9980
9981	/* The extent size must be sane. */
9982	if (encoded->unencoded_len > BTRFS_MAX_UNCOMPRESSED \|\|
9983	orig_count > BTRFS_MAX_COMPRESSED \|\| orig_count == 0)
9984	return -EINVAL;
9985
9986	/*
9987	* The compressed data must be smaller than the decompressed data.
9988	*
9989	* It's of course possible for data to compress to larger or the same
9990	* size, but the buffered I/O path falls back to no compression for such
9991	* data, and we don't want to break any assumptions by creating these
9992	* extents.
9993	*
9994	* Note that this is less strict than the current check we have that the
9995	* compressed data must be at least one sector smaller than the
9996	* decompressed data. We only want to enforce the weaker requirement
9997	* from old kernels that it is at least one byte smaller.
9998	*/
9999	if (orig_count >= encoded->unencoded_len)
10000	return -EINVAL;
10001
10002	/* The extent must start on a sector boundary. */
10003	start = iocb->ki_pos;
10004	if (!IS_ALIGNED(start, fs_info->sectorsize))
10005	return -EINVAL;
10006
10007	/*
10008	* The extent must end on a sector boundary. However, we allow a write
10009	* which ends at or extends i_size to have an unaligned length; we round
10010	* up the extent size and set i_size to the unaligned end.
10011	*/
10012	if (start + encoded->len < inode->vfs_inode.i_size &&
10013	!IS_ALIGNED(start + encoded->len, fs_info->sectorsize))
10014	return -EINVAL;
10015
10016	/* Finally, the offset in the unencoded data must be sector-aligned. */
10017	if (!IS_ALIGNED(encoded->unencoded_offset, fs_info->sectorsize))
10018	return -EINVAL;
10019
10020	num_bytes = ALIGN(encoded->len, fs_info->sectorsize);
10021	ram_bytes = ALIGN(encoded->unencoded_len, fs_info->sectorsize);
10022	end = start + num_bytes - 1;
10023
10024	/*
10025	* If the extent cannot be inline, the compressed data on disk must be
10026	* sector-aligned. For convenience, we extend it with zeroes if it
10027	* isn't.
10028	*/
10029	disk_num_bytes = ALIGN(orig_count, fs_info->sectorsize);
10030
10031	cb = btrfs_alloc_compressed_write(inode, start, num_bytes);
10032	for (int i = 0; i * min_folio_size < disk_num_bytes; i++) {
10033	struct folio *folio;
10034	size_t bytes = min(min_folio_size, iov_iter_count(from));
10035	char *kaddr;
10036
10037	folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS);
10038	if (!folio) {
10039	ret = -ENOMEM;
10040	goto out_cb;
10041	}
10042	kaddr = kmap_local_folio(folio, 0);
10043	ret = copy_from_iter(kaddr, bytes, from);
10044	kunmap_local(kaddr);
10045	if (ret != bytes) {
10046	folio_put(folio);
10047	ret = -EFAULT;
10048	goto out_cb;
10049	}
10050	if (!IS_ALIGNED(bytes, blocksize))
10051	folio_zero_range(folio, bytes, round_up(bytes, blocksize) - bytes);
10052	ret = bio_add_folio(&cb->bbio.bio, folio, round_up(bytes, blocksize), 0);
10053	if (unlikely(!ret)) {
10054	folio_put(folio);
10055	ret = -EINVAL;
10056	goto out_cb;
10057	}
10058	}
10059	ASSERT(cb->bbio.bio.bi_iter.bi_size == disk_num_bytes);
10060
10061	for (;;) {
10062	ret = btrfs_wait_ordered_range(inode, start, num_bytes);
10063	if (ret)
10064	goto out_cb;
10065	ret = invalidate_inode_pages2_range(inode->vfs_inode.i_mapping,
10066	start >> PAGE_SHIFT,
10067	end >> PAGE_SHIFT);
10068	if (ret)
10069	goto out_cb;
10070	btrfs_lock_extent(io_tree, start, end, &cached_state);
10071	ordered = btrfs_lookup_ordered_range(inode, start, num_bytes);
10072	if (!ordered &&
10073	!filemap_range_has_page(inode->vfs_inode.i_mapping, start, end))
10074	break;
10075	if (ordered)
10076	btrfs_put_ordered_extent(ordered);
10077	btrfs_unlock_extent(io_tree, start, end, &cached_state);
10078	cond_resched();
10079	}
10080
10081	/*
10082	* We don't use the higher-level delalloc space functions because our
10083	* num_bytes and disk_num_bytes are different.
10084	*/
10085	ret = btrfs_alloc_data_chunk_ondemand(inode, disk_num_bytes);
10086	if (ret)
10087	goto out_unlock;
10088	ret = btrfs_qgroup_reserve_data(inode, &data_reserved, start, num_bytes);
10089	if (ret)
10090	goto out_free_data_space;
10091	ret = btrfs_delalloc_reserve_metadata(inode, num_bytes, disk_num_bytes,
10092	false);
10093	if (ret)
10094	goto out_qgroup_free_data;
10095
10096	/* Try an inline extent first. */
10097	if (encoded->unencoded_len == encoded->len &&
10098	encoded->unencoded_offset == 0 &&
10099	can_cow_file_range_inline(inode, start, encoded->len, orig_count)) {
10100	ret = __cow_file_range_inline(inode, encoded->len,
10101	orig_count, compression,
10102	bio_first_folio_all(&cb->bbio.bio),
10103	true);
10104	if (ret <= 0) {
10105	if (ret == 0)
10106	ret = orig_count;
10107	goto out_delalloc_release;
10108	}
10109	}
10110
10111	ret = btrfs_reserve_extent(root, disk_num_bytes, disk_num_bytes,
10112	disk_num_bytes, 0, 0, &ins, true, true);
10113	if (ret)
10114	goto out_delalloc_release;
10115	extent_reserved = true;
10116
10117	file_extent.disk_bytenr = ins.objectid;
10118	file_extent.disk_num_bytes = ins.offset;
10119	file_extent.num_bytes = num_bytes;
10120	file_extent.ram_bytes = ram_bytes;
10121	file_extent.offset = encoded->unencoded_offset;
10122	file_extent.compression = compression;
10123	em = btrfs_create_io_em(inode, start, &file_extent, BTRFS_ORDERED_COMPRESSED);
10124	if (IS_ERR(em)) {
10125	ret = PTR_ERR(em);
10126	goto out_free_reserved;
10127	}
10128	btrfs_free_extent_map(em);
10129
10130	ordered = btrfs_alloc_ordered_extent(inode, start, &file_extent,
10131	(1U << BTRFS_ORDERED_ENCODED) \|
10132	(1U << BTRFS_ORDERED_COMPRESSED));
10133	if (IS_ERR(ordered)) {
10134	btrfs_drop_extent_map_range(inode, start, end, false);
10135	ret = PTR_ERR(ordered);
10136	goto out_free_reserved;
10137	}
10138	btrfs_dec_block_group_reservations(fs_info, ins.objectid);
10139
10140	if (start + encoded->len > inode->vfs_inode.i_size)
10141	i_size_write(&inode->vfs_inode, start + encoded->len);
10142
10143	btrfs_unlock_extent(io_tree, start, end, &cached_state);
10144
10145	btrfs_delalloc_release_extents(inode, num_bytes);
10146
10147	btrfs_submit_compressed_write(ordered, cb);
10148	ret = orig_count;
10149	goto out;
10150
10151	out_free_reserved:
10152	btrfs_dec_block_group_reservations(fs_info, ins.objectid);
10153	btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, true);
10154	out_delalloc_release:
10155	btrfs_delalloc_release_extents(inode, num_bytes);
10156	btrfs_delalloc_release_metadata(inode, disk_num_bytes, ret < 0);
10157	out_qgroup_free_data:
10158	if (ret < 0)
10159	btrfs_qgroup_free_data(inode, data_reserved, start, num_bytes, NULL);
10160	out_free_data_space:
10161	/*
10162	* If btrfs_reserve_extent() succeeded, then we already decremented
10163	* bytes_may_use.
10164	*/
10165	if (!extent_reserved)
10166	btrfs_free_reserved_data_space_noquota(inode, disk_num_bytes);
10167	out_unlock:
10168	btrfs_unlock_extent(io_tree, start, end, &cached_state);
10169	out_cb:
10170	if (cb)
10171	cleanup_compressed_bio(cb);
10172	out:
10173	extent_changeset_free(data_reserved);
10174	if (ret >= 0)
10175	iocb->ki_pos += encoded->len;
10176	return ret;
10177	}
10178
10179	#ifdef CONFIG_SWAP
10180	/*
10181	* Add an entry indicating a block group or device which is pinned by a
10182	* swapfile. Returns 0 on success, 1 if there is already an entry for it, or a
10183	* negative errno on failure.
10184	*/
10185	static int btrfs_add_swapfile_pin(struct inode inode, void ptr,
10186	bool is_block_group)
10187	{
10188	struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
10189	struct btrfs_swapfile_pin sp, entry;
10190	struct rb_node **p;
10191	struct rb_node *parent = NULL;
10192
10193	sp = kmalloc_obj(*sp, GFP_NOFS);
10194	if (!sp)
10195	return -ENOMEM;
10196	sp->ptr = ptr;
10197	sp->inode = inode;
10198	sp->is_block_group = is_block_group;
10199	sp->bg_extent_count = 1;
10200
10201	spin_lock(&fs_info->swapfile_pins_lock);
10202	p = &fs_info->swapfile_pins.rb_node;
10203	while (*p) {
10204	parent = *p;
10205	entry = rb_entry(parent, struct btrfs_swapfile_pin, node);
10206	if (sp->ptr < entry->ptr \|\|
10207	(sp->ptr == entry->ptr && sp->inode < entry->inode)) {
10208	p = &(*p)->rb_left;
10209	} else if (sp->ptr > entry->ptr \|\|
10210	(sp->ptr == entry->ptr && sp->inode > entry->inode)) {
10211	p = &(*p)->rb_right;
10212	} else {
10213	if (is_block_group)
10214	entry->bg_extent_count++;
10215	spin_unlock(&fs_info->swapfile_pins_lock);
10216	kfree(sp);
10217	return 1;
10218	}
10219	}
10220	rb_link_node(&sp->node, parent, p);
10221	rb_insert_color(&sp->node, &fs_info->swapfile_pins);
10222	spin_unlock(&fs_info->swapfile_pins_lock);
10223	return 0;
10224	}
10225
10226	/* Free all of the entries pinned by this swapfile. */
10227	static void btrfs_free_swapfile_pins(struct inode *inode)
10228	{
10229	struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
10230	struct btrfs_swapfile_pin *sp;
10231	struct rb_node node, next;
10232	u64 bg_bytes_released = 0;
10233	u32 bg_nr_released = 0;
10234
10235	spin_lock(&fs_info->swapfile_pins_lock);
10236	node = rb_first(&fs_info->swapfile_pins);
10237	while (node) {
10238	next = rb_next(node);
10239	sp = rb_entry(node, struct btrfs_swapfile_pin, node);
10240	if (sp->inode == inode) {
10241	rb_erase(&sp->node, &fs_info->swapfile_pins);
10242	if (sp->is_block_group) {
10243	struct btrfs_block_group *bg = sp->ptr;
10244
10245	bg_bytes_released += bg->length;
10246	bg_nr_released++;
10247	btrfs_dec_block_group_swap_extents(bg,
10248	sp->bg_extent_count);
10249	btrfs_put_block_group(bg);
10250	}
10251	kfree(sp);
10252	}
10253	node = next;
10254	}
10255	spin_unlock(&fs_info->swapfile_pins_lock);
10256	btrfs_info(fs_info,
10257	"swapfile deactivated on root %llu ino %llu, released %llu bytes from %u block group(s)",
10258	btrfs_root_id(BTRFS_I(inode)->root),
10259	btrfs_ino(BTRFS_I(inode)), bg_bytes_released,
10260	bg_nr_released);
10261	}
10262
10263	struct btrfs_swap_info {
10264	u64 start;
10265	u64 block_start;
10266	u64 block_len;
10267	u64 lowest_ppage;
10268	u64 highest_ppage;
10269	unsigned long nr_pages;
10270	int nr_extents;
10271	};
10272
10273	static int btrfs_add_swap_extent(struct swap_info_struct *sis,
10274	struct btrfs_swap_info *bsi)
10275	{
10276	unsigned long nr_pages;
10277	unsigned long max_pages;
10278	u64 first_ppage, first_ppage_reported, next_ppage;
10279	int ret;
10280
10281	/*
10282	* Our swapfile may have had its size extended after the swap header was
10283	* written. In that case activating the swapfile should not go beyond
10284	* the max size set in the swap header.
10285	*/
10286	if (bsi->nr_pages >= sis->max)
10287	return 0;
10288
10289	max_pages = sis->max - bsi->nr_pages;
10290	first_ppage = PAGE_ALIGN(bsi->block_start) >> PAGE_SHIFT;
10291	next_ppage = PAGE_ALIGN_DOWN(bsi->block_start + bsi->block_len) >> PAGE_SHIFT;
10292
10293	if (first_ppage >= next_ppage)
10294	return 0;
10295	nr_pages = next_ppage - first_ppage;
10296	nr_pages = min(nr_pages, max_pages);
10297
10298	first_ppage_reported = first_ppage;
10299	if (bsi->start == 0)
10300	first_ppage_reported++;
10301	if (bsi->lowest_ppage > first_ppage_reported)
10302	bsi->lowest_ppage = first_ppage_reported;
10303	if (bsi->highest_ppage < (next_ppage - 1))
10304	bsi->highest_ppage = next_ppage - 1;
10305
10306	ret = add_swap_extent(sis, bsi->nr_pages, nr_pages, first_ppage);
10307	if (ret < 0)
10308	return ret;
10309	bsi->nr_extents += ret;
10310	bsi->nr_pages += nr_pages;
10311	return 0;
10312	}
10313
10314	static void btrfs_swap_deactivate(struct file *file)
10315	{
10316	struct inode *inode = file_inode(file);
10317
10318	btrfs_free_swapfile_pins(inode);
10319	atomic_dec(&BTRFS_I(inode)->root->nr_swapfiles);
10320	}
10321
10322	static int btrfs_swap_activate(struct swap_info_struct sis, struct file file,
10323	sector_t *span)
10324	{
10325	struct inode *inode = file_inode(file);
10326	struct btrfs_root *root = BTRFS_I(inode)->root;
10327	struct btrfs_fs_info *fs_info = root->fs_info;
10328	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
10329	struct extent_state *cached_state = NULL;
10330	struct btrfs_chunk_map *map = NULL;
10331	struct btrfs_device *device = NULL;
10332	struct btrfs_swap_info bsi = {
10333	.lowest_ppage = (sector_t)-1ULL,
10334	};
10335	struct btrfs_backref_share_check_ctx *backref_ctx = NULL;
10336	struct btrfs_path *path = NULL;
10337	int ret = 0;
10338	u32 pinned_bg_nr = 0;
10339	u64 isize;
10340	u64 prev_extent_end = 0;
10341	u64 pinned_bg_size = 0;
10342
10343	/*
10344	* Acquire the inode's mmap lock to prevent races with memory mapped
10345	* writes, as they could happen after we flush delalloc below and before
10346	* we lock the extent range further below. The inode was already locked
10347	* up in the call chain.
10348	*/
10349	btrfs_assert_inode_locked(BTRFS_I(inode));
10350	down_write(&BTRFS_I(inode)->i_mmap_lock);
10351
10352	/*
10353	* If the swap file was just created, make sure delalloc is done. If the
10354	* file changes again after this, the user is doing something stupid and
10355	* we don't really care.
10356	*/
10357	ret = btrfs_wait_ordered_range(BTRFS_I(inode), 0, (u64)-1);
10358	if (ret)
10359	goto out_unlock_mmap;
10360
10361	/*
10362	* The inode is locked, so these flags won't change after we check them.
10363	*/
10364	if (BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS) {
10365	btrfs_warn(fs_info, "swapfile must not be compressed");
10366	ret = -EINVAL;
10367	goto out_unlock_mmap;
10368	}
10369	if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW)) {
10370	btrfs_warn(fs_info, "swapfile must not be copy-on-write");
10371	ret = -EINVAL;
10372	goto out_unlock_mmap;
10373	}
10374	if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) {
10375	btrfs_warn(fs_info, "swapfile must not be checksummed");
10376	ret = -EINVAL;
10377	goto out_unlock_mmap;
10378	}
10379
10380	path = btrfs_alloc_path();
10381	backref_ctx = btrfs_alloc_backref_share_check_ctx();
10382	if (!path \|\| !backref_ctx) {
10383	ret = -ENOMEM;
10384	goto out_unlock_mmap;
10385	}
10386
10387	/*
10388	* Balance or device remove/replace/resize can move stuff around from
10389	* under us. The exclop protection makes sure they aren't running/won't
10390	* run concurrently while we are mapping the swap extents, and
10391	* fs_info->swapfile_pins prevents them from running while the swap
10392	* file is active and moving the extents. Note that this also prevents
10393	* a concurrent device add which isn't actually necessary, but it's not
10394	* really worth the trouble to allow it.
10395	*/
10396	if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_SWAP_ACTIVATE)) {
10397	btrfs_warn(fs_info,
10398	"cannot activate swapfile while exclusive operation is running");
10399	ret = -EBUSY;
10400	goto out_unlock_mmap;
10401	}
10402
10403	/*
10404	* Prevent snapshot creation while we are activating the swap file.
10405	* We do not want to race with snapshot creation. If snapshot creation
10406	* already started before we bumped nr_swapfiles from 0 to 1 and
10407	* completes before the first write into the swap file after it is
10408	* activated, than that write would fallback to COW.
10409	*/
10410	if (!btrfs_drew_try_write_lock(&root->snapshot_lock)) {
10411	btrfs_exclop_finish(fs_info);
10412	btrfs_warn(fs_info,
10413	"cannot activate swapfile because snapshot creation is in progress");
10414	ret = -EINVAL;
10415	goto out_unlock_mmap;
10416	}
10417	/*
10418	* Snapshots can create extents which require COW even if NODATACOW is
10419	* set. We use this counter to prevent snapshots. We must increment it
10420	* before walking the extents because we don't want a concurrent
10421	* snapshot to run after we've already checked the extents.
10422	*
10423	* It is possible that subvolume is marked for deletion but still not
10424	* removed yet. To prevent this race, we check the root status before
10425	* activating the swapfile.
10426	*/
10427	spin_lock(&root->root_item_lock);
10428	if (btrfs_root_dead(root)) {
10429	spin_unlock(&root->root_item_lock);
10430
10431	btrfs_drew_write_unlock(&root->snapshot_lock);
10432	btrfs_exclop_finish(fs_info);
10433	btrfs_warn(fs_info,
10434	"cannot activate swapfile because subvolume %llu is being deleted",
10435	btrfs_root_id(root));
10436	ret = -EPERM;
10437	goto out_unlock_mmap;
10438	}
10439	atomic_inc(&root->nr_swapfiles);
10440	spin_unlock(&root->root_item_lock);
10441
10442	isize = ALIGN_DOWN(inode->i_size, fs_info->sectorsize);
10443
10444	btrfs_lock_extent(io_tree, 0, isize - 1, &cached_state);
10445	while (prev_extent_end < isize) {
10446	struct btrfs_key key;
10447	struct extent_buffer *leaf;
10448	struct btrfs_file_extent_item *ei;
10449	struct btrfs_block_group *bg;
10450	u64 logical_block_start;
10451	u64 physical_block_start;
10452	u64 extent_gen;
10453	u64 disk_bytenr;
10454	u64 len;
10455
10456	key.objectid = btrfs_ino(BTRFS_I(inode));
10457	key.type = BTRFS_EXTENT_DATA_KEY;
10458	key.offset = prev_extent_end;
10459
10460	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
10461	if (ret < 0)
10462	goto out;
10463
10464	/*
10465	* If key not found it means we have an implicit hole (NO_HOLES
10466	* is enabled).
10467	*/
10468	if (ret > 0) {
10469	btrfs_warn(fs_info, "swapfile must not have holes");
10470	ret = -EINVAL;
10471	goto out;
10472	}
10473
10474	leaf = path->nodes[0];
10475	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
10476
10477	if (btrfs_file_extent_type(leaf, ei) == BTRFS_FILE_EXTENT_INLINE) {
10478	/*
10479	* It's unlikely we'll ever actually find ourselves
10480	* here, as a file small enough to fit inline won't be
10481	* big enough to store more than the swap header, but in
10482	* case something changes in the future, let's catch it
10483	* here rather than later.
10484	*/
10485	btrfs_warn(fs_info, "swapfile must not be inline");
10486	ret = -EINVAL;
10487	goto out;
10488	}
10489
10490	if (btrfs_file_extent_compression(leaf, ei) != BTRFS_COMPRESS_NONE) {
10491	btrfs_warn(fs_info, "swapfile must not be compressed");
10492	ret = -EINVAL;
10493	goto out;
10494	}
10495
10496	disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
10497	if (disk_bytenr == 0) {
10498	btrfs_warn(fs_info, "swapfile must not have holes");
10499	ret = -EINVAL;
10500	goto out;
10501	}
10502
10503	logical_block_start = disk_bytenr + btrfs_file_extent_offset(leaf, ei);
10504	extent_gen = btrfs_file_extent_generation(leaf, ei);
10505	prev_extent_end = btrfs_file_extent_end(path);
10506
10507	if (prev_extent_end > isize)
10508	len = isize - key.offset;
10509	else
10510	len = btrfs_file_extent_num_bytes(leaf, ei);
10511
10512	backref_ctx->curr_leaf_bytenr = leaf->start;
10513
10514	/*
10515	* Don't need the path anymore, release to avoid deadlocks when
10516	* calling btrfs_is_data_extent_shared() because when joining a
10517	* transaction it can block waiting for the current one's commit
10518	* which in turn may be trying to lock the same leaf to flush
10519	* delayed items for example.
10520	*/
10521	btrfs_release_path(path);
10522
10523	ret = btrfs_is_data_extent_shared(BTRFS_I(inode), disk_bytenr,
10524	extent_gen, backref_ctx);
10525	if (ret < 0) {
10526	goto out;
10527	} else if (ret > 0) {
10528	btrfs_warn(fs_info,
10529	"swapfile must not be copy-on-write");
10530	ret = -EINVAL;
10531	goto out;
10532	}
10533
10534	map = btrfs_get_chunk_map(fs_info, logical_block_start, len);
10535	if (IS_ERR(map)) {
10536	ret = PTR_ERR(map);
10537	goto out;
10538	}
10539
10540	if (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) {
10541	btrfs_warn(fs_info,
10542	"swapfile must have single data profile");
10543	ret = -EINVAL;
10544	goto out;
10545	}
10546
10547	if (device == NULL) {
10548	device = map->stripes[0].dev;
10549	ret = btrfs_add_swapfile_pin(inode, device, false);
10550	if (ret == 1)
10551	ret = 0;
10552	else if (ret)
10553	goto out;
10554	} else if (device != map->stripes[0].dev) {
10555	btrfs_warn(fs_info, "swapfile must be on one device");
10556	ret = -EINVAL;
10557	goto out;
10558	}
10559
10560	physical_block_start = (map->stripes[0].physical +
10561	(logical_block_start - map->start));
10562	btrfs_free_chunk_map(map);
10563	map = NULL;
10564
10565	bg = btrfs_lookup_block_group(fs_info, logical_block_start);
10566	if (!bg) {
10567	btrfs_warn(fs_info,
10568	"could not find block group containing swapfile");
10569	ret = -EINVAL;
10570	goto out;
10571	}
10572
10573	if (!btrfs_inc_block_group_swap_extents(bg)) {
10574	btrfs_warn(fs_info,
10575	"block group for swapfile at %llu is read-only%s",
10576	bg->start,
10577	atomic_read(&fs_info->scrubs_running) ?
10578	" (scrub running)" : "");
10579	btrfs_put_block_group(bg);
10580	ret = -EINVAL;
10581	goto out;
10582	}
10583
10584	ret = btrfs_add_swapfile_pin(inode, bg, true);
10585	if (ret) {
10586	btrfs_put_block_group(bg);
10587	if (ret == 1)
10588	ret = 0;
10589	else
10590	goto out;
10591	} else {
10592	pinned_bg_size += bg->length;
10593	pinned_bg_nr++;
10594	}
10595
10596	if (bsi.block_len &&
10597	bsi.block_start + bsi.block_len == physical_block_start) {
10598	bsi.block_len += len;
10599	} else {
10600	if (bsi.block_len) {
10601	ret = btrfs_add_swap_extent(sis, &bsi);
10602	if (ret)
10603	goto out;
10604	}
10605	bsi.start = key.offset;
10606	bsi.block_start = physical_block_start;
10607	bsi.block_len = len;
10608	}
10609
10610	if (fatal_signal_pending(current)) {
10611	ret = -EINTR;
10612	goto out;
10613	}
10614
10615	cond_resched();
10616	}
10617
10618	if (bsi.block_len)
10619	ret = btrfs_add_swap_extent(sis, &bsi);
10620
10621	out:
10622	if (!IS_ERR_OR_NULL(map))
10623	btrfs_free_chunk_map(map);
10624
10625	btrfs_unlock_extent(io_tree, 0, isize - 1, &cached_state);
10626
10627	if (ret)
10628	btrfs_swap_deactivate(file);
10629
10630	btrfs_drew_write_unlock(&root->snapshot_lock);
10631
10632	btrfs_exclop_finish(fs_info);
10633
10634	out_unlock_mmap:
10635	up_write(&BTRFS_I(inode)->i_mmap_lock);
10636	btrfs_free_backref_share_ctx(backref_ctx);
10637	btrfs_free_path(path);
10638	if (ret)
10639	return ret;
10640
10641	btrfs_info(fs_info,
10642	"swapfile activated on root %llu ino %llu, pinned down %llu bytes from %u block group(s)",
10643	btrfs_root_id(BTRFS_I(inode)->root),
10644	btrfs_ino(BTRFS_I(inode)),
10645	pinned_bg_size, pinned_bg_nr);
10646	btrfs_warn(fs_info,
10647	"block groups with swapfile extents will not be scrubbed or balanced");
10648
10649	if (device)
10650	sis->bdev = device->bdev;
10651	*span = bsi.highest_ppage - bsi.lowest_ppage + 1;
10652	sis->max = bsi.nr_pages;
10653	sis->pages = bsi.nr_pages - 1;
10654	return bsi.nr_extents;
10655	}
10656	#else
10657	static void btrfs_swap_deactivate(struct file *file)
10658	{
10659	}
10660
10661	static int btrfs_swap_activate(struct swap_info_struct sis, struct file file,
10662	sector_t *span)
10663	{
10664	return -EOPNOTSUPP;
10665	}
10666	#endif
10667
10668	/*
10669	* Update the number of bytes used in the VFS' inode. When we replace extents in
10670	* a range (clone, dedupe, fallocate's zero range), we must update the number of
10671	* bytes used by the inode in an atomic manner, so that concurrent stat(2) calls
10672	* always get a correct value.
10673	*/
10674	void btrfs_update_inode_bytes(struct btrfs_inode *inode,
10675	const u64 add_bytes,
10676	const u64 del_bytes)
10677	{
10678	if (add_bytes == del_bytes)
10679	return;
10680
10681	spin_lock(&inode->lock);
10682	if (del_bytes > 0)
10683	inode_sub_bytes(&inode->vfs_inode, del_bytes);
10684	if (add_bytes > 0)
10685	inode_add_bytes(&inode->vfs_inode, add_bytes);
10686	spin_unlock(&inode->lock);
10687	}
10688
10689	/*
10690	* Verify that there are no ordered extents for a given file range.
10691	*
10692	* @inode: The target inode.
10693	* @start: Start offset of the file range, should be sector size aligned.
10694	* @end: End offset (inclusive) of the file range, its value +1 should be
10695	* sector size aligned.
10696	*
10697	* This should typically be used for cases where we locked an inode's VFS lock in
10698	* exclusive mode, we have also locked the inode's i_mmap_lock in exclusive mode,
10699	* we have flushed all delalloc in the range, we have waited for all ordered
10700	* extents in the range to complete and finally we have locked the file range in
10701	* the inode's io_tree.
10702	*/
10703	void btrfs_assert_inode_range_clean(struct btrfs_inode *inode, u64 start, u64 end)
10704	{
10705	struct btrfs_root *root = inode->root;
10706	struct btrfs_ordered_extent *ordered;
10707
10708	if (!IS_ENABLED(CONFIG_BTRFS_ASSERT))
10709	return;
10710
10711	ordered = btrfs_lookup_first_ordered_range(inode, start, end + 1 - start);
10712	if (ordered) {
10713	btrfs_err(root->fs_info,
10714	"found unexpected ordered extent in file range [%llu, %llu] for inode %llu root %llu (ordered range [%llu, %llu])",
10715	start, end, btrfs_ino(inode), btrfs_root_id(root),
10716	ordered->file_offset,
10717	ordered->file_offset + ordered->num_bytes - 1);
10718	btrfs_put_ordered_extent(ordered);
10719	}
10720
10721	ASSERT(ordered == NULL);
10722	}
10723
10724	/*
10725	* Find the first inode with a minimum number.
10726	*
10727	* @root: The root to search for.
10728	* @min_ino: The minimum inode number.
10729	*
10730	* Find the first inode in the @root with a number >= @min_ino and return it.
10731	* Returns NULL if no such inode found.
10732	*/
10733	struct btrfs_inode btrfs_find_first_inode(struct btrfs_root root, u64 min_ino)
10734	{
10735	struct btrfs_inode *inode;
10736	unsigned long from = min_ino;
10737
10738	xa_lock(&root->inodes);
10739	while (true) {
10740	inode = xa_find(&root->inodes, &from, ULONG_MAX, XA_PRESENT);
10741	if (!inode)
10742	break;
10743	if (igrab(&inode->vfs_inode))
10744	break;
10745
10746	from = btrfs_ino(inode) + 1;
10747	xa_unlock(&root->inodes);
10748	cond_resched();
10749	xa_lock(&root->inodes);
10750	}
10751	xa_unlock(&root->inodes);
10752
10753	return inode;
10754	}
10755
10756	static const struct inode_operations btrfs_dir_inode_operations = {
10757	.getattr = btrfs_getattr,
10758	.lookup = btrfs_lookup,
10759	.create = btrfs_create,
10760	.unlink = btrfs_unlink,
10761	.link = btrfs_link,
10762	.mkdir = btrfs_mkdir,
10763	.rmdir = btrfs_rmdir,
10764	.rename = btrfs_rename2,
10765	.symlink = btrfs_symlink,
10766	.setattr = btrfs_setattr,
10767	.mknod = btrfs_mknod,
10768	.listxattr = btrfs_listxattr,
10769	.permission = btrfs_permission,
10770	.get_inode_acl = btrfs_get_acl,
10771	.set_acl = btrfs_set_acl,
10772	.update_time = btrfs_update_time,
10773	.tmpfile = btrfs_tmpfile,
10774	.fileattr_get = btrfs_fileattr_get,
10775	.fileattr_set = btrfs_fileattr_set,
10776	};
10777
10778	static const struct file_operations btrfs_dir_file_operations = {
10779	.llseek = btrfs_dir_llseek,
10780	.read = generic_read_dir,
10781	.iterate_shared = btrfs_real_readdir,
10782	.open = btrfs_opendir,
10783	.unlocked_ioctl = btrfs_ioctl,
10784	#ifdef CONFIG_COMPAT
10785	.compat_ioctl = btrfs_compat_ioctl,
10786	#endif
10787	.release = btrfs_release_file,
10788	.fsync = btrfs_sync_file,
10789	.setlease = generic_setlease,
10790	};
10791
10792	/*
10793	* The folio is going dirty without a btrfs delalloc space reservation.
10794	* This requires a fixup before writeback which we might sleep so cannot
10795	* run in this context, so we merely set state on the folio indicating it
10796	* needs fixup before writeback.
10797	*
10798	* Note that there is no range in the input, so the whole folio is marked
10799	* dirty and fixup.
10800	*
10801	* We believe that all callers of dirty_folio either:
10802	* - take the folio lock (e.g. pinned folio release notification).
10803	* - take the pte lock but must be running on a dirty pte which means
10804	* page_mkwrite() ran on it and reserved the space. zap_pte_range() cannot
10805	* race with writeback cleaning the folio because writeback runs
10806	* folio_mkclean() which also uses the pte lock and revokes outstanding
10807	* writable mappings.
10808	* Therefore, an additional folio private lock (a la bfs->lock for all cases,
10809	* not just subpage) is not necessary.
10810	*/
10811	static bool btrfs_data_dirty_folio(struct address_space *mapping,
10812	struct folio *folio)
10813	{
10814	struct btrfs_inode *inode = BTRFS_I(mapping->host);
10815	struct btrfs_fs_info *fs_info = inode->root->fs_info;
10816	const u64 page_start = folio_pos(folio);
10817	const u64 range_end = min_t(u64, folio_next_pos(folio),
10818	round_up(i_size_read(&inode->vfs_inode),
10819	fs_info->sectorsize));
10820
10821	if (range_end > page_start)
10822	btrfs_folio_set_fixup_dirty(fs_info, folio, page_start,
10823	range_end - page_start);
10824	return filemap_dirty_folio(mapping, folio);
10825	}
10826
10827	/*
10828	* btrfs doesn't support the bmap operation because swapfiles
10829	* use bmap to make a mapping of extents in the file. They assume
10830	* these extents won't change over the life of the file and they
10831	* use the bmap result to do IO directly to the drive.
10832	*
10833	* the btrfs bmap call would return logical addresses that aren't
10834	* suitable for IO and they also will change frequently as COW
10835	* operations happen. So, swapfile + btrfs == corruption.
10836	*
10837	* For now we're avoiding this by dropping bmap.
10838	*/
10839	static const struct address_space_operations btrfs_aops = {
10840	.read_folio = btrfs_read_folio,
10841	.writepages = btrfs_writepages,
10842	.readahead = btrfs_readahead,
10843	.invalidate_folio = btrfs_invalidate_folio,
10844	.launder_folio = btrfs_launder_folio,
10845	.release_folio = btrfs_release_folio,
10846	.migrate_folio = btrfs_migrate_folio,
10847	.dirty_folio = btrfs_data_dirty_folio,
10848	.error_remove_folio = generic_error_remove_folio,
10849	.swap_activate = btrfs_swap_activate,
10850	.swap_deactivate = btrfs_swap_deactivate,
10851	};
10852
10853	static const struct inode_operations btrfs_file_inode_operations = {
10854	.getattr = btrfs_getattr,
10855	.setattr = btrfs_setattr,
10856	.listxattr = btrfs_listxattr,
10857	.permission = btrfs_permission,
10858	.fiemap = btrfs_fiemap,
10859	.get_inode_acl = btrfs_get_acl,
10860	.set_acl = btrfs_set_acl,
10861	.update_time = btrfs_update_time,
10862	.fileattr_get = btrfs_fileattr_get,
10863	.fileattr_set = btrfs_fileattr_set,
10864	};
10865	static const struct inode_operations btrfs_special_inode_operations = {
10866	.getattr = btrfs_getattr,
10867	.setattr = btrfs_setattr,
10868	.permission = btrfs_permission,
10869	.listxattr = btrfs_listxattr,
10870	.get_inode_acl = btrfs_get_acl,
10871	.set_acl = btrfs_set_acl,
10872	.update_time = btrfs_update_time,
10873	};
10874	static const struct inode_operations btrfs_symlink_inode_operations = {
10875	.get_link = page_get_link,
10876	.getattr = btrfs_getattr,
10877	.setattr = btrfs_setattr,
10878	.permission = btrfs_permission,
10879	.listxattr = btrfs_listxattr,
10880	.update_time = btrfs_update_time,
10881	};
10882
10883	const struct dentry_operations btrfs_dentry_operations = {
10884	.d_delete = btrfs_dentry_delete,
10885	};