projects/linux·blob master · refs

8304 lines · 217.2 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2012 Alexander Block. All rights reserved.
4 */
5
6#include <linux/bsearch.h>
7#include <linux/falloc.h>
8#include <linux/fs.h>
9#include <linux/file.h>
10#include <linux/sort.h>
11#include <linux/mount.h>
12#include <linux/xattr.h>
13#include <linux/posix_acl_xattr.h>
14#include <linux/radix-tree.h>
15#include <linux/vmalloc.h>
16#include <linux/string.h>
17#include <linux/compat.h>
18#include <linux/crc32c.h>
19#include <linux/fsverity.h>
20#include <linux/cleanup.h>
21#include "send.h"
22#include "ctree.h"
23#include "backref.h"
24#include "locking.h"
25#include "disk-io.h"
26#include "btrfs_inode.h"
27#include "transaction.h"
28#include "compression.h"
29#include "print-tree.h"
30#include "accessors.h"
31#include "dir-item.h"
32#include "file-item.h"
33#include "ioctl.h"
34#include "verity.h"
35#include "lru_cache.h"
36
37/*
38 * Maximum number of references an extent can have in order for us to attempt to
39 * issue clone operations instead of write operations. This currently exists to
40 * avoid hitting limitations of the backreference walking code (taking a lot of
41 * time and using too much memory for extents with large number of references).
42 */
43#define SEND_MAX_EXTENT_REFS 1024
44
45/*
46 * A fs_path is a helper to dynamically build path names with unknown size.
47 * It reallocates the internal buffer on demand.
48 * It allows fast adding of path elements on the right side (normal path) and
49 * fast adding to the left side (reversed path). A reversed path can also be
50 * unreversed if needed.
51 *
52 * The definition of struct fs_path relies on -fms-extensions to allow
53 * including a tagged struct as an anonymous member.
54 */
55struct __fs_path {
56 char *start;
57 char *end;
58
59 char *buf;
60 unsigned short buf_len:15;
61 unsigned short reversed:1;
62};
63static_assert(sizeof(struct __fs_path) < 256);
64struct fs_path {
65 struct __fs_path;
66 /*
67 * Average path length does not exceed 200 bytes, we'll have
68 * better packing in the slab and higher chance to satisfy
69 * an allocation later during send.
70 */
71 char inline_buf[256 - sizeof(struct __fs_path)];
72};
73#define FS_PATH_INLINE_SIZE \
74 sizeof_field(struct fs_path, inline_buf)
75
76static void fs_path_free(struct fs_path *p);
77DEFINE_FREE(fs_path_free, struct fs_path *, fs_path_free(_T))
78
79/* reused for each extent */
80struct clone_root {
81 struct btrfs_root *root;
82 u64 ino;
83 u64 offset;
84 u64 num_bytes;
85 bool found_ref;
86};
87
88#define SEND_MAX_NAME_CACHE_SIZE 256
89
90/*
91 * Limit the root_ids array of struct backref_cache_entry to 17 elements.
92 * This makes the size of a cache entry to be exactly 192 bytes on x86_64, which
93 * can be satisfied from the kmalloc-192 slab, without wasting any space.
94 * The most common case is to have a single root for cloning, which corresponds
95 * to the send root. Having the user specify more than 16 clone roots is not
96 * common, and in such rare cases we simply don't use caching if the number of
97 * cloning roots that lead down to a leaf is more than 17.
98 */
99#define SEND_MAX_BACKREF_CACHE_ROOTS 17
100
101/*
102 * Max number of entries in the cache.
103 * With SEND_MAX_BACKREF_CACHE_ROOTS as 17, the size in bytes, excluding
104 * maple tree's internal nodes, is 24K.
105 */
106#define SEND_MAX_BACKREF_CACHE_SIZE 128
107
108/*
109 * A backref cache entry maps a leaf to a list of IDs of roots from which the
110 * leaf is accessible and we can use for clone operations.
111 * With SEND_MAX_BACKREF_CACHE_ROOTS as 12, each cache entry is 128 bytes (on
112 * x86_64).
113 */
114struct backref_cache_entry {
115 struct btrfs_lru_cache_entry entry;
116 u64 root_ids[SEND_MAX_BACKREF_CACHE_ROOTS];
117 /* Number of valid elements in the root_ids array. */
118 int num_roots;
119};
120
121/* See the comment at lru_cache.h about struct btrfs_lru_cache_entry. */
122static_assert(offsetof(struct backref_cache_entry, entry) == 0);
123
124/*
125 * Max number of entries in the cache that stores directories that were already
126 * created. The cache uses raw struct btrfs_lru_cache_entry entries, so it uses
127 * at most 4096 bytes - sizeof(struct btrfs_lru_cache_entry) is 48 bytes, but
128 * the kmalloc-64 slab is used, so we get 4096 bytes (64 bytes * 64).
129 */
130#define SEND_MAX_DIR_CREATED_CACHE_SIZE 64
131
132/*
133 * Maximum number of entries in the cache that stores utimes values for directories.
134 * The cache uses raw struct btrfs_lru_cache_entry entries, so it uses at most
135 * 4096 bytes - sizeof(struct btrfs_lru_cache_entry) is 48 bytes, but the
136 * kmalloc-64 slab is used, so we get 4096 bytes (64 bytes * 64).
137 */
138#define SEND_MAX_DIR_UTIMES_CACHE_SIZE 64
139
140struct send_ctx {
141 struct file *send_filp;
142 loff_t send_off;
143 char *send_buf;
144 u32 send_size;
145 u32 send_max_size;
146 /*
147 * Whether BTRFS_SEND_A_DATA attribute was already added to current
148 * command (since protocol v2, data must be the last attribute).
149 */
150 bool put_data;
151 struct page **send_buf_pages;
152 u64 flags; /* 'flags' member of btrfs_ioctl_send_args is u64 */
153 /* Protocol version compatibility requested */
154 u32 proto;
155
156 struct btrfs_root *send_root;
157 struct btrfs_root *parent_root;
158 struct clone_root *clone_roots;
159 int clone_roots_cnt;
160
161 /* current state of the compare_tree call */
162 struct btrfs_path *left_path;
163 struct btrfs_path *right_path;
164 struct btrfs_key *cmp_key;
165
166 /*
167 * Keep track of the generation of the last transaction that was used
168 * for relocating a block group. This is periodically checked in order
169 * to detect if a relocation happened since the last check, so that we
170 * don't operate on stale extent buffers for nodes (level >= 1) or on
171 * stale disk_bytenr values of file extent items.
172 */
173 u64 last_reloc_trans;
174
175 /*
176 * infos of the currently processed inode. In case of deleted inodes,
177 * these are the values from the deleted inode.
178 */
179 u64 cur_ino;
180 u64 cur_inode_gen;
181 u64 cur_inode_size;
182 u64 cur_inode_mode;
183 u64 cur_inode_rdev;
184 u64 cur_inode_last_extent;
185 u64 cur_inode_next_write_offset;
186 bool cur_inode_new;
187 bool cur_inode_new_gen;
188 bool cur_inode_deleted;
189 bool ignore_cur_inode;
190 bool cur_inode_needs_verity;
191 void *verity_descriptor;
192
193 u64 send_progress;
194
195 struct list_head new_refs;
196 struct list_head deleted_refs;
197
198 struct btrfs_lru_cache name_cache;
199
200 /*
201 * The inode we are currently processing. It's not NULL only when we
202 * need to issue write commands for data extents from this inode.
203 */
204 struct inode *cur_inode;
205 struct file_ra_state ra;
206 u64 page_cache_clear_start;
207 bool clean_page_cache;
208
209 /*
210 * We process inodes by their increasing order, so if before an
211 * incremental send we reverse the parent/child relationship of
212 * directories such that a directory with a lower inode number was
213 * the parent of a directory with a higher inode number, and the one
214 * becoming the new parent got renamed too, we can't rename/move the
215 * directory with lower inode number when we finish processing it - we
216 * must process the directory with higher inode number first, then
217 * rename/move it and then rename/move the directory with lower inode
218 * number. Example follows.
219 *
220 * Tree state when the first send was performed:
221 *
222 * .
223 * |-- a (ino 257)
224 * |-- b (ino 258)
225 * |
226 * |
227 * |-- c (ino 259)
228 * | |-- d (ino 260)
229 * |
230 * |-- c2 (ino 261)
231 *
232 * Tree state when the second (incremental) send is performed:
233 *
234 * .
235 * |-- a (ino 257)
236 * |-- b (ino 258)
237 * |-- c2 (ino 261)
238 * |-- d2 (ino 260)
239 * |-- cc (ino 259)
240 *
241 * The sequence of steps that lead to the second state was:
242 *
243 * mv /a/b/c/d /a/b/c2/d2
244 * mv /a/b/c /a/b/c2/d2/cc
245 *
246 * "c" has lower inode number, but we can't move it (2nd mv operation)
247 * before we move "d", which has higher inode number.
248 *
249 * So we just memorize which move/rename operations must be performed
250 * later when their respective parent is processed and moved/renamed.
251 */
252
253 /* Indexed by parent directory inode number. */
254 struct rb_root pending_dir_moves;
255
256 /*
257 * Reverse index, indexed by the inode number of a directory that
258 * is waiting for the move/rename of its immediate parent before its
259 * own move/rename can be performed.
260 */
261 struct rb_root waiting_dir_moves;
262
263 /*
264 * A directory that is going to be rm'ed might have a child directory
265 * which is in the pending directory moves index above. In this case,
266 * the directory can only be removed after the move/rename of its child
267 * is performed. Example:
268 *
269 * Parent snapshot:
270 *
271 * . (ino 256)
272 * |-- a/ (ino 257)
273 * |-- b/ (ino 258)
274 * |-- c/ (ino 259)
275 * | |-- x/ (ino 260)
276 * |
277 * |-- y/ (ino 261)
278 *
279 * Send snapshot:
280 *
281 * . (ino 256)
282 * |-- a/ (ino 257)
283 * |-- b/ (ino 258)
284 * |-- YY/ (ino 261)
285 * |-- x/ (ino 260)
286 *
287 * Sequence of steps that lead to the send snapshot:
288 * rm -f /a/b/c/foo.txt
289 * mv /a/b/y /a/b/YY
290 * mv /a/b/c/x /a/b/YY
291 * rmdir /a/b/c
292 *
293 * When the child is processed, its move/rename is delayed until its
294 * parent is processed (as explained above), but all other operations
295 * like update utimes, chown, chgrp, etc, are performed and the paths
296 * that it uses for those operations must use the orphanized name of
297 * its parent (the directory we're going to rm later), so we need to
298 * memorize that name.
299 *
300 * Indexed by the inode number of the directory to be deleted.
301 */
302 struct rb_root orphan_dirs;
303
304 struct rb_root rbtree_new_refs;
305 struct rb_root rbtree_deleted_refs;
306
307 struct btrfs_lru_cache backref_cache;
308 u64 backref_cache_last_reloc_trans;
309
310 struct btrfs_lru_cache dir_created_cache;
311 struct btrfs_lru_cache dir_utimes_cache;
312
313 struct fs_path cur_inode_path;
314};
315
316struct pending_dir_move {
317 struct rb_node node;
318 struct list_head list;
319 u64 parent_ino;
320 u64 ino;
321 u64 gen;
322 struct list_head update_refs;
323};
324
325struct waiting_dir_move {
326 struct rb_node node;
327 u64 ino;
328 /*
329 * There might be some directory that could not be removed because it
330 * was waiting for this directory inode to be moved first. Therefore
331 * after this directory is moved, we can try to rmdir the ino rmdir_ino.
332 */
333 u64 rmdir_ino;
334 u64 rmdir_gen;
335 bool orphanized;
336};
337
338struct orphan_dir_info {
339 struct rb_node node;
340 u64 ino;
341 u64 gen;
342 u64 last_dir_index_offset;
343 u64 dir_high_seq_ino;
344};
345
346struct name_cache_entry {
347 /*
348 * The key in the entry is an inode number, and the generation matches
349 * the inode's generation.
350 */
351 struct btrfs_lru_cache_entry entry;
352 u64 parent_ino;
353 u64 parent_gen;
354 int ret;
355 int need_later_update;
356 /* Name length without NUL terminator. */
357 int name_len;
358 /* Not NUL terminated. */
359 char name[] __counted_by(name_len) __nonstring;
360};
361
362/* See the comment at lru_cache.h about struct btrfs_lru_cache_entry. */
363static_assert(offsetof(struct name_cache_entry, entry) == 0);
364
365#define ADVANCE 1
366#define ADVANCE_ONLY_NEXT -1
367
368enum btrfs_compare_tree_result {
369 BTRFS_COMPARE_TREE_NEW,
370 BTRFS_COMPARE_TREE_DELETED,
371 BTRFS_COMPARE_TREE_CHANGED,
372 BTRFS_COMPARE_TREE_SAME,
373};
374
375__cold
376static void inconsistent_snapshot_error(struct send_ctx *sctx,
377 enum btrfs_compare_tree_result result,
378 const char *what)
379{
380 const char *result_string;
381
382 switch (result) {
383 case BTRFS_COMPARE_TREE_NEW:
384 result_string = "new";
385 break;
386 case BTRFS_COMPARE_TREE_DELETED:
387 result_string = "deleted";
388 break;
389 case BTRFS_COMPARE_TREE_CHANGED:
390 result_string = "updated";
391 break;
392 case BTRFS_COMPARE_TREE_SAME:
393 DEBUG_WARN("no change between trees");
394 result_string = "unchanged";
395 break;
396 default:
397 DEBUG_WARN("unexpected comparison result %d", result);
398 result_string = "unexpected";
399 }
400
401 btrfs_err(sctx->send_root->fs_info,
402 "Send: inconsistent snapshot, found %s %s for inode %llu without updated inode item, send root is %llu, parent root is %llu",
403 result_string, what, sctx->cmp_key->objectid,
404 btrfs_root_id(sctx->send_root),
405 (sctx->parent_root ? btrfs_root_id(sctx->parent_root) : 0));
406}
407
408__maybe_unused
409static bool proto_cmd_ok(const struct send_ctx *sctx, int cmd)
410{
411 switch (sctx->proto) {
412 case 1: return cmd <= BTRFS_SEND_C_MAX_V1;
413 case 2: return cmd <= BTRFS_SEND_C_MAX_V2;
414 case 3: return cmd <= BTRFS_SEND_C_MAX_V3;
415 default: return false;
416 }
417}
418
419static int is_waiting_for_move(struct send_ctx *sctx, u64 ino);
420
421static struct waiting_dir_move *
422get_waiting_dir_move(struct send_ctx *sctx, u64 ino);
423
424static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino, u64 gen);
425
426static int need_send_hole(struct send_ctx *sctx)
427{
428 return (sctx->parent_root && !sctx->cur_inode_new &&
429 !sctx->cur_inode_new_gen && !sctx->cur_inode_deleted &&
430 S_ISREG(sctx->cur_inode_mode));
431}
432
433static void fs_path_reset(struct fs_path *p)
434{
435 if (p->reversed)
436 p->start = p->buf + p->buf_len - 1;
437 else
438 p->start = p->buf;
439
440 p->end = p->start;
441 *p->start = 0;
442}
443
444static void init_path(struct fs_path *p)
445{
446 p->reversed = 0;
447 p->buf = p->inline_buf;
448 p->buf_len = FS_PATH_INLINE_SIZE;
449 fs_path_reset(p);
450}
451
452static struct fs_path *fs_path_alloc(void)
453{
454 struct fs_path *p;
455
456 p = kmalloc_obj(*p);
457 if (!p)
458 return NULL;
459 init_path(p);
460 return p;
461}
462
463static struct fs_path *fs_path_alloc_reversed(void)
464{
465 struct fs_path *p;
466
467 p = fs_path_alloc();
468 if (!p)
469 return NULL;
470 p->reversed = 1;
471 fs_path_reset(p);
472 return p;
473}
474
475static void fs_path_free(struct fs_path *p)
476{
477 if (!p)
478 return;
479 if (p->buf != p->inline_buf)
480 kfree(p->buf);
481 kfree(p);
482}
483
484static inline int fs_path_len(const struct fs_path *p)
485{
486 return p->end - p->start;
487}
488
489static int fs_path_ensure_buf(struct fs_path *p, int len)
490{
491 char *tmp_buf;
492 int path_len;
493 int old_buf_len;
494
495 len++;
496
497 if (p->buf_len >= len)
498 return 0;
499
500 if (WARN_ON(len > PATH_MAX))
501 return -ENAMETOOLONG;
502
503 path_len = fs_path_len(p);
504 old_buf_len = p->buf_len;
505
506 /*
507 * Allocate to the next largest kmalloc bucket size, to let
508 * the fast path happen most of the time.
509 */
510 len = kmalloc_size_roundup(len);
511 /*
512 * First time the inline_buf does not suffice
513 */
514 if (p->buf == p->inline_buf) {
515 tmp_buf = kmalloc(len, GFP_KERNEL);
516 if (tmp_buf)
517 memcpy(tmp_buf, p->buf, old_buf_len);
518 } else {
519 tmp_buf = krealloc(p->buf, len, GFP_KERNEL);
520 }
521 if (!tmp_buf)
522 return -ENOMEM;
523 p->buf = tmp_buf;
524 p->buf_len = len;
525
526 if (p->reversed) {
527 tmp_buf = p->buf + old_buf_len - path_len - 1;
528 p->end = p->buf + p->buf_len - 1;
529 p->start = p->end - path_len;
530 memmove(p->start, tmp_buf, path_len + 1);
531 } else {
532 p->start = p->buf;
533 p->end = p->start + path_len;
534 }
535 return 0;
536}
537
538static int fs_path_prepare_for_add(struct fs_path *p, int name_len,
539 char **prepared)
540{
541 int ret;
542 int new_len;
543
544 new_len = fs_path_len(p) + name_len;
545 if (p->start != p->end)
546 new_len++;
547 ret = fs_path_ensure_buf(p, new_len);
548 if (ret < 0)
549 return ret;
550
551 if (p->reversed) {
552 if (p->start != p->end)
553 *--p->start = '/';
554 p->start -= name_len;
555 *prepared = p->start;
556 } else {
557 if (p->start != p->end)
558 *p->end++ = '/';
559 *prepared = p->end;
560 p->end += name_len;
561 *p->end = 0;
562 }
563
564 return 0;
565}
566
567static int fs_path_add(struct fs_path *p, const char *name, int name_len)
568{
569 int ret;
570 char *prepared;
571
572 ret = fs_path_prepare_for_add(p, name_len, &prepared);
573 if (ret < 0)
574 return ret;
575 memcpy(prepared, name, name_len);
576
577 return 0;
578}
579
580static inline int fs_path_add_path(struct fs_path *p, const struct fs_path *p2)
581{
582 return fs_path_add(p, p2->start, fs_path_len(p2));
583}
584
585static int fs_path_add_from_extent_buffer(struct fs_path *p,
586 struct extent_buffer *eb,
587 unsigned long off, int len)
588{
589 int ret;
590 char *prepared;
591
592 ret = fs_path_prepare_for_add(p, len, &prepared);
593 if (ret < 0)
594 return ret;
595
596 read_extent_buffer(eb, prepared, off, len);
597
598 return 0;
599}
600
601static int fs_path_copy(struct fs_path *p, struct fs_path *from)
602{
603 p->reversed = from->reversed;
604 fs_path_reset(p);
605
606 return fs_path_add_path(p, from);
607}
608
609static void fs_path_unreverse(struct fs_path *p)
610{
611 char *tmp;
612 int len;
613
614 if (!p->reversed)
615 return;
616
617 tmp = p->start;
618 len = fs_path_len(p);
619 p->start = p->buf;
620 p->end = p->start + len;
621 memmove(p->start, tmp, len + 1);
622 p->reversed = 0;
623}
624
625static inline bool is_current_inode_path(const struct send_ctx *sctx,
626 const struct fs_path *path)
627{
628 /* Paths are always nul terminated. */
629 return (strcmp(path->start, sctx->cur_inode_path.start) == 0);
630}
631
632static struct btrfs_path *alloc_path_for_send(void)
633{
634 struct btrfs_path *path;
635
636 path = btrfs_alloc_path();
637 if (!path)
638 return NULL;
639 path->search_commit_root = true;
640 path->skip_locking = true;
641 path->need_commit_sem = true;
642 return path;
643}
644
645static int write_buf(struct file *filp, const void *buf, u32 len, loff_t *off)
646{
647 int ret;
648 u32 pos = 0;
649
650 while (pos < len) {
651 ret = kernel_write(filp, buf + pos, len - pos, off);
652 if (ret < 0)
653 return ret;
654 if (unlikely(ret == 0))
655 return -EIO;
656 pos += ret;
657 }
658
659 return 0;
660}
661
662static int tlv_put(struct send_ctx *sctx, u16 attr, const void *data, int len)
663{
664 struct btrfs_tlv_header *hdr;
665 int total_len = sizeof(*hdr) + len;
666 int left = sctx->send_max_size - sctx->send_size;
667
668 if (WARN_ON_ONCE(sctx->put_data))
669 return -EINVAL;
670
671 if (unlikely(left < total_len))
672 return -EOVERFLOW;
673
674 hdr = (struct btrfs_tlv_header *) (sctx->send_buf + sctx->send_size);
675 put_unaligned_le16(attr, &hdr->tlv_type);
676 put_unaligned_le16(len, &hdr->tlv_len);
677 memcpy(hdr + 1, data, len);
678 sctx->send_size += total_len;
679
680 return 0;
681}
682
683#define TLV_PUT_DEFINE_INT(bits) \
684 static int tlv_put_u##bits(struct send_ctx *sctx, \
685 u##bits attr, u##bits value) \
686 { \
687 __le##bits __tmp = cpu_to_le##bits(value); \
688 return tlv_put(sctx, attr, &__tmp, sizeof(__tmp)); \
689 }
690
691TLV_PUT_DEFINE_INT(8)
692TLV_PUT_DEFINE_INT(32)
693TLV_PUT_DEFINE_INT(64)
694
695static int tlv_put_string(struct send_ctx *sctx, u16 attr,
696 const char *str, int len)
697{
698 if (len == -1)
699 len = strlen(str);
700 return tlv_put(sctx, attr, str, len);
701}
702
703static int tlv_put_uuid(struct send_ctx *sctx, u16 attr,
704 const u8 *uuid)
705{
706 return tlv_put(sctx, attr, uuid, BTRFS_UUID_SIZE);
707}
708
709static int tlv_put_btrfs_timespec(struct send_ctx *sctx, u16 attr,
710 struct extent_buffer *eb,
711 struct btrfs_timespec *ts)
712{
713 struct btrfs_timespec bts;
714 read_extent_buffer(eb, &bts, (unsigned long)ts, sizeof(bts));
715 return tlv_put(sctx, attr, &bts, sizeof(bts));
716}
717
718
719#define TLV_PUT(sctx, attrtype, data, attrlen) \
720 do { \
721 ret = tlv_put(sctx, attrtype, data, attrlen); \
722 if (ret < 0) \
723 goto tlv_put_failure; \
724 } while (0)
725
726#define TLV_PUT_INT(sctx, attrtype, bits, value) \
727 do { \
728 ret = tlv_put_u##bits(sctx, attrtype, value); \
729 if (ret < 0) \
730 goto tlv_put_failure; \
731 } while (0)
732
733#define TLV_PUT_U8(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 8, data)
734#define TLV_PUT_U16(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 16, data)
735#define TLV_PUT_U32(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 32, data)
736#define TLV_PUT_U64(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 64, data)
737#define TLV_PUT_STRING(sctx, attrtype, str, len) \
738 do { \
739 ret = tlv_put_string(sctx, attrtype, str, len); \
740 if (ret < 0) \
741 goto tlv_put_failure; \
742 } while (0)
743#define TLV_PUT_PATH(sctx, attrtype, p) \
744 do { \
745 ret = tlv_put_string(sctx, attrtype, p->start, \
746 fs_path_len((p))); \
747 if (ret < 0) \
748 goto tlv_put_failure; \
749 } while(0)
750#define TLV_PUT_UUID(sctx, attrtype, uuid) \
751 do { \
752 ret = tlv_put_uuid(sctx, attrtype, uuid); \
753 if (ret < 0) \
754 goto tlv_put_failure; \
755 } while (0)
756#define TLV_PUT_BTRFS_TIMESPEC(sctx, attrtype, eb, ts) \
757 do { \
758 ret = tlv_put_btrfs_timespec(sctx, attrtype, eb, ts); \
759 if (ret < 0) \
760 goto tlv_put_failure; \
761 } while (0)
762
763static int send_header(struct send_ctx *sctx)
764{
765 struct btrfs_stream_header hdr;
766
767 strscpy(hdr.magic, BTRFS_SEND_STREAM_MAGIC);
768 hdr.version = cpu_to_le32(sctx->proto);
769 return write_buf(sctx->send_filp, &hdr, sizeof(hdr),
770 &sctx->send_off);
771}
772
773/*
774 * For each command/item we want to send to userspace, we call this function.
775 */
776static int begin_cmd(struct send_ctx *sctx, int cmd)
777{
778 struct btrfs_cmd_header *hdr;
779
780 if (WARN_ON(!sctx->send_buf))
781 return -EINVAL;
782
783 if (unlikely(sctx->send_size != 0)) {
784 btrfs_err(sctx->send_root->fs_info,
785 "send: command header buffer not empty cmd %d offset %llu",
786 cmd, sctx->send_off);
787 return -EINVAL;
788 }
789
790 sctx->send_size += sizeof(*hdr);
791 hdr = (struct btrfs_cmd_header *)sctx->send_buf;
792 put_unaligned_le16(cmd, &hdr->cmd);
793
794 return 0;
795}
796
797static int send_cmd(struct send_ctx *sctx)
798{
799 int ret;
800 struct btrfs_cmd_header *hdr;
801 u32 crc;
802
803 hdr = (struct btrfs_cmd_header *)sctx->send_buf;
804 put_unaligned_le32(sctx->send_size - sizeof(*hdr), &hdr->len);
805 put_unaligned_le32(0, &hdr->crc);
806
807 crc = crc32c(0, (unsigned char *)sctx->send_buf, sctx->send_size);
808 put_unaligned_le32(crc, &hdr->crc);
809
810 ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size,
811 &sctx->send_off);
812
813 sctx->send_size = 0;
814 sctx->put_data = false;
815
816 return ret;
817}
818
819/*
820 * Sends a move instruction to user space
821 */
822static int send_rename(struct send_ctx *sctx,
823 struct fs_path *from, struct fs_path *to)
824{
825 int ret;
826
827 ret = begin_cmd(sctx, BTRFS_SEND_C_RENAME);
828 if (ret < 0)
829 return ret;
830
831 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, from);
832 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_TO, to);
833
834 ret = send_cmd(sctx);
835
836tlv_put_failure:
837 return ret;
838}
839
840/*
841 * Sends a link instruction to user space
842 */
843static int send_link(struct send_ctx *sctx,
844 struct fs_path *path, struct fs_path *lnk)
845{
846 int ret;
847
848 ret = begin_cmd(sctx, BTRFS_SEND_C_LINK);
849 if (ret < 0)
850 return ret;
851
852 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
853 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_LINK, lnk);
854
855 ret = send_cmd(sctx);
856
857tlv_put_failure:
858 return ret;
859}
860
861/*
862 * Sends an unlink instruction to user space
863 */
864static int send_unlink(struct send_ctx *sctx, struct fs_path *path)
865{
866 int ret;
867
868 ret = begin_cmd(sctx, BTRFS_SEND_C_UNLINK);
869 if (ret < 0)
870 return ret;
871
872 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
873
874 ret = send_cmd(sctx);
875
876tlv_put_failure:
877 return ret;
878}
879
880/*
881 * Sends a rmdir instruction to user space
882 */
883static int send_rmdir(struct send_ctx *sctx, struct fs_path *path)
884{
885 int ret;
886
887 ret = begin_cmd(sctx, BTRFS_SEND_C_RMDIR);
888 if (ret < 0)
889 return ret;
890
891 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
892
893 ret = send_cmd(sctx);
894
895tlv_put_failure:
896 return ret;
897}
898
899struct btrfs_inode_info {
900 u64 size;
901 u64 gen;
902 u64 mode;
903 u64 uid;
904 u64 gid;
905 u64 rdev;
906 u64 fileattr;
907 u64 nlink;
908};
909
910/*
911 * Helper function to retrieve some fields from an inode item.
912 */
913static int get_inode_info(struct btrfs_root *root, u64 ino,
914 struct btrfs_inode_info *info)
915{
916 int ret;
917 BTRFS_PATH_AUTO_FREE(path);
918 struct btrfs_inode_item *ii;
919 struct btrfs_key key;
920
921 path = alloc_path_for_send();
922 if (!path)
923 return -ENOMEM;
924
925 key.objectid = ino;
926 key.type = BTRFS_INODE_ITEM_KEY;
927 key.offset = 0;
928 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
929 if (ret) {
930 if (ret > 0)
931 ret = -ENOENT;
932 return ret;
933 }
934
935 if (!info)
936 return 0;
937
938 ii = btrfs_item_ptr(path->nodes[0], path->slots[0],
939 struct btrfs_inode_item);
940 info->size = btrfs_inode_size(path->nodes[0], ii);
941 info->gen = btrfs_inode_generation(path->nodes[0], ii);
942 info->mode = btrfs_inode_mode(path->nodes[0], ii);
943 info->uid = btrfs_inode_uid(path->nodes[0], ii);
944 info->gid = btrfs_inode_gid(path->nodes[0], ii);
945 info->rdev = btrfs_inode_rdev(path->nodes[0], ii);
946 info->nlink = btrfs_inode_nlink(path->nodes[0], ii);
947 /*
948 * Transfer the unchanged u64 value of btrfs_inode_item::flags, that's
949 * otherwise logically split to 32/32 parts.
950 */
951 info->fileattr = btrfs_inode_flags(path->nodes[0], ii);
952
953 return 0;
954}
955
956static int get_inode_gen(struct btrfs_root *root, u64 ino, u64 *gen)
957{
958 int ret;
959 struct btrfs_inode_info info = { 0 };
960
961 ASSERT(gen);
962
963 ret = get_inode_info(root, ino, &info);
964 *gen = info.gen;
965 return ret;
966}
967
968typedef int (*iterate_inode_ref_t)(u64 dir, struct fs_path *p, void *ctx);
969
970/*
971 * Helper function to iterate the entries in ONE btrfs_inode_ref or
972 * btrfs_inode_extref.
973 * The iterate callback may return a non zero value to stop iteration. This can
974 * be a negative value for error codes or 1 to simply stop it.
975 *
976 * path must point to the INODE_REF or INODE_EXTREF when called.
977 */
978static int iterate_inode_ref(struct btrfs_root *root, struct btrfs_path *path,
979 struct btrfs_key *found_key, bool resolve,
980 iterate_inode_ref_t iterate, void *ctx)
981{
982 struct extent_buffer *eb = path->nodes[0];
983 struct btrfs_inode_ref *iref;
984 struct btrfs_inode_extref *extref;
985 BTRFS_PATH_AUTO_FREE(tmp_path);
986 struct fs_path *p __free(fs_path_free) = NULL;
987 u32 cur = 0;
988 u32 total;
989 int slot = path->slots[0];
990 u32 name_len;
991 char *start;
992 int ret = 0;
993 u64 dir;
994 unsigned long name_off;
995 unsigned long elem_size;
996 unsigned long ptr;
997
998 p = fs_path_alloc_reversed();
999 if (!p)
1000 return -ENOMEM;
1001
1002 tmp_path = alloc_path_for_send();
1003 if (!tmp_path)
1004 return -ENOMEM;
1005
1006 if (found_key->type == BTRFS_INODE_REF_KEY) {
1007 ptr = (unsigned long)btrfs_item_ptr(eb, slot,
1008 struct btrfs_inode_ref);
1009 total = btrfs_item_size(eb, slot);
1010 elem_size = sizeof(*iref);
1011 } else {
1012 ptr = btrfs_item_ptr_offset(eb, slot);
1013 total = btrfs_item_size(eb, slot);
1014 elem_size = sizeof(*extref);
1015 }
1016
1017 while (cur < total) {
1018 fs_path_reset(p);
1019
1020 if (found_key->type == BTRFS_INODE_REF_KEY) {
1021 iref = (struct btrfs_inode_ref *)(ptr + cur);
1022 name_len = btrfs_inode_ref_name_len(eb, iref);
1023 name_off = (unsigned long)(iref + 1);
1024 dir = found_key->offset;
1025 } else {
1026 extref = (struct btrfs_inode_extref *)(ptr + cur);
1027 name_len = btrfs_inode_extref_name_len(eb, extref);
1028 name_off = (unsigned long)&extref->name;
1029 dir = btrfs_inode_extref_parent(eb, extref);
1030 }
1031
1032 if (resolve) {
1033 start = btrfs_ref_to_path(root, tmp_path, name_len,
1034 name_off, eb, dir,
1035 p->buf, p->buf_len);
1036 if (IS_ERR(start))
1037 return PTR_ERR(start);
1038
1039 if (start < p->buf) {
1040 /* overflow , try again with larger buffer */
1041 ret = fs_path_ensure_buf(p,
1042 p->buf_len + p->buf - start);
1043 if (ret < 0)
1044 return ret;
1045 start = btrfs_ref_to_path(root, tmp_path,
1046 name_len, name_off,
1047 eb, dir,
1048 p->buf, p->buf_len);
1049 if (IS_ERR(start))
1050 return PTR_ERR(start);
1051
1052 if (unlikely(start < p->buf)) {
1053 btrfs_err(root->fs_info,
1054 "send: path ref buffer underflow for key " BTRFS_KEY_FMT,
1055 BTRFS_KEY_FMT_VALUE(found_key));
1056 return -EINVAL;
1057 }
1058 }
1059 p->start = start;
1060 } else {
1061 ret = fs_path_add_from_extent_buffer(p, eb, name_off,
1062 name_len);
1063 if (ret < 0)
1064 return ret;
1065 }
1066
1067 cur += elem_size + name_len;
1068 ret = iterate(dir, p, ctx);
1069 if (ret)
1070 return ret;
1071 }
1072
1073 return ret;
1074}
1075
1076typedef int (*iterate_dir_item_t)(int num, struct btrfs_key *di_key,
1077 const char *name, int name_len,
1078 const char *data, int data_len,
1079 void *ctx);
1080
1081/*
1082 * Helper function to iterate the entries in ONE btrfs_dir_item.
1083 * The iterate callback may return a non zero value to stop iteration. This can
1084 * be a negative value for error codes or 1 to simply stop it.
1085 *
1086 * path must point to the dir item when called.
1087 */
1088static int iterate_dir_item(struct btrfs_root *root, struct btrfs_path *path,
1089 iterate_dir_item_t iterate, void *ctx)
1090{
1091 int ret = 0;
1092 struct extent_buffer *eb;
1093 struct btrfs_dir_item *di;
1094 struct btrfs_key di_key;
1095 char *buf = NULL;
1096 int buf_len;
1097 u32 name_len;
1098 u32 data_len;
1099 u32 cur;
1100 u32 len;
1101 u32 total;
1102 int slot;
1103 int num;
1104
1105 /*
1106 * Start with a small buffer (1 page). If later we end up needing more
1107 * space, which can happen for xattrs on a fs with a leaf size greater
1108 * than the page size, attempt to increase the buffer. Typically xattr
1109 * values are small.
1110 */
1111 buf_len = PATH_MAX;
1112 buf = kmalloc(buf_len, GFP_KERNEL);
1113 if (!buf) {
1114 ret = -ENOMEM;
1115 goto out;
1116 }
1117
1118 eb = path->nodes[0];
1119 slot = path->slots[0];
1120 di = btrfs_item_ptr(eb, slot, struct btrfs_dir_item);
1121 cur = 0;
1122 len = 0;
1123 total = btrfs_item_size(eb, slot);
1124
1125 num = 0;
1126 while (cur < total) {
1127 name_len = btrfs_dir_name_len(eb, di);
1128 data_len = btrfs_dir_data_len(eb, di);
1129 btrfs_dir_item_key_to_cpu(eb, di, &di_key);
1130
1131 if (btrfs_dir_ftype(eb, di) == BTRFS_FT_XATTR) {
1132 if (unlikely(name_len > XATTR_NAME_MAX)) {
1133 ret = -ENAMETOOLONG;
1134 goto out;
1135 }
1136 if (unlikely(name_len + data_len >
1137 BTRFS_MAX_XATTR_SIZE(root->fs_info))) {
1138 ret = -E2BIG;
1139 goto out;
1140 }
1141 } else {
1142 /*
1143 * Path too long
1144 */
1145 if (unlikely(name_len + data_len > PATH_MAX)) {
1146 ret = -ENAMETOOLONG;
1147 goto out;
1148 }
1149 }
1150
1151 if (name_len + data_len > buf_len) {
1152 buf_len = name_len + data_len;
1153 if (is_vmalloc_addr(buf)) {
1154 vfree(buf);
1155 buf = NULL;
1156 } else {
1157 char *tmp = krealloc(buf, buf_len,
1158 GFP_KERNEL | __GFP_NOWARN);
1159
1160 if (!tmp)
1161 kfree(buf);
1162 buf = tmp;
1163 }
1164 if (!buf) {
1165 buf = kvmalloc(buf_len, GFP_KERNEL);
1166 if (!buf) {
1167 ret = -ENOMEM;
1168 goto out;
1169 }
1170 }
1171 }
1172
1173 read_extent_buffer(eb, buf, (unsigned long)(di + 1),
1174 name_len + data_len);
1175
1176 len = sizeof(*di) + name_len + data_len;
1177 di = (struct btrfs_dir_item *)((char *)di + len);
1178 cur += len;
1179
1180 ret = iterate(num, &di_key, buf, name_len, buf + name_len,
1181 data_len, ctx);
1182 if (ret < 0)
1183 goto out;
1184 if (ret) {
1185 ret = 0;
1186 goto out;
1187 }
1188
1189 num++;
1190 }
1191
1192out:
1193 kvfree(buf);
1194 return ret;
1195}
1196
1197static int __copy_first_ref(u64 dir, struct fs_path *p, void *ctx)
1198{
1199 int ret;
1200 struct fs_path *pt = ctx;
1201
1202 ret = fs_path_copy(pt, p);
1203 if (ret < 0)
1204 return ret;
1205
1206 /* we want the first only */
1207 return 1;
1208}
1209
1210/*
1211 * Retrieve the first path of an inode. If an inode has more then one
1212 * ref/hardlink, this is ignored.
1213 */
1214static int get_inode_path(struct btrfs_root *root,
1215 u64 ino, struct fs_path *path)
1216{
1217 int ret;
1218 struct btrfs_key key, found_key;
1219 BTRFS_PATH_AUTO_FREE(p);
1220
1221 p = alloc_path_for_send();
1222 if (!p)
1223 return -ENOMEM;
1224
1225 fs_path_reset(path);
1226
1227 key.objectid = ino;
1228 key.type = BTRFS_INODE_REF_KEY;
1229 key.offset = 0;
1230
1231 ret = btrfs_search_slot_for_read(root, &key, p, 1, 0);
1232 if (ret < 0)
1233 return ret;
1234 if (ret)
1235 return 1;
1236
1237 btrfs_item_key_to_cpu(p->nodes[0], &found_key, p->slots[0]);
1238 if (found_key.objectid != ino ||
1239 (found_key.type != BTRFS_INODE_REF_KEY &&
1240 found_key.type != BTRFS_INODE_EXTREF_KEY))
1241 return -ENOENT;
1242
1243 ret = iterate_inode_ref(root, p, &found_key, true, __copy_first_ref, path);
1244 if (ret < 0)
1245 return ret;
1246 return 0;
1247}
1248
1249struct backref_ctx {
1250 struct send_ctx *sctx;
1251
1252 /* number of total found references */
1253 u64 found;
1254
1255 /*
1256 * used for clones found in send_root. clones found behind cur_objectid
1257 * and cur_offset are not considered as allowed clones.
1258 */
1259 u64 cur_objectid;
1260 u64 cur_offset;
1261
1262 /* may be truncated in case it's the last extent in a file */
1263 u64 extent_len;
1264
1265 /* The bytenr the file extent item we are processing refers to. */
1266 u64 bytenr;
1267 /* The owner (root id) of the data backref for the current extent. */
1268 u64 backref_owner;
1269 /* The offset of the data backref for the current extent. */
1270 u64 backref_offset;
1271};
1272
1273static int __clone_root_cmp_bsearch(const void *key, const void *elt)
1274{
1275 u64 root = (u64)(uintptr_t)key;
1276 const struct clone_root *cr = elt;
1277
1278 if (root < btrfs_root_id(cr->root))
1279 return -1;
1280 if (root > btrfs_root_id(cr->root))
1281 return 1;
1282 return 0;
1283}
1284
1285static int __clone_root_cmp_sort(const void *e1, const void *e2)
1286{
1287 const struct clone_root *cr1 = e1;
1288 const struct clone_root *cr2 = e2;
1289
1290 if (btrfs_root_id(cr1->root) < btrfs_root_id(cr2->root))
1291 return -1;
1292 if (btrfs_root_id(cr1->root) > btrfs_root_id(cr2->root))
1293 return 1;
1294 return 0;
1295}
1296
1297/*
1298 * Called for every backref that is found for the current extent.
1299 * Results are collected in sctx->clone_roots->ino/offset.
1300 */
1301static int iterate_backrefs(u64 ino, u64 offset, u64 num_bytes, u64 root_id,
1302 void *ctx_)
1303{
1304 struct backref_ctx *bctx = ctx_;
1305 struct clone_root *clone_root;
1306
1307 /* First check if the root is in the list of accepted clone sources */
1308 clone_root = bsearch((void *)(uintptr_t)root_id, bctx->sctx->clone_roots,
1309 bctx->sctx->clone_roots_cnt,
1310 sizeof(struct clone_root),
1311 __clone_root_cmp_bsearch);
1312 if (!clone_root)
1313 return 0;
1314
1315 /* This is our own reference, bail out as we can't clone from it. */
1316 if (clone_root->root == bctx->sctx->send_root &&
1317 ino == bctx->cur_objectid &&
1318 offset == bctx->cur_offset)
1319 return 0;
1320
1321 /*
1322 * Make sure we don't consider clones from send_root that are
1323 * behind the current inode/offset.
1324 */
1325 if (clone_root->root == bctx->sctx->send_root) {
1326 /*
1327 * If the source inode was not yet processed we can't issue a
1328 * clone operation, as the source extent does not exist yet at
1329 * the destination of the stream.
1330 */
1331 if (ino > bctx->cur_objectid)
1332 return 0;
1333 /*
1334 * We clone from the inode currently being sent as long as the
1335 * source extent is already processed, otherwise we could try
1336 * to clone from an extent that does not exist yet at the
1337 * destination of the stream.
1338 */
1339 if (ino == bctx->cur_objectid &&
1340 offset + bctx->extent_len >
1341 bctx->sctx->cur_inode_next_write_offset)
1342 return 0;
1343 }
1344
1345 bctx->found++;
1346 clone_root->found_ref = true;
1347
1348 /*
1349 * If the given backref refers to a file extent item with a larger
1350 * number of bytes than what we found before, use the new one so that
1351 * we clone more optimally and end up doing less writes and getting
1352 * less exclusive, non-shared extents at the destination.
1353 */
1354 if (num_bytes > clone_root->num_bytes) {
1355 clone_root->ino = ino;
1356 clone_root->offset = offset;
1357 clone_root->num_bytes = num_bytes;
1358
1359 /*
1360 * Found a perfect candidate, so there's no need to continue
1361 * backref walking.
1362 */
1363 if (num_bytes >= bctx->extent_len)
1364 return BTRFS_ITERATE_EXTENT_INODES_STOP;
1365 }
1366
1367 return 0;
1368}
1369
1370static bool lookup_backref_cache(u64 leaf_bytenr, void *ctx,
1371 const u64 **root_ids_ret, int *root_count_ret)
1372{
1373 struct backref_ctx *bctx = ctx;
1374 struct send_ctx *sctx = bctx->sctx;
1375 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
1376 const u64 key = leaf_bytenr >> fs_info->nodesize_bits;
1377 struct btrfs_lru_cache_entry *raw_entry;
1378 struct backref_cache_entry *entry;
1379
1380 if (sctx->backref_cache.size == 0)
1381 return false;
1382
1383 /*
1384 * If relocation happened since we first filled the cache, then we must
1385 * empty the cache and can not use it, because even though we operate on
1386 * read-only roots, their leaves and nodes may have been reallocated and
1387 * now be used for different nodes/leaves of the same tree or some other
1388 * tree.
1389 *
1390 * We are called from iterate_extent_inodes() while either holding a
1391 * transaction handle or holding fs_info->commit_root_sem, so no need
1392 * to take any lock here.
1393 */
1394 if (fs_info->last_reloc_trans > sctx->backref_cache_last_reloc_trans) {
1395 btrfs_lru_cache_clear(&sctx->backref_cache);
1396 return false;
1397 }
1398
1399 raw_entry = btrfs_lru_cache_lookup(&sctx->backref_cache, key, 0);
1400 if (!raw_entry)
1401 return false;
1402
1403 entry = container_of(raw_entry, struct backref_cache_entry, entry);
1404 *root_ids_ret = entry->root_ids;
1405 *root_count_ret = entry->num_roots;
1406
1407 return true;
1408}
1409
1410static void store_backref_cache(u64 leaf_bytenr, const struct ulist *root_ids,
1411 void *ctx)
1412{
1413 struct backref_ctx *bctx = ctx;
1414 struct send_ctx *sctx = bctx->sctx;
1415 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
1416 struct backref_cache_entry *new_entry;
1417 struct ulist_iterator uiter;
1418 struct ulist_node *node;
1419 int ret;
1420
1421 /*
1422 * We're called while holding a transaction handle or while holding
1423 * fs_info->commit_root_sem (at iterate_extent_inodes()), so must do a
1424 * NOFS allocation.
1425 */
1426 new_entry = kmalloc_obj(struct backref_cache_entry, GFP_NOFS);
1427 /* No worries, cache is optional. */
1428 if (!new_entry)
1429 return;
1430
1431 new_entry->entry.key = leaf_bytenr >> fs_info->nodesize_bits;
1432 new_entry->entry.gen = 0;
1433 new_entry->num_roots = 0;
1434 ULIST_ITER_INIT(&uiter);
1435 while ((node = ulist_next(root_ids, &uiter)) != NULL) {
1436 const u64 root_id = node->val;
1437 struct clone_root *root;
1438
1439 root = bsearch((void *)(uintptr_t)root_id, sctx->clone_roots,
1440 sctx->clone_roots_cnt, sizeof(struct clone_root),
1441 __clone_root_cmp_bsearch);
1442 if (!root)
1443 continue;
1444
1445 /* Too many roots, just exit, no worries as caching is optional. */
1446 if (new_entry->num_roots >= SEND_MAX_BACKREF_CACHE_ROOTS) {
1447 kfree(new_entry);
1448 return;
1449 }
1450
1451 new_entry->root_ids[new_entry->num_roots] = root_id;
1452 new_entry->num_roots++;
1453 }
1454
1455 /*
1456 * We may have not added any roots to the new cache entry, which means
1457 * none of the roots is part of the list of roots from which we are
1458 * allowed to clone. Cache the new entry as it's still useful to avoid
1459 * backref walking to determine which roots have a path to the leaf.
1460 *
1461 * Also use GFP_NOFS because we're called while holding a transaction
1462 * handle or while holding fs_info->commit_root_sem.
1463 */
1464 ret = btrfs_lru_cache_store(&sctx->backref_cache, &new_entry->entry,
1465 GFP_NOFS);
1466 ASSERT(ret == 0 || ret == -ENOMEM);
1467 if (ret) {
1468 /* Caching is optional, no worries. */
1469 kfree(new_entry);
1470 return;
1471 }
1472
1473 /*
1474 * We are called from iterate_extent_inodes() while either holding a
1475 * transaction handle or holding fs_info->commit_root_sem, so no need
1476 * to take any lock here.
1477 */
1478 if (sctx->backref_cache.size == 1)
1479 sctx->backref_cache_last_reloc_trans = fs_info->last_reloc_trans;
1480}
1481
1482static int check_extent_item(u64 bytenr, const struct btrfs_extent_item *ei,
1483 const struct extent_buffer *leaf, void *ctx)
1484{
1485 const u64 refs = btrfs_extent_refs(leaf, ei);
1486 const struct backref_ctx *bctx = ctx;
1487 const struct send_ctx *sctx = bctx->sctx;
1488
1489 if (bytenr == bctx->bytenr) {
1490 const u64 flags = btrfs_extent_flags(leaf, ei);
1491
1492 if (WARN_ON(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK))
1493 return -EUCLEAN;
1494
1495 /*
1496 * If we have only one reference and only the send root as a
1497 * clone source - meaning no clone roots were given in the
1498 * struct btrfs_ioctl_send_args passed to the send ioctl - then
1499 * it's our reference and there's no point in doing backref
1500 * walking which is expensive, so exit early.
1501 */
1502 if (refs == 1 && sctx->clone_roots_cnt == 1)
1503 return -ENOENT;
1504 }
1505
1506 /*
1507 * Backreference walking (iterate_extent_inodes() below) is currently
1508 * too expensive when an extent has a large number of references, both
1509 * in time spent and used memory. So for now just fallback to write
1510 * operations instead of clone operations when an extent has more than
1511 * a certain amount of references.
1512 */
1513 if (refs > SEND_MAX_EXTENT_REFS)
1514 return -ENOENT;
1515
1516 return 0;
1517}
1518
1519static bool skip_self_data_ref(u64 root, u64 ino, u64 offset, void *ctx)
1520{
1521 const struct backref_ctx *bctx = ctx;
1522
1523 if (ino == bctx->cur_objectid &&
1524 root == bctx->backref_owner &&
1525 offset == bctx->backref_offset)
1526 return true;
1527
1528 return false;
1529}
1530
1531/*
1532 * Given an inode, offset and extent item, it finds a good clone for a clone
1533 * instruction. Returns -ENOENT when none could be found. The function makes
1534 * sure that the returned clone is usable at the point where sending is at the
1535 * moment. This means, that no clones are accepted which lie behind the current
1536 * inode+offset.
1537 *
1538 * path must point to the extent item when called.
1539 */
1540static int find_extent_clone(struct send_ctx *sctx,
1541 struct btrfs_path *path,
1542 u64 ino, u64 data_offset,
1543 u64 ino_size,
1544 struct clone_root **found)
1545{
1546 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
1547 int ret;
1548 int extent_type;
1549 u64 disk_byte;
1550 u64 num_bytes;
1551 struct btrfs_file_extent_item *fi;
1552 struct extent_buffer *eb = path->nodes[0];
1553 struct backref_ctx backref_ctx = { 0 };
1554 struct btrfs_backref_walk_ctx backref_walk_ctx = { 0 };
1555 struct clone_root *cur_clone_root;
1556 int compressed;
1557 u32 i;
1558
1559 /*
1560 * With fallocate we can get prealloc extents beyond the inode's i_size,
1561 * so we don't do anything here because clone operations can not clone
1562 * to a range beyond i_size without increasing the i_size of the
1563 * destination inode.
1564 */
1565 if (data_offset >= ino_size)
1566 return 0;
1567
1568 fi = btrfs_item_ptr(eb, path->slots[0], struct btrfs_file_extent_item);
1569 extent_type = btrfs_file_extent_type(eb, fi);
1570 if (extent_type == BTRFS_FILE_EXTENT_INLINE)
1571 return -ENOENT;
1572
1573 disk_byte = btrfs_file_extent_disk_bytenr(eb, fi);
1574 if (disk_byte == 0)
1575 return -ENOENT;
1576
1577 compressed = btrfs_file_extent_compression(eb, fi);
1578 num_bytes = btrfs_file_extent_num_bytes(eb, fi);
1579
1580 /*
1581 * Setup the clone roots.
1582 */
1583 for (i = 0; i < sctx->clone_roots_cnt; i++) {
1584 cur_clone_root = sctx->clone_roots + i;
1585 cur_clone_root->ino = (u64)-1;
1586 cur_clone_root->offset = 0;
1587 cur_clone_root->num_bytes = 0;
1588 cur_clone_root->found_ref = false;
1589 }
1590
1591 backref_ctx.sctx = sctx;
1592 backref_ctx.cur_objectid = ino;
1593 backref_ctx.cur_offset = data_offset;
1594 backref_ctx.bytenr = disk_byte;
1595 /*
1596 * Use the header owner and not the send root's id, because in case of a
1597 * snapshot we can have shared subtrees.
1598 */
1599 backref_ctx.backref_owner = btrfs_header_owner(eb);
1600 backref_ctx.backref_offset = data_offset - btrfs_file_extent_offset(eb, fi);
1601
1602 /*
1603 * The last extent of a file may be too large due to page alignment.
1604 * We need to adjust extent_len in this case so that the checks in
1605 * iterate_backrefs() work.
1606 */
1607 if (data_offset + num_bytes >= ino_size)
1608 backref_ctx.extent_len = ino_size - data_offset;
1609 else
1610 backref_ctx.extent_len = num_bytes;
1611
1612 /*
1613 * Now collect all backrefs.
1614 */
1615 backref_walk_ctx.bytenr = disk_byte;
1616 if (compressed == BTRFS_COMPRESS_NONE)
1617 backref_walk_ctx.extent_item_pos = btrfs_file_extent_offset(eb, fi);
1618 backref_walk_ctx.fs_info = fs_info;
1619 backref_walk_ctx.cache_lookup = lookup_backref_cache;
1620 backref_walk_ctx.cache_store = store_backref_cache;
1621 backref_walk_ctx.indirect_ref_iterator = iterate_backrefs;
1622 backref_walk_ctx.check_extent_item = check_extent_item;
1623 backref_walk_ctx.user_ctx = &backref_ctx;
1624
1625 /*
1626 * If have a single clone root, then it's the send root and we can tell
1627 * the backref walking code to skip our own backref and not resolve it,
1628 * since we can not use it for cloning - the source and destination
1629 * ranges can't overlap and in case the leaf is shared through a subtree
1630 * due to snapshots, we can't use those other roots since they are not
1631 * in the list of clone roots.
1632 */
1633 if (sctx->clone_roots_cnt == 1)
1634 backref_walk_ctx.skip_data_ref = skip_self_data_ref;
1635
1636 ret = iterate_extent_inodes(&backref_walk_ctx, true, iterate_backrefs,
1637 &backref_ctx);
1638 if (ret < 0)
1639 return ret;
1640
1641 down_read(&fs_info->commit_root_sem);
1642 if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
1643 /*
1644 * A transaction commit for a transaction in which block group
1645 * relocation was done just happened.
1646 * The disk_bytenr of the file extent item we processed is
1647 * possibly stale, referring to the extent's location before
1648 * relocation. So act as if we haven't found any clone sources
1649 * and fallback to write commands, which will read the correct
1650 * data from the new extent location. Otherwise we will fail
1651 * below because we haven't found our own back reference or we
1652 * could be getting incorrect sources in case the old extent
1653 * was already reallocated after the relocation.
1654 */
1655 up_read(&fs_info->commit_root_sem);
1656 return -ENOENT;
1657 }
1658 up_read(&fs_info->commit_root_sem);
1659
1660 if (!backref_ctx.found)
1661 return -ENOENT;
1662
1663 cur_clone_root = NULL;
1664 for (i = 0; i < sctx->clone_roots_cnt; i++) {
1665 struct clone_root *clone_root = &sctx->clone_roots[i];
1666
1667 if (!clone_root->found_ref)
1668 continue;
1669
1670 /*
1671 * Choose the root from which we can clone more bytes, to
1672 * minimize write operations and therefore have more extent
1673 * sharing at the destination (the same as in the source).
1674 */
1675 if (!cur_clone_root ||
1676 clone_root->num_bytes > cur_clone_root->num_bytes) {
1677 cur_clone_root = clone_root;
1678
1679 /*
1680 * We found an optimal clone candidate (any inode from
1681 * any root is fine), so we're done.
1682 */
1683 if (clone_root->num_bytes >= backref_ctx.extent_len)
1684 break;
1685 }
1686 }
1687
1688 if (cur_clone_root) {
1689 *found = cur_clone_root;
1690 ret = 0;
1691 } else {
1692 ret = -ENOENT;
1693 }
1694
1695 return ret;
1696}
1697
1698static int read_symlink(struct btrfs_root *root,
1699 u64 ino,
1700 struct fs_path *dest)
1701{
1702 int ret;
1703 BTRFS_PATH_AUTO_FREE(path);
1704 struct btrfs_key key;
1705 struct btrfs_file_extent_item *ei;
1706 u8 type;
1707 u8 compression;
1708 unsigned long off;
1709 int len;
1710
1711 path = alloc_path_for_send();
1712 if (!path)
1713 return -ENOMEM;
1714
1715 key.objectid = ino;
1716 key.type = BTRFS_EXTENT_DATA_KEY;
1717 key.offset = 0;
1718 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
1719 if (ret < 0)
1720 return ret;
1721 if (unlikely(ret)) {
1722 /*
1723 * An empty symlink inode. Can happen in rare error paths when
1724 * creating a symlink (transaction committed before the inode
1725 * eviction handler removed the symlink inode items and a crash
1726 * happened in between or the subvol was snapshotted in between).
1727 * Print an informative message to dmesg/syslog so that the user
1728 * can delete the symlink.
1729 */
1730 btrfs_err(root->fs_info,
1731 "Found empty symlink inode %llu at root %llu",
1732 ino, btrfs_root_id(root));
1733 return -EIO;
1734 }
1735
1736 ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
1737 struct btrfs_file_extent_item);
1738 type = btrfs_file_extent_type(path->nodes[0], ei);
1739 if (unlikely(type != BTRFS_FILE_EXTENT_INLINE)) {
1740 ret = -EUCLEAN;
1741 btrfs_crit(root->fs_info,
1742"send: found symlink extent that is not inline, ino %llu root %llu extent type %d",
1743 ino, btrfs_root_id(root), type);
1744 return ret;
1745 }
1746 compression = btrfs_file_extent_compression(path->nodes[0], ei);
1747 if (unlikely(compression != BTRFS_COMPRESS_NONE)) {
1748 ret = -EUCLEAN;
1749 btrfs_crit(root->fs_info,
1750"send: found symlink extent with compression, ino %llu root %llu compression type %d",
1751 ino, btrfs_root_id(root), compression);
1752 return ret;
1753 }
1754
1755 off = btrfs_file_extent_inline_start(ei);
1756 len = btrfs_file_extent_ram_bytes(path->nodes[0], ei);
1757
1758 return fs_path_add_from_extent_buffer(dest, path->nodes[0], off, len);
1759}
1760
1761/*
1762 * Helper function to generate a file name that is unique in the root of
1763 * send_root and parent_root. This is used to generate names for orphan inodes.
1764 */
1765static int gen_unique_name(struct send_ctx *sctx,
1766 u64 ino, u64 gen,
1767 struct fs_path *dest)
1768{
1769 BTRFS_PATH_AUTO_FREE(path);
1770 struct btrfs_dir_item *di;
1771 char tmp[64];
1772 int len;
1773 u64 idx = 0;
1774
1775 path = alloc_path_for_send();
1776 if (!path)
1777 return -ENOMEM;
1778
1779 while (1) {
1780 struct fscrypt_str tmp_name;
1781
1782 len = snprintf(tmp, sizeof(tmp), "o%llu-%llu-%llu",
1783 ino, gen, idx);
1784 ASSERT(len < sizeof(tmp));
1785 tmp_name.name = tmp;
1786 tmp_name.len = len;
1787
1788 di = btrfs_lookup_dir_item(NULL, sctx->send_root,
1789 path, BTRFS_FIRST_FREE_OBJECTID,
1790 &tmp_name, 0);
1791 btrfs_release_path(path);
1792 if (IS_ERR(di))
1793 return PTR_ERR(di);
1794
1795 if (di) {
1796 /* not unique, try again */
1797 idx++;
1798 continue;
1799 }
1800
1801 if (!sctx->parent_root) {
1802 /* unique */
1803 break;
1804 }
1805
1806 di = btrfs_lookup_dir_item(NULL, sctx->parent_root,
1807 path, BTRFS_FIRST_FREE_OBJECTID,
1808 &tmp_name, 0);
1809 btrfs_release_path(path);
1810 if (IS_ERR(di))
1811 return PTR_ERR(di);
1812
1813 if (di) {
1814 /* not unique, try again */
1815 idx++;
1816 continue;
1817 }
1818 /* unique */
1819 break;
1820 }
1821
1822 return fs_path_add(dest, tmp, len);
1823}
1824
1825enum inode_state {
1826 inode_state_no_change,
1827 inode_state_will_create,
1828 inode_state_did_create,
1829 inode_state_will_delete,
1830 inode_state_did_delete,
1831};
1832
1833static int get_cur_inode_state(struct send_ctx *sctx, u64 ino, u64 gen,
1834 u64 *send_gen, u64 *parent_gen)
1835{
1836 int ret;
1837 int left_ret;
1838 int right_ret;
1839 u64 left_gen;
1840 u64 right_gen = 0;
1841 struct btrfs_inode_info info;
1842
1843 ret = get_inode_info(sctx->send_root, ino, &info);
1844 if (ret < 0 && ret != -ENOENT)
1845 return ret;
1846 left_ret = (info.nlink == 0) ? -ENOENT : ret;
1847 left_gen = info.gen;
1848 if (send_gen)
1849 *send_gen = ((left_ret == -ENOENT) ? 0 : info.gen);
1850
1851 if (!sctx->parent_root) {
1852 right_ret = -ENOENT;
1853 } else {
1854 ret = get_inode_info(sctx->parent_root, ino, &info);
1855 if (ret < 0 && ret != -ENOENT)
1856 return ret;
1857 right_ret = (info.nlink == 0) ? -ENOENT : ret;
1858 right_gen = info.gen;
1859 if (parent_gen)
1860 *parent_gen = ((right_ret == -ENOENT) ? 0 : info.gen);
1861 }
1862
1863 if (!left_ret && !right_ret) {
1864 if (left_gen == gen && right_gen == gen) {
1865 ret = inode_state_no_change;
1866 } else if (left_gen == gen) {
1867 if (ino < sctx->send_progress)
1868 ret = inode_state_did_create;
1869 else
1870 ret = inode_state_will_create;
1871 } else if (right_gen == gen) {
1872 if (ino < sctx->send_progress)
1873 ret = inode_state_did_delete;
1874 else
1875 ret = inode_state_will_delete;
1876 } else {
1877 ret = -ENOENT;
1878 }
1879 } else if (!left_ret) {
1880 if (left_gen == gen) {
1881 if (ino < sctx->send_progress)
1882 ret = inode_state_did_create;
1883 else
1884 ret = inode_state_will_create;
1885 } else {
1886 ret = -ENOENT;
1887 }
1888 } else if (!right_ret) {
1889 if (right_gen == gen) {
1890 if (ino < sctx->send_progress)
1891 ret = inode_state_did_delete;
1892 else
1893 ret = inode_state_will_delete;
1894 } else {
1895 ret = -ENOENT;
1896 }
1897 } else {
1898 ret = -ENOENT;
1899 }
1900
1901 return ret;
1902}
1903
1904static int is_inode_existent(struct send_ctx *sctx, u64 ino, u64 gen,
1905 u64 *send_gen, u64 *parent_gen)
1906{
1907 int ret;
1908
1909 if (ino == BTRFS_FIRST_FREE_OBJECTID)
1910 return 1;
1911
1912 ret = get_cur_inode_state(sctx, ino, gen, send_gen, parent_gen);
1913 if (ret < 0)
1914 return ret;
1915
1916 if (ret == inode_state_no_change ||
1917 ret == inode_state_did_create ||
1918 ret == inode_state_will_delete)
1919 return 1;
1920
1921 return 0;
1922}
1923
1924/*
1925 * Helper function to lookup a dir item in a dir.
1926 */
1927static int lookup_dir_item_inode(struct btrfs_root *root,
1928 u64 dir, const char *name, int name_len,
1929 u64 *found_inode)
1930{
1931 int ret = 0;
1932 struct btrfs_dir_item *di;
1933 struct btrfs_key key;
1934 BTRFS_PATH_AUTO_FREE(path);
1935 struct fscrypt_str name_str = FSTR_INIT((char *)name, name_len);
1936
1937 path = alloc_path_for_send();
1938 if (!path)
1939 return -ENOMEM;
1940
1941 di = btrfs_lookup_dir_item(NULL, root, path, dir, &name_str, 0);
1942 if (IS_ERR_OR_NULL(di))
1943 return di ? PTR_ERR(di) : -ENOENT;
1944
1945 btrfs_dir_item_key_to_cpu(path->nodes[0], di, &key);
1946 if (key.type == BTRFS_ROOT_ITEM_KEY)
1947 return -ENOENT;
1948
1949 *found_inode = key.objectid;
1950
1951 return ret;
1952}
1953
1954/*
1955 * Looks up the first btrfs_inode_ref of a given ino. It returns the parent dir,
1956 * generation of the parent dir and the name of the dir entry.
1957 */
1958static int get_first_ref(struct btrfs_root *root, u64 ino,
1959 u64 *dir, u64 *dir_gen, struct fs_path *name)
1960{
1961 int ret;
1962 struct btrfs_key key;
1963 struct btrfs_key found_key;
1964 BTRFS_PATH_AUTO_FREE(path);
1965 int len;
1966 u64 parent_dir;
1967
1968 path = alloc_path_for_send();
1969 if (!path)
1970 return -ENOMEM;
1971
1972 key.objectid = ino;
1973 key.type = BTRFS_INODE_REF_KEY;
1974 key.offset = 0;
1975
1976 ret = btrfs_search_slot_for_read(root, &key, path, 1, 0);
1977 if (ret < 0)
1978 return ret;
1979 if (!ret)
1980 btrfs_item_key_to_cpu(path->nodes[0], &found_key,
1981 path->slots[0]);
1982 if (ret || found_key.objectid != ino ||
1983 (found_key.type != BTRFS_INODE_REF_KEY &&
1984 found_key.type != BTRFS_INODE_EXTREF_KEY))
1985 return -ENOENT;
1986
1987 if (found_key.type == BTRFS_INODE_REF_KEY) {
1988 struct btrfs_inode_ref *iref;
1989 iref = btrfs_item_ptr(path->nodes[0], path->slots[0],
1990 struct btrfs_inode_ref);
1991 len = btrfs_inode_ref_name_len(path->nodes[0], iref);
1992 ret = fs_path_add_from_extent_buffer(name, path->nodes[0],
1993 (unsigned long)(iref + 1),
1994 len);
1995 parent_dir = found_key.offset;
1996 } else {
1997 struct btrfs_inode_extref *extref;
1998 extref = btrfs_item_ptr(path->nodes[0], path->slots[0],
1999 struct btrfs_inode_extref);
2000 len = btrfs_inode_extref_name_len(path->nodes[0], extref);
2001 ret = fs_path_add_from_extent_buffer(name, path->nodes[0],
2002 (unsigned long)&extref->name, len);
2003 parent_dir = btrfs_inode_extref_parent(path->nodes[0], extref);
2004 }
2005 if (ret < 0)
2006 return ret;
2007 btrfs_release_path(path);
2008
2009 if (dir_gen) {
2010 ret = get_inode_gen(root, parent_dir, dir_gen);
2011 if (ret < 0)
2012 return ret;
2013 }
2014
2015 *dir = parent_dir;
2016
2017 return ret;
2018}
2019
2020static int is_first_ref(struct btrfs_root *root,
2021 u64 ino, u64 dir,
2022 const char *name, int name_len)
2023{
2024 int ret;
2025 struct fs_path *tmp_name __free(fs_path_free) = NULL;
2026 u64 tmp_dir;
2027
2028 tmp_name = fs_path_alloc();
2029 if (!tmp_name)
2030 return -ENOMEM;
2031
2032 ret = get_first_ref(root, ino, &tmp_dir, NULL, tmp_name);
2033 if (ret < 0)
2034 return ret;
2035
2036 if (dir != tmp_dir || name_len != fs_path_len(tmp_name))
2037 return 0;
2038
2039 ret = !memcmp(tmp_name->start, name, name_len);
2040
2041 return ret;
2042}
2043
2044/*
2045 * Used by process_recorded_refs to determine if a new ref would overwrite an
2046 * already existing ref. In case it detects an overwrite, it returns the
2047 * inode/gen in who_ino/who_gen.
2048 * When an overwrite is detected, process_recorded_refs does proper orphanizing
2049 * to make sure later references to the overwritten inode are possible.
2050 * Orphanizing is however only required for the first ref of an inode.
2051 * process_recorded_refs does an additional is_first_ref check to see if
2052 * orphanizing is really required.
2053 */
2054static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen,
2055 const char *name, int name_len,
2056 u64 *who_ino, u64 *who_gen, u64 *who_mode)
2057{
2058 int ret;
2059 u64 parent_root_dir_gen;
2060 u64 other_inode = 0;
2061 struct btrfs_inode_info info;
2062
2063 if (!sctx->parent_root)
2064 return 0;
2065
2066 ret = is_inode_existent(sctx, dir, dir_gen, NULL, &parent_root_dir_gen);
2067 if (ret <= 0)
2068 return ret;
2069
2070 /*
2071 * If we have a parent root we need to verify that the parent dir was
2072 * not deleted and then re-created, if it was then we have no overwrite
2073 * and we can just unlink this entry.
2074 *
2075 * @parent_root_dir_gen was set to 0 if the inode does not exist in the
2076 * parent root.
2077 */
2078 if (sctx->parent_root && dir != BTRFS_FIRST_FREE_OBJECTID &&
2079 parent_root_dir_gen != dir_gen)
2080 return 0;
2081
2082 ret = lookup_dir_item_inode(sctx->parent_root, dir, name, name_len,
2083 &other_inode);
2084 if (ret == -ENOENT)
2085 return 0;
2086 else if (ret < 0)
2087 return ret;
2088
2089 /*
2090 * Check if the overwritten ref was already processed. If yes, the ref
2091 * was already unlinked/moved, so we can safely assume that we will not
2092 * overwrite anything at this point in time.
2093 */
2094 if (other_inode > sctx->send_progress ||
2095 is_waiting_for_move(sctx, other_inode)) {
2096 ret = get_inode_info(sctx->parent_root, other_inode, &info);
2097 if (ret < 0)
2098 return ret;
2099
2100 *who_ino = other_inode;
2101 *who_gen = info.gen;
2102 *who_mode = info.mode;
2103 return 1;
2104 }
2105
2106 return 0;
2107}
2108
2109/*
2110 * Checks if the ref was overwritten by an already processed inode. This is
2111 * used by __get_cur_name_and_parent to find out if the ref was orphanized and
2112 * thus the orphan name needs be used.
2113 * process_recorded_refs also uses it to avoid unlinking of refs that were
2114 * overwritten.
2115 */
2116static int did_overwrite_ref(struct send_ctx *sctx,
2117 u64 dir, u64 dir_gen,
2118 u64 ino, u64 ino_gen,
2119 const char *name, int name_len)
2120{
2121 int ret;
2122 u64 ow_inode;
2123 u64 ow_gen = 0;
2124 u64 send_root_dir_gen;
2125
2126 if (!sctx->parent_root)
2127 return 0;
2128
2129 ret = is_inode_existent(sctx, dir, dir_gen, &send_root_dir_gen, NULL);
2130 if (ret <= 0)
2131 return ret;
2132
2133 /*
2134 * @send_root_dir_gen was set to 0 if the inode does not exist in the
2135 * send root.
2136 */
2137 if (dir != BTRFS_FIRST_FREE_OBJECTID && send_root_dir_gen != dir_gen)
2138 return 0;
2139
2140 /* check if the ref was overwritten by another ref */
2141 ret = lookup_dir_item_inode(sctx->send_root, dir, name, name_len,
2142 &ow_inode);
2143 if (ret == -ENOENT) {
2144 /* was never and will never be overwritten */
2145 return 0;
2146 } else if (ret < 0) {
2147 return ret;
2148 }
2149
2150 if (ow_inode == ino) {
2151 ret = get_inode_gen(sctx->send_root, ow_inode, &ow_gen);
2152 if (ret < 0)
2153 return ret;
2154
2155 /* It's the same inode, so no overwrite happened. */
2156 if (ow_gen == ino_gen)
2157 return 0;
2158 }
2159
2160 /*
2161 * We know that it is or will be overwritten. Check this now.
2162 * The current inode being processed might have been the one that caused
2163 * inode 'ino' to be orphanized, therefore check if ow_inode matches
2164 * the current inode being processed.
2165 */
2166 if (ow_inode < sctx->send_progress)
2167 return 1;
2168
2169 if (ino != sctx->cur_ino && ow_inode == sctx->cur_ino) {
2170 if (ow_gen == 0) {
2171 ret = get_inode_gen(sctx->send_root, ow_inode, &ow_gen);
2172 if (ret < 0)
2173 return ret;
2174 }
2175 if (ow_gen == sctx->cur_inode_gen)
2176 return 1;
2177 }
2178
2179 return 0;
2180}
2181
2182/*
2183 * Same as did_overwrite_ref, but also checks if it is the first ref of an inode
2184 * that got overwritten. This is used by process_recorded_refs to determine
2185 * if it has to use the path as returned by get_cur_path or the orphan name.
2186 */
2187static int did_overwrite_first_ref(struct send_ctx *sctx, u64 ino, u64 gen)
2188{
2189 int ret;
2190 struct fs_path *name __free(fs_path_free) = NULL;
2191 u64 dir;
2192 u64 dir_gen;
2193
2194 if (!sctx->parent_root)
2195 return 0;
2196
2197 name = fs_path_alloc();
2198 if (!name)
2199 return -ENOMEM;
2200
2201 ret = get_first_ref(sctx->parent_root, ino, &dir, &dir_gen, name);
2202 if (ret < 0)
2203 return ret;
2204
2205 return did_overwrite_ref(sctx, dir, dir_gen, ino, gen,
2206 name->start, fs_path_len(name));
2207}
2208
2209static inline struct name_cache_entry *name_cache_search(struct send_ctx *sctx,
2210 u64 ino, u64 gen)
2211{
2212 struct btrfs_lru_cache_entry *entry;
2213
2214 entry = btrfs_lru_cache_lookup(&sctx->name_cache, ino, gen);
2215 if (!entry)
2216 return NULL;
2217
2218 return container_of(entry, struct name_cache_entry, entry);
2219}
2220
2221/*
2222 * Used by get_cur_path for each ref up to the root.
2223 * Returns 0 if it succeeded.
2224 * Returns 1 if the inode is not existent or got overwritten. In that case, the
2225 * name is an orphan name. This instructs get_cur_path to stop iterating. If 1
2226 * is returned, parent_ino/parent_gen are not guaranteed to be valid.
2227 * Returns <0 in case of error.
2228 */
2229static int __get_cur_name_and_parent(struct send_ctx *sctx,
2230 u64 ino, u64 gen,
2231 u64 *parent_ino,
2232 u64 *parent_gen,
2233 struct fs_path *dest)
2234{
2235 int ret;
2236 int nce_ret;
2237 struct name_cache_entry *nce;
2238
2239 /*
2240 * First check if we already did a call to this function with the same
2241 * ino/gen. If yes, check if the cache entry is still up-to-date. If yes
2242 * return the cached result.
2243 */
2244 nce = name_cache_search(sctx, ino, gen);
2245 if (nce) {
2246 if (ino < sctx->send_progress && nce->need_later_update) {
2247 btrfs_lru_cache_remove(&sctx->name_cache, &nce->entry);
2248 nce = NULL;
2249 } else {
2250 *parent_ino = nce->parent_ino;
2251 *parent_gen = nce->parent_gen;
2252 ret = fs_path_add(dest, nce->name, nce->name_len);
2253 if (ret < 0)
2254 return ret;
2255 return nce->ret;
2256 }
2257 }
2258
2259 /*
2260 * If the inode is not existent yet, add the orphan name and return 1.
2261 * This should only happen for the parent dir that we determine in
2262 * record_new_ref_if_needed().
2263 */
2264 ret = is_inode_existent(sctx, ino, gen, NULL, NULL);
2265 if (ret < 0)
2266 return ret;
2267
2268 if (!ret) {
2269 ret = gen_unique_name(sctx, ino, gen, dest);
2270 if (ret < 0)
2271 return ret;
2272 ret = 1;
2273 goto out_cache;
2274 }
2275
2276 /*
2277 * Depending on whether the inode was already processed or not, use
2278 * send_root or parent_root for ref lookup.
2279 */
2280 if (ino < sctx->send_progress)
2281 ret = get_first_ref(sctx->send_root, ino,
2282 parent_ino, parent_gen, dest);
2283 else
2284 ret = get_first_ref(sctx->parent_root, ino,
2285 parent_ino, parent_gen, dest);
2286 if (ret < 0)
2287 return ret;
2288
2289 /*
2290 * Check if the ref was overwritten by an inode's ref that was processed
2291 * earlier. If yes, treat as orphan and return 1.
2292 */
2293 ret = did_overwrite_ref(sctx, *parent_ino, *parent_gen, ino, gen,
2294 dest->start, fs_path_len(dest));
2295 if (ret < 0)
2296 return ret;
2297 if (ret) {
2298 fs_path_reset(dest);
2299 ret = gen_unique_name(sctx, ino, gen, dest);
2300 if (ret < 0)
2301 return ret;
2302 ret = 1;
2303 }
2304
2305out_cache:
2306 /*
2307 * Store the result of the lookup in the name cache.
2308 */
2309 nce = kmalloc(sizeof(*nce) + fs_path_len(dest), GFP_KERNEL);
2310 if (!nce)
2311 return -ENOMEM;
2312
2313 nce->entry.key = ino;
2314 nce->entry.gen = gen;
2315 nce->parent_ino = *parent_ino;
2316 nce->parent_gen = *parent_gen;
2317 nce->name_len = fs_path_len(dest);
2318 nce->ret = ret;
2319 memcpy(nce->name, dest->start, nce->name_len);
2320
2321 if (ino < sctx->send_progress)
2322 nce->need_later_update = 0;
2323 else
2324 nce->need_later_update = 1;
2325
2326 nce_ret = btrfs_lru_cache_store(&sctx->name_cache, &nce->entry, GFP_KERNEL);
2327 if (nce_ret < 0) {
2328 kfree(nce);
2329 return nce_ret;
2330 }
2331
2332 return ret;
2333}
2334
2335/*
2336 * Magic happens here. This function returns the first ref to an inode as it
2337 * would look like while receiving the stream at this point in time.
2338 * We walk the path up to the root. For every inode in between, we check if it
2339 * was already processed/sent. If yes, we continue with the parent as found
2340 * in send_root. If not, we continue with the parent as found in parent_root.
2341 * If we encounter an inode that was deleted at this point in time, we use the
2342 * inodes "orphan" name instead of the real name and stop. Same with new inodes
2343 * that were not created yet and overwritten inodes/refs.
2344 *
2345 * When do we have orphan inodes:
2346 * 1. When an inode is freshly created and thus no valid refs are available yet
2347 * 2. When a directory lost all it's refs (deleted) but still has dir items
2348 * inside which were not processed yet (pending for move/delete). If anyone
2349 * tried to get the path to the dir items, it would get a path inside that
2350 * orphan directory.
2351 * 3. When an inode is moved around or gets new links, it may overwrite the ref
2352 * of an unprocessed inode. If in that case the first ref would be
2353 * overwritten, the overwritten inode gets "orphanized". Later when we
2354 * process this overwritten inode, it is restored at a new place by moving
2355 * the orphan inode.
2356 *
2357 * sctx->send_progress tells this function at which point in time receiving
2358 * would be.
2359 */
2360static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen,
2361 struct fs_path *dest)
2362{
2363 int ret = 0;
2364 struct fs_path *name __free(fs_path_free) = NULL;
2365 u64 parent_inode = 0;
2366 u64 parent_gen = 0;
2367 bool stop = false;
2368 const bool is_cur_inode = (ino == sctx->cur_ino && gen == sctx->cur_inode_gen);
2369
2370 if (is_cur_inode && fs_path_len(&sctx->cur_inode_path) > 0) {
2371 if (dest != &sctx->cur_inode_path)
2372 return fs_path_copy(dest, &sctx->cur_inode_path);
2373
2374 return 0;
2375 }
2376
2377 name = fs_path_alloc();
2378 if (!name)
2379 return -ENOMEM;
2380
2381 dest->reversed = 1;
2382 fs_path_reset(dest);
2383
2384 while (!stop && ino != BTRFS_FIRST_FREE_OBJECTID) {
2385 struct waiting_dir_move *wdm;
2386
2387 fs_path_reset(name);
2388
2389 if (is_waiting_for_rm(sctx, ino, gen)) {
2390 ret = gen_unique_name(sctx, ino, gen, name);
2391 if (ret < 0)
2392 goto out;
2393 ret = fs_path_add_path(dest, name);
2394 break;
2395 }
2396
2397 wdm = get_waiting_dir_move(sctx, ino);
2398 if (wdm && wdm->orphanized) {
2399 ret = gen_unique_name(sctx, ino, gen, name);
2400 stop = true;
2401 } else if (wdm) {
2402 ret = get_first_ref(sctx->parent_root, ino,
2403 &parent_inode, &parent_gen, name);
2404 } else {
2405 ret = __get_cur_name_and_parent(sctx, ino, gen,
2406 &parent_inode,
2407 &parent_gen, name);
2408 if (ret)
2409 stop = true;
2410 }
2411
2412 if (ret < 0)
2413 goto out;
2414
2415 ret = fs_path_add_path(dest, name);
2416 if (ret < 0)
2417 goto out;
2418
2419 ino = parent_inode;
2420 gen = parent_gen;
2421 }
2422
2423out:
2424 if (!ret) {
2425 fs_path_unreverse(dest);
2426 if (is_cur_inode && dest != &sctx->cur_inode_path)
2427 ret = fs_path_copy(&sctx->cur_inode_path, dest);
2428 }
2429
2430 return ret;
2431}
2432
2433/*
2434 * Sends a BTRFS_SEND_C_SUBVOL command/item to userspace
2435 */
2436static int send_subvol_begin(struct send_ctx *sctx)
2437{
2438 int ret;
2439 struct btrfs_root *send_root = sctx->send_root;
2440 struct btrfs_root *parent_root = sctx->parent_root;
2441 BTRFS_PATH_AUTO_FREE(path);
2442 struct btrfs_key key;
2443 struct btrfs_root_ref *ref;
2444 struct extent_buffer *leaf;
2445 char AUTO_KFREE(name);
2446 int namelen;
2447
2448 path = btrfs_alloc_path();
2449 if (!path)
2450 return -ENOMEM;
2451
2452 name = kmalloc(BTRFS_PATH_NAME_MAX, GFP_KERNEL);
2453 if (!name)
2454 return -ENOMEM;
2455
2456 key.objectid = btrfs_root_id(send_root);
2457 key.type = BTRFS_ROOT_BACKREF_KEY;
2458 key.offset = 0;
2459
2460 ret = btrfs_search_slot_for_read(send_root->fs_info->tree_root,
2461 &key, path, 1, 0);
2462 if (ret < 0)
2463 return ret;
2464 if (ret)
2465 return -ENOENT;
2466
2467 leaf = path->nodes[0];
2468 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2469 if (key.type != BTRFS_ROOT_BACKREF_KEY ||
2470 key.objectid != btrfs_root_id(send_root)) {
2471 return -ENOENT;
2472 }
2473 ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref);
2474 namelen = btrfs_root_ref_name_len(leaf, ref);
2475 read_extent_buffer(leaf, name, (unsigned long)(ref + 1), namelen);
2476 btrfs_release_path(path);
2477
2478 if (parent_root) {
2479 ret = begin_cmd(sctx, BTRFS_SEND_C_SNAPSHOT);
2480 if (ret < 0)
2481 return ret;
2482 } else {
2483 ret = begin_cmd(sctx, BTRFS_SEND_C_SUBVOL);
2484 if (ret < 0)
2485 return ret;
2486 }
2487
2488 TLV_PUT_STRING(sctx, BTRFS_SEND_A_PATH, name, namelen);
2489
2490 if (!btrfs_is_empty_uuid(sctx->send_root->root_item.received_uuid))
2491 TLV_PUT_UUID(sctx, BTRFS_SEND_A_UUID,
2492 sctx->send_root->root_item.received_uuid);
2493 else
2494 TLV_PUT_UUID(sctx, BTRFS_SEND_A_UUID,
2495 sctx->send_root->root_item.uuid);
2496
2497 TLV_PUT_U64(sctx, BTRFS_SEND_A_CTRANSID,
2498 btrfs_root_ctransid(&sctx->send_root->root_item));
2499 if (parent_root) {
2500 if (!btrfs_is_empty_uuid(parent_root->root_item.received_uuid))
2501 TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
2502 parent_root->root_item.received_uuid);
2503 else
2504 TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
2505 parent_root->root_item.uuid);
2506 TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_CTRANSID,
2507 btrfs_root_ctransid(&sctx->parent_root->root_item));
2508 }
2509
2510 ret = send_cmd(sctx);
2511
2512tlv_put_failure:
2513 return ret;
2514}
2515
2516static struct fs_path *get_cur_inode_path(struct send_ctx *sctx)
2517{
2518 if (fs_path_len(&sctx->cur_inode_path) == 0) {
2519 int ret;
2520
2521 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen,
2522 &sctx->cur_inode_path);
2523 if (ret < 0)
2524 return ERR_PTR(ret);
2525 }
2526
2527 return &sctx->cur_inode_path;
2528}
2529
2530static struct fs_path *get_path_for_command(struct send_ctx *sctx, u64 ino, u64 gen)
2531{
2532 struct fs_path *path;
2533 int ret;
2534
2535 if (ino == sctx->cur_ino && gen == sctx->cur_inode_gen)
2536 return get_cur_inode_path(sctx);
2537
2538 path = fs_path_alloc();
2539 if (!path)
2540 return ERR_PTR(-ENOMEM);
2541
2542 ret = get_cur_path(sctx, ino, gen, path);
2543 if (ret < 0) {
2544 fs_path_free(path);
2545 return ERR_PTR(ret);
2546 }
2547
2548 return path;
2549}
2550
2551static void free_path_for_command(const struct send_ctx *sctx, struct fs_path *path)
2552{
2553 if (path != &sctx->cur_inode_path)
2554 fs_path_free(path);
2555}
2556
2557static int send_truncate(struct send_ctx *sctx, u64 ino, u64 gen, u64 size)
2558{
2559 int ret = 0;
2560 struct fs_path *p;
2561
2562 p = get_path_for_command(sctx, ino, gen);
2563 if (IS_ERR(p))
2564 return PTR_ERR(p);
2565
2566 ret = begin_cmd(sctx, BTRFS_SEND_C_TRUNCATE);
2567 if (ret < 0)
2568 goto out;
2569
2570 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2571 TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, size);
2572
2573 ret = send_cmd(sctx);
2574
2575tlv_put_failure:
2576out:
2577 free_path_for_command(sctx, p);
2578 return ret;
2579}
2580
2581static int send_chmod(struct send_ctx *sctx, u64 ino, u64 gen, u64 mode)
2582{
2583 int ret = 0;
2584 struct fs_path *p;
2585
2586 p = get_path_for_command(sctx, ino, gen);
2587 if (IS_ERR(p))
2588 return PTR_ERR(p);
2589
2590 ret = begin_cmd(sctx, BTRFS_SEND_C_CHMOD);
2591 if (ret < 0)
2592 goto out;
2593
2594 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2595 TLV_PUT_U64(sctx, BTRFS_SEND_A_MODE, mode & 07777);
2596
2597 ret = send_cmd(sctx);
2598
2599tlv_put_failure:
2600out:
2601 free_path_for_command(sctx, p);
2602 return ret;
2603}
2604
2605static int send_fileattr(struct send_ctx *sctx, u64 ino, u64 gen, u64 fileattr)
2606{
2607 int ret = 0;
2608 struct fs_path *p;
2609
2610 if (sctx->proto < 2)
2611 return 0;
2612
2613 p = get_path_for_command(sctx, ino, gen);
2614 if (IS_ERR(p))
2615 return PTR_ERR(p);
2616
2617 ret = begin_cmd(sctx, BTRFS_SEND_C_FILEATTR);
2618 if (ret < 0)
2619 goto out;
2620
2621 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2622 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILEATTR, fileattr);
2623
2624 ret = send_cmd(sctx);
2625
2626tlv_put_failure:
2627out:
2628 free_path_for_command(sctx, p);
2629 return ret;
2630}
2631
2632static int send_chown(struct send_ctx *sctx, u64 ino, u64 gen, u64 uid, u64 gid)
2633{
2634 int ret = 0;
2635 struct fs_path *p;
2636
2637 p = get_path_for_command(sctx, ino, gen);
2638 if (IS_ERR(p))
2639 return PTR_ERR(p);
2640
2641 ret = begin_cmd(sctx, BTRFS_SEND_C_CHOWN);
2642 if (ret < 0)
2643 goto out;
2644
2645 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2646 TLV_PUT_U64(sctx, BTRFS_SEND_A_UID, uid);
2647 TLV_PUT_U64(sctx, BTRFS_SEND_A_GID, gid);
2648
2649 ret = send_cmd(sctx);
2650
2651tlv_put_failure:
2652out:
2653 free_path_for_command(sctx, p);
2654 return ret;
2655}
2656
2657static int send_utimes(struct send_ctx *sctx, u64 ino, u64 gen)
2658{
2659 int ret = 0;
2660 struct fs_path *p = NULL;
2661 struct btrfs_inode_item *ii;
2662 BTRFS_PATH_AUTO_FREE(path);
2663 struct extent_buffer *eb;
2664 struct btrfs_key key;
2665 int slot;
2666
2667 p = get_path_for_command(sctx, ino, gen);
2668 if (IS_ERR(p))
2669 return PTR_ERR(p);
2670
2671 path = alloc_path_for_send();
2672 if (!path) {
2673 ret = -ENOMEM;
2674 goto out;
2675 }
2676
2677 key.objectid = ino;
2678 key.type = BTRFS_INODE_ITEM_KEY;
2679 key.offset = 0;
2680 ret = btrfs_search_slot(NULL, sctx->send_root, &key, path, 0, 0);
2681 if (ret > 0)
2682 ret = -ENOENT;
2683 if (ret < 0)
2684 goto out;
2685
2686 eb = path->nodes[0];
2687 slot = path->slots[0];
2688 ii = btrfs_item_ptr(eb, slot, struct btrfs_inode_item);
2689
2690 ret = begin_cmd(sctx, BTRFS_SEND_C_UTIMES);
2691 if (ret < 0)
2692 goto out;
2693
2694 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2695 TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_ATIME, eb, &ii->atime);
2696 TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_MTIME, eb, &ii->mtime);
2697 TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_CTIME, eb, &ii->ctime);
2698 if (sctx->proto >= 2)
2699 TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_OTIME, eb, &ii->otime);
2700
2701 ret = send_cmd(sctx);
2702
2703tlv_put_failure:
2704out:
2705 free_path_for_command(sctx, p);
2706 return ret;
2707}
2708
2709/*
2710 * If the cache is full, we can't remove entries from it and do a call to
2711 * send_utimes() for each respective inode, because we might be finishing
2712 * processing an inode that is a directory and it just got renamed, and existing
2713 * entries in the cache may refer to inodes that have the directory in their
2714 * full path - in which case we would generate outdated paths (pre-rename)
2715 * for the inodes that the cache entries point to. Instead of pruning the
2716 * cache when inserting, do it after we finish processing each inode at
2717 * finish_inode_if_needed().
2718 */
2719static int cache_dir_utimes(struct send_ctx *sctx, u64 dir, u64 gen)
2720{
2721 struct btrfs_lru_cache_entry *entry;
2722 int ret;
2723
2724 entry = btrfs_lru_cache_lookup(&sctx->dir_utimes_cache, dir, gen);
2725 if (entry != NULL)
2726 return 0;
2727
2728 /* Caching is optional, don't fail if we can't allocate memory. */
2729 entry = kmalloc_obj(*entry);
2730 if (!entry)
2731 return send_utimes(sctx, dir, gen);
2732
2733 entry->key = dir;
2734 entry->gen = gen;
2735
2736 ret = btrfs_lru_cache_store(&sctx->dir_utimes_cache, entry, GFP_KERNEL);
2737 ASSERT(ret != -EEXIST);
2738 if (ret) {
2739 kfree(entry);
2740 return send_utimes(sctx, dir, gen);
2741 }
2742
2743 return 0;
2744}
2745
2746static int trim_dir_utimes_cache(struct send_ctx *sctx)
2747{
2748 while (sctx->dir_utimes_cache.size > SEND_MAX_DIR_UTIMES_CACHE_SIZE) {
2749 struct btrfs_lru_cache_entry *lru;
2750 int ret;
2751
2752 lru = btrfs_lru_cache_lru_entry(&sctx->dir_utimes_cache);
2753 ASSERT(lru != NULL);
2754
2755 ret = send_utimes(sctx, lru->key, lru->gen);
2756 if (ret)
2757 return ret;
2758
2759 btrfs_lru_cache_remove(&sctx->dir_utimes_cache, lru);
2760 }
2761
2762 return 0;
2763}
2764
2765/*
2766 * Sends a BTRFS_SEND_C_MKXXX or SYMLINK command to user space. We don't have
2767 * a valid path yet because we did not process the refs yet. So, the inode
2768 * is created as orphan.
2769 */
2770static int send_create_inode(struct send_ctx *sctx, u64 ino)
2771{
2772 int ret = 0;
2773 struct fs_path *p __free(fs_path_free) = NULL;
2774 int cmd;
2775 struct btrfs_inode_info info;
2776 u64 gen;
2777 u64 mode;
2778 u64 rdev;
2779
2780 p = fs_path_alloc();
2781 if (!p)
2782 return -ENOMEM;
2783
2784 if (ino != sctx->cur_ino) {
2785 ret = get_inode_info(sctx->send_root, ino, &info);
2786 if (ret < 0)
2787 return ret;
2788 gen = info.gen;
2789 mode = info.mode;
2790 rdev = info.rdev;
2791 } else {
2792 gen = sctx->cur_inode_gen;
2793 mode = sctx->cur_inode_mode;
2794 rdev = sctx->cur_inode_rdev;
2795 }
2796
2797 if (S_ISREG(mode)) {
2798 cmd = BTRFS_SEND_C_MKFILE;
2799 } else if (S_ISDIR(mode)) {
2800 cmd = BTRFS_SEND_C_MKDIR;
2801 } else if (S_ISLNK(mode)) {
2802 cmd = BTRFS_SEND_C_SYMLINK;
2803 } else if (S_ISCHR(mode) || S_ISBLK(mode)) {
2804 cmd = BTRFS_SEND_C_MKNOD;
2805 } else if (S_ISFIFO(mode)) {
2806 cmd = BTRFS_SEND_C_MKFIFO;
2807 } else if (S_ISSOCK(mode)) {
2808 cmd = BTRFS_SEND_C_MKSOCK;
2809 } else {
2810 btrfs_warn(sctx->send_root->fs_info, "unexpected inode type %o",
2811 (int)(mode & S_IFMT));
2812 return -EOPNOTSUPP;
2813 }
2814
2815 ret = begin_cmd(sctx, cmd);
2816 if (ret < 0)
2817 return ret;
2818
2819 ret = gen_unique_name(sctx, ino, gen, p);
2820 if (ret < 0)
2821 return ret;
2822
2823 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
2824 TLV_PUT_U64(sctx, BTRFS_SEND_A_INO, ino);
2825
2826 if (S_ISLNK(mode)) {
2827 fs_path_reset(p);
2828 ret = read_symlink(sctx->send_root, ino, p);
2829 if (ret < 0)
2830 return ret;
2831 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_LINK, p);
2832 } else if (S_ISCHR(mode) || S_ISBLK(mode) ||
2833 S_ISFIFO(mode) || S_ISSOCK(mode)) {
2834 TLV_PUT_U64(sctx, BTRFS_SEND_A_RDEV, new_encode_dev(rdev));
2835 TLV_PUT_U64(sctx, BTRFS_SEND_A_MODE, mode);
2836 }
2837
2838 ret = send_cmd(sctx);
2839 if (ret < 0)
2840 return ret;
2841
2842tlv_put_failure:
2843 return ret;
2844}
2845
2846static void cache_dir_created(struct send_ctx *sctx, u64 dir)
2847{
2848 struct btrfs_lru_cache_entry *entry;
2849 int ret;
2850
2851 /* Caching is optional, ignore any failures. */
2852 entry = kmalloc_obj(*entry);
2853 if (!entry)
2854 return;
2855
2856 entry->key = dir;
2857 entry->gen = 0;
2858 ret = btrfs_lru_cache_store(&sctx->dir_created_cache, entry, GFP_KERNEL);
2859 if (ret < 0)
2860 kfree(entry);
2861}
2862
2863/*
2864 * We need some special handling for inodes that get processed before the parent
2865 * directory got created. See process_recorded_refs for details.
2866 * This function does the check if we already created the dir out of order.
2867 */
2868static int did_create_dir(struct send_ctx *sctx, u64 dir)
2869{
2870 int ret = 0;
2871 int iter_ret = 0;
2872 BTRFS_PATH_AUTO_FREE(path);
2873 struct btrfs_key key;
2874 struct btrfs_key found_key;
2875 struct btrfs_key di_key;
2876 struct btrfs_dir_item *di;
2877
2878 if (btrfs_lru_cache_lookup(&sctx->dir_created_cache, dir, 0))
2879 return 1;
2880
2881 path = alloc_path_for_send();
2882 if (!path)
2883 return -ENOMEM;
2884
2885 key.objectid = dir;
2886 key.type = BTRFS_DIR_INDEX_KEY;
2887 key.offset = 0;
2888
2889 btrfs_for_each_slot(sctx->send_root, &key, &found_key, path, iter_ret) {
2890 struct extent_buffer *eb = path->nodes[0];
2891
2892 if (found_key.objectid != key.objectid ||
2893 found_key.type != key.type) {
2894 ret = 0;
2895 break;
2896 }
2897
2898 di = btrfs_item_ptr(eb, path->slots[0], struct btrfs_dir_item);
2899 btrfs_dir_item_key_to_cpu(eb, di, &di_key);
2900
2901 if (di_key.type != BTRFS_ROOT_ITEM_KEY &&
2902 di_key.objectid < sctx->send_progress) {
2903 ret = 1;
2904 cache_dir_created(sctx, dir);
2905 break;
2906 }
2907 }
2908 /* Catch error found during iteration */
2909 if (iter_ret < 0)
2910 ret = iter_ret;
2911
2912 return ret;
2913}
2914
2915/*
2916 * Only creates the inode if it is:
2917 * 1. Not a directory
2918 * 2. Or a directory which was not created already due to out of order
2919 * directories. See did_create_dir and process_recorded_refs for details.
2920 */
2921static int send_create_inode_if_needed(struct send_ctx *sctx)
2922{
2923 int ret;
2924
2925 if (S_ISDIR(sctx->cur_inode_mode)) {
2926 ret = did_create_dir(sctx, sctx->cur_ino);
2927 if (ret < 0)
2928 return ret;
2929 else if (ret > 0)
2930 return 0;
2931 }
2932
2933 ret = send_create_inode(sctx, sctx->cur_ino);
2934
2935 if (ret == 0 && S_ISDIR(sctx->cur_inode_mode))
2936 cache_dir_created(sctx, sctx->cur_ino);
2937
2938 return ret;
2939}
2940
2941struct recorded_ref {
2942 struct list_head list;
2943 char *name;
2944 struct fs_path *full_path;
2945 u64 dir;
2946 u64 dir_gen;
2947 int name_len;
2948 struct rb_node node;
2949 struct rb_root *root;
2950};
2951
2952static struct recorded_ref *recorded_ref_alloc(void)
2953{
2954 struct recorded_ref *ref;
2955
2956 ref = kzalloc_obj(*ref);
2957 if (!ref)
2958 return NULL;
2959 RB_CLEAR_NODE(&ref->node);
2960 INIT_LIST_HEAD(&ref->list);
2961 return ref;
2962}
2963
2964static void recorded_ref_free(struct recorded_ref *ref)
2965{
2966 if (!ref)
2967 return;
2968 if (!RB_EMPTY_NODE(&ref->node))
2969 rb_erase(&ref->node, ref->root);
2970 list_del(&ref->list);
2971 fs_path_free(ref->full_path);
2972 kfree(ref);
2973}
2974
2975static void set_ref_path(struct recorded_ref *ref, struct fs_path *path)
2976{
2977 ref->full_path = path;
2978 ref->name = (char *)kbasename(ref->full_path->start);
2979 ref->name_len = ref->full_path->end - ref->name;
2980}
2981
2982static int dup_ref(struct recorded_ref *ref, struct list_head *list)
2983{
2984 struct recorded_ref *new;
2985
2986 new = recorded_ref_alloc();
2987 if (!new)
2988 return -ENOMEM;
2989
2990 new->dir = ref->dir;
2991 new->dir_gen = ref->dir_gen;
2992 list_add_tail(&new->list, list);
2993 return 0;
2994}
2995
2996static void __free_recorded_refs(struct list_head *head)
2997{
2998 struct recorded_ref *cur;
2999
3000 while (!list_empty(head)) {
3001 cur = list_first_entry(head, struct recorded_ref, list);
3002 recorded_ref_free(cur);
3003 }
3004}
3005
3006static void free_recorded_refs(struct send_ctx *sctx)
3007{
3008 __free_recorded_refs(&sctx->new_refs);
3009 __free_recorded_refs(&sctx->deleted_refs);
3010}
3011
3012/*
3013 * Renames/moves a file/dir to its orphan name. Used when the first
3014 * ref of an unprocessed inode gets overwritten and for all non empty
3015 * directories.
3016 */
3017static int orphanize_inode(struct send_ctx *sctx, u64 ino, u64 gen,
3018 struct fs_path *path)
3019{
3020 int ret;
3021 struct fs_path *orphan __free(fs_path_free) = NULL;
3022
3023 orphan = fs_path_alloc();
3024 if (!orphan)
3025 return -ENOMEM;
3026
3027 ret = gen_unique_name(sctx, ino, gen, orphan);
3028 if (ret < 0)
3029 return ret;
3030
3031 ret = send_rename(sctx, path, orphan);
3032 if (ret < 0)
3033 return ret;
3034
3035 if (ino == sctx->cur_ino && gen == sctx->cur_inode_gen)
3036 ret = fs_path_copy(&sctx->cur_inode_path, orphan);
3037
3038 return ret;
3039}
3040
3041static struct orphan_dir_info *add_orphan_dir_info(struct send_ctx *sctx,
3042 u64 dir_ino, u64 dir_gen)
3043{
3044 struct rb_node **p = &sctx->orphan_dirs.rb_node;
3045 struct rb_node *parent = NULL;
3046 struct orphan_dir_info *entry, *odi;
3047
3048 while (*p) {
3049 parent = *p;
3050 entry = rb_entry(parent, struct orphan_dir_info, node);
3051 if (dir_ino < entry->ino)
3052 p = &(*p)->rb_left;
3053 else if (dir_ino > entry->ino)
3054 p = &(*p)->rb_right;
3055 else if (dir_gen < entry->gen)
3056 p = &(*p)->rb_left;
3057 else if (dir_gen > entry->gen)
3058 p = &(*p)->rb_right;
3059 else
3060 return entry;
3061 }
3062
3063 odi = kmalloc_obj(*odi);
3064 if (!odi)
3065 return ERR_PTR(-ENOMEM);
3066 odi->ino = dir_ino;
3067 odi->gen = dir_gen;
3068 odi->last_dir_index_offset = 0;
3069 odi->dir_high_seq_ino = 0;
3070
3071 rb_link_node(&odi->node, parent, p);
3072 rb_insert_color(&odi->node, &sctx->orphan_dirs);
3073 return odi;
3074}
3075
3076static struct orphan_dir_info *get_orphan_dir_info(struct send_ctx *sctx,
3077 u64 dir_ino, u64 gen)
3078{
3079 struct rb_node *n = sctx->orphan_dirs.rb_node;
3080 struct orphan_dir_info *entry;
3081
3082 while (n) {
3083 entry = rb_entry(n, struct orphan_dir_info, node);
3084 if (dir_ino < entry->ino)
3085 n = n->rb_left;
3086 else if (dir_ino > entry->ino)
3087 n = n->rb_right;
3088 else if (gen < entry->gen)
3089 n = n->rb_left;
3090 else if (gen > entry->gen)
3091 n = n->rb_right;
3092 else
3093 return entry;
3094 }
3095 return NULL;
3096}
3097
3098static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino, u64 gen)
3099{
3100 struct orphan_dir_info *odi = get_orphan_dir_info(sctx, dir_ino, gen);
3101
3102 return odi != NULL;
3103}
3104
3105static void free_orphan_dir_info(struct send_ctx *sctx,
3106 struct orphan_dir_info *odi)
3107{
3108 if (!odi)
3109 return;
3110 rb_erase(&odi->node, &sctx->orphan_dirs);
3111 kfree(odi);
3112}
3113
3114/*
3115 * Returns 1 if a directory can be removed at this point in time.
3116 * We check this by iterating all dir items and checking if the inode behind
3117 * the dir item was already processed.
3118 */
3119static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen)
3120{
3121 int ret = 0;
3122 int iter_ret = 0;
3123 struct btrfs_root *root = sctx->parent_root;
3124 struct btrfs_path *path;
3125 struct btrfs_key key;
3126 struct btrfs_key found_key;
3127 struct btrfs_key loc;
3128 struct btrfs_dir_item *di;
3129 struct orphan_dir_info *odi = NULL;
3130 u64 dir_high_seq_ino = 0;
3131 u64 last_dir_index_offset = 0;
3132
3133 /*
3134 * Don't try to rmdir the top/root subvolume dir.
3135 */
3136 if (dir == BTRFS_FIRST_FREE_OBJECTID)
3137 return 0;
3138
3139 odi = get_orphan_dir_info(sctx, dir, dir_gen);
3140 if (odi && sctx->cur_ino < odi->dir_high_seq_ino)
3141 return 0;
3142
3143 path = alloc_path_for_send();
3144 if (!path)
3145 return -ENOMEM;
3146
3147 if (!odi) {
3148 /*
3149 * Find the inode number associated with the last dir index
3150 * entry. This is very likely the inode with the highest number
3151 * of all inodes that have an entry in the directory. We can
3152 * then use it to avoid future calls to can_rmdir(), when
3153 * processing inodes with a lower number, from having to search
3154 * the parent root b+tree for dir index keys.
3155 */
3156 key.objectid = dir;
3157 key.type = BTRFS_DIR_INDEX_KEY;
3158 key.offset = (u64)-1;
3159
3160 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3161 if (ret < 0) {
3162 goto out;
3163 } else if (ret > 0) {
3164 /* Can't happen, the root is never empty. */
3165 ASSERT(path->slots[0] > 0);
3166 if (WARN_ON(path->slots[0] == 0)) {
3167 ret = -EUCLEAN;
3168 goto out;
3169 }
3170 path->slots[0]--;
3171 }
3172
3173 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
3174 if (key.objectid != dir || key.type != BTRFS_DIR_INDEX_KEY) {
3175 /* No index keys, dir can be removed. */
3176 ret = 1;
3177 goto out;
3178 }
3179
3180 di = btrfs_item_ptr(path->nodes[0], path->slots[0],
3181 struct btrfs_dir_item);
3182 btrfs_dir_item_key_to_cpu(path->nodes[0], di, &loc);
3183 dir_high_seq_ino = loc.objectid;
3184 if (sctx->cur_ino < dir_high_seq_ino) {
3185 ret = 0;
3186 goto out;
3187 }
3188
3189 btrfs_release_path(path);
3190 }
3191
3192 key.objectid = dir;
3193 key.type = BTRFS_DIR_INDEX_KEY;
3194 key.offset = (odi ? odi->last_dir_index_offset : 0);
3195
3196 btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
3197 struct waiting_dir_move *dm;
3198
3199 if (found_key.objectid != key.objectid ||
3200 found_key.type != key.type)
3201 break;
3202
3203 di = btrfs_item_ptr(path->nodes[0], path->slots[0],
3204 struct btrfs_dir_item);
3205 btrfs_dir_item_key_to_cpu(path->nodes[0], di, &loc);
3206
3207 dir_high_seq_ino = max(dir_high_seq_ino, loc.objectid);
3208 last_dir_index_offset = found_key.offset;
3209
3210 dm = get_waiting_dir_move(sctx, loc.objectid);
3211 if (dm) {
3212 dm->rmdir_ino = dir;
3213 dm->rmdir_gen = dir_gen;
3214 ret = 0;
3215 goto out;
3216 }
3217
3218 if (loc.objectid > sctx->cur_ino) {
3219 ret = 0;
3220 goto out;
3221 }
3222 }
3223 if (iter_ret < 0) {
3224 ret = iter_ret;
3225 goto out;
3226 }
3227 free_orphan_dir_info(sctx, odi);
3228
3229 ret = 1;
3230
3231out:
3232 btrfs_free_path(path);
3233
3234 if (ret)
3235 return ret;
3236
3237 if (!odi) {
3238 odi = add_orphan_dir_info(sctx, dir, dir_gen);
3239 if (IS_ERR(odi))
3240 return PTR_ERR(odi);
3241
3242 odi->gen = dir_gen;
3243 }
3244
3245 odi->last_dir_index_offset = last_dir_index_offset;
3246 odi->dir_high_seq_ino = max(odi->dir_high_seq_ino, dir_high_seq_ino);
3247
3248 return 0;
3249}
3250
3251static int is_waiting_for_move(struct send_ctx *sctx, u64 ino)
3252{
3253 struct waiting_dir_move *entry = get_waiting_dir_move(sctx, ino);
3254
3255 return entry != NULL;
3256}
3257
3258static int add_waiting_dir_move(struct send_ctx *sctx, u64 ino, bool orphanized)
3259{
3260 struct rb_node **p = &sctx->waiting_dir_moves.rb_node;
3261 struct rb_node *parent = NULL;
3262 struct waiting_dir_move *entry, *dm;
3263
3264 dm = kmalloc_obj(*dm);
3265 if (!dm)
3266 return -ENOMEM;
3267 dm->ino = ino;
3268 dm->rmdir_ino = 0;
3269 dm->rmdir_gen = 0;
3270 dm->orphanized = orphanized;
3271
3272 while (*p) {
3273 parent = *p;
3274 entry = rb_entry(parent, struct waiting_dir_move, node);
3275 if (ino < entry->ino) {
3276 p = &(*p)->rb_left;
3277 } else if (ino > entry->ino) {
3278 p = &(*p)->rb_right;
3279 } else {
3280 kfree(dm);
3281 return -EEXIST;
3282 }
3283 }
3284
3285 rb_link_node(&dm->node, parent, p);
3286 rb_insert_color(&dm->node, &sctx->waiting_dir_moves);
3287 return 0;
3288}
3289
3290static struct waiting_dir_move *
3291get_waiting_dir_move(struct send_ctx *sctx, u64 ino)
3292{
3293 struct rb_node *n = sctx->waiting_dir_moves.rb_node;
3294 struct waiting_dir_move *entry;
3295
3296 while (n) {
3297 entry = rb_entry(n, struct waiting_dir_move, node);
3298 if (ino < entry->ino)
3299 n = n->rb_left;
3300 else if (ino > entry->ino)
3301 n = n->rb_right;
3302 else
3303 return entry;
3304 }
3305 return NULL;
3306}
3307
3308static void free_waiting_dir_move(struct send_ctx *sctx,
3309 struct waiting_dir_move *dm)
3310{
3311 if (!dm)
3312 return;
3313 rb_erase(&dm->node, &sctx->waiting_dir_moves);
3314 kfree(dm);
3315}
3316
3317static int add_pending_dir_move(struct send_ctx *sctx,
3318 u64 ino,
3319 u64 ino_gen,
3320 u64 parent_ino,
3321 struct list_head *new_refs,
3322 struct list_head *deleted_refs,
3323 const bool is_orphan)
3324{
3325 struct rb_node **p = &sctx->pending_dir_moves.rb_node;
3326 struct rb_node *parent = NULL;
3327 struct pending_dir_move *entry = NULL, *pm;
3328 struct recorded_ref *cur;
3329 bool exists = false;
3330 int ret;
3331
3332 pm = kmalloc_obj(*pm);
3333 if (!pm)
3334 return -ENOMEM;
3335 pm->parent_ino = parent_ino;
3336 pm->ino = ino;
3337 pm->gen = ino_gen;
3338 INIT_LIST_HEAD(&pm->list);
3339 INIT_LIST_HEAD(&pm->update_refs);
3340 RB_CLEAR_NODE(&pm->node);
3341
3342 while (*p) {
3343 parent = *p;
3344 entry = rb_entry(parent, struct pending_dir_move, node);
3345 if (parent_ino < entry->parent_ino) {
3346 p = &(*p)->rb_left;
3347 } else if (parent_ino > entry->parent_ino) {
3348 p = &(*p)->rb_right;
3349 } else {
3350 exists = true;
3351 break;
3352 }
3353 }
3354
3355 list_for_each_entry(cur, deleted_refs, list) {
3356 ret = dup_ref(cur, &pm->update_refs);
3357 if (ret < 0)
3358 goto out;
3359 }
3360 list_for_each_entry(cur, new_refs, list) {
3361 ret = dup_ref(cur, &pm->update_refs);
3362 if (ret < 0)
3363 goto out;
3364 }
3365
3366 ret = add_waiting_dir_move(sctx, pm->ino, is_orphan);
3367 if (ret)
3368 goto out;
3369
3370 if (exists) {
3371 list_add_tail(&pm->list, &entry->list);
3372 } else {
3373 rb_link_node(&pm->node, parent, p);
3374 rb_insert_color(&pm->node, &sctx->pending_dir_moves);
3375 }
3376 ret = 0;
3377out:
3378 if (ret) {
3379 __free_recorded_refs(&pm->update_refs);
3380 kfree(pm);
3381 }
3382 return ret;
3383}
3384
3385static struct pending_dir_move *get_pending_dir_moves(struct send_ctx *sctx,
3386 u64 parent_ino)
3387{
3388 struct rb_node *n = sctx->pending_dir_moves.rb_node;
3389 struct pending_dir_move *entry;
3390
3391 while (n) {
3392 entry = rb_entry(n, struct pending_dir_move, node);
3393 if (parent_ino < entry->parent_ino)
3394 n = n->rb_left;
3395 else if (parent_ino > entry->parent_ino)
3396 n = n->rb_right;
3397 else
3398 return entry;
3399 }
3400 return NULL;
3401}
3402
3403static int path_loop(struct send_ctx *sctx, struct fs_path *name,
3404 u64 ino, u64 gen, u64 *ancestor_ino)
3405{
3406 int ret = 0;
3407 u64 parent_inode = 0;
3408 u64 parent_gen = 0;
3409 u64 start_ino = ino;
3410
3411 *ancestor_ino = 0;
3412 while (ino != BTRFS_FIRST_FREE_OBJECTID) {
3413 fs_path_reset(name);
3414
3415 if (is_waiting_for_rm(sctx, ino, gen))
3416 break;
3417 if (is_waiting_for_move(sctx, ino)) {
3418 if (*ancestor_ino == 0)
3419 *ancestor_ino = ino;
3420 ret = get_first_ref(sctx->parent_root, ino,
3421 &parent_inode, &parent_gen, name);
3422 } else {
3423 ret = __get_cur_name_and_parent(sctx, ino, gen,
3424 &parent_inode,
3425 &parent_gen, name);
3426 if (ret > 0) {
3427 ret = 0;
3428 break;
3429 }
3430 }
3431 if (ret < 0)
3432 break;
3433 if (parent_inode == start_ino) {
3434 ret = 1;
3435 if (*ancestor_ino == 0)
3436 *ancestor_ino = ino;
3437 break;
3438 }
3439 ino = parent_inode;
3440 gen = parent_gen;
3441 }
3442 return ret;
3443}
3444
3445static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm)
3446{
3447 struct fs_path *from_path __free(fs_path_free) = NULL;
3448 struct fs_path *to_path __free(fs_path_free) = NULL;
3449 struct fs_path *name __free(fs_path_free) = NULL;
3450 u64 orig_progress = sctx->send_progress;
3451 struct recorded_ref *cur;
3452 u64 parent_ino, parent_gen;
3453 struct waiting_dir_move *dm = NULL;
3454 u64 rmdir_ino = 0;
3455 u64 rmdir_gen;
3456 u64 ancestor;
3457 bool is_orphan;
3458 int ret;
3459
3460 name = fs_path_alloc();
3461 from_path = fs_path_alloc();
3462 if (!name || !from_path)
3463 return -ENOMEM;
3464
3465 dm = get_waiting_dir_move(sctx, pm->ino);
3466 ASSERT(dm);
3467 rmdir_ino = dm->rmdir_ino;
3468 rmdir_gen = dm->rmdir_gen;
3469 is_orphan = dm->orphanized;
3470 free_waiting_dir_move(sctx, dm);
3471
3472 if (is_orphan) {
3473 ret = gen_unique_name(sctx, pm->ino,
3474 pm->gen, from_path);
3475 } else {
3476 ret = get_first_ref(sctx->parent_root, pm->ino,
3477 &parent_ino, &parent_gen, name);
3478 if (ret < 0)
3479 goto out;
3480 ret = get_cur_path(sctx, parent_ino, parent_gen,
3481 from_path);
3482 if (ret < 0)
3483 goto out;
3484 ret = fs_path_add_path(from_path, name);
3485 }
3486 if (ret < 0)
3487 goto out;
3488
3489 sctx->send_progress = sctx->cur_ino + 1;
3490 ret = path_loop(sctx, name, pm->ino, pm->gen, &ancestor);
3491 if (ret < 0)
3492 goto out;
3493 if (ret) {
3494 LIST_HEAD(deleted_refs);
3495 ASSERT(ancestor > BTRFS_FIRST_FREE_OBJECTID);
3496 ret = add_pending_dir_move(sctx, pm->ino, pm->gen, ancestor,
3497 &pm->update_refs, &deleted_refs,
3498 is_orphan);
3499 if (ret < 0)
3500 goto out;
3501 if (rmdir_ino) {
3502 dm = get_waiting_dir_move(sctx, pm->ino);
3503 ASSERT(dm);
3504 dm->rmdir_ino = rmdir_ino;
3505 dm->rmdir_gen = rmdir_gen;
3506 }
3507 goto out;
3508 }
3509 fs_path_reset(name);
3510 to_path = name;
3511 name = NULL;
3512 ret = get_cur_path(sctx, pm->ino, pm->gen, to_path);
3513 if (ret < 0)
3514 goto out;
3515
3516 ret = send_rename(sctx, from_path, to_path);
3517 if (ret < 0)
3518 goto out;
3519
3520 if (rmdir_ino) {
3521 struct orphan_dir_info *odi;
3522 u64 gen;
3523
3524 odi = get_orphan_dir_info(sctx, rmdir_ino, rmdir_gen);
3525 if (!odi) {
3526 /* already deleted */
3527 goto finish;
3528 }
3529 gen = odi->gen;
3530
3531 ret = can_rmdir(sctx, rmdir_ino, gen);
3532 if (ret < 0)
3533 goto out;
3534 if (!ret)
3535 goto finish;
3536
3537 name = fs_path_alloc();
3538 if (!name) {
3539 ret = -ENOMEM;
3540 goto out;
3541 }
3542 ret = get_cur_path(sctx, rmdir_ino, gen, name);
3543 if (ret < 0)
3544 goto out;
3545 ret = send_rmdir(sctx, name);
3546 if (ret < 0)
3547 goto out;
3548 }
3549
3550finish:
3551 ret = cache_dir_utimes(sctx, pm->ino, pm->gen);
3552 if (ret < 0)
3553 goto out;
3554
3555 /*
3556 * After rename/move, need to update the utimes of both new parent(s)
3557 * and old parent(s).
3558 */
3559 list_for_each_entry(cur, &pm->update_refs, list) {
3560 /*
3561 * The parent inode might have been deleted in the send snapshot
3562 */
3563 ret = get_inode_info(sctx->send_root, cur->dir, NULL);
3564 if (ret == -ENOENT) {
3565 ret = 0;
3566 continue;
3567 }
3568 if (ret < 0)
3569 goto out;
3570
3571 ret = cache_dir_utimes(sctx, cur->dir, cur->dir_gen);
3572 if (ret < 0)
3573 goto out;
3574 }
3575
3576out:
3577 sctx->send_progress = orig_progress;
3578
3579 return ret;
3580}
3581
3582static void free_pending_move(struct send_ctx *sctx, struct pending_dir_move *m)
3583{
3584 if (!list_empty(&m->list))
3585 list_del(&m->list);
3586 if (!RB_EMPTY_NODE(&m->node))
3587 rb_erase(&m->node, &sctx->pending_dir_moves);
3588 __free_recorded_refs(&m->update_refs);
3589 kfree(m);
3590}
3591
3592static void tail_append_pending_moves(struct send_ctx *sctx,
3593 struct pending_dir_move *moves,
3594 struct list_head *stack)
3595{
3596 if (list_empty(&moves->list)) {
3597 list_add_tail(&moves->list, stack);
3598 } else {
3599 LIST_HEAD(list);
3600 list_splice_init(&moves->list, &list);
3601 list_add_tail(&moves->list, stack);
3602 list_splice_tail(&list, stack);
3603 }
3604 if (!RB_EMPTY_NODE(&moves->node)) {
3605 rb_erase(&moves->node, &sctx->pending_dir_moves);
3606 RB_CLEAR_NODE(&moves->node);
3607 }
3608}
3609
3610static int apply_children_dir_moves(struct send_ctx *sctx)
3611{
3612 struct pending_dir_move *pm;
3613 LIST_HEAD(stack);
3614 u64 parent_ino = sctx->cur_ino;
3615 int ret = 0;
3616
3617 pm = get_pending_dir_moves(sctx, parent_ino);
3618 if (!pm)
3619 return 0;
3620
3621 tail_append_pending_moves(sctx, pm, &stack);
3622
3623 while (!list_empty(&stack)) {
3624 pm = list_first_entry(&stack, struct pending_dir_move, list);
3625 parent_ino = pm->ino;
3626 ret = apply_dir_move(sctx, pm);
3627 free_pending_move(sctx, pm);
3628 if (ret)
3629 goto out;
3630 pm = get_pending_dir_moves(sctx, parent_ino);
3631 if (pm)
3632 tail_append_pending_moves(sctx, pm, &stack);
3633 }
3634 return 0;
3635
3636out:
3637 while (!list_empty(&stack)) {
3638 pm = list_first_entry(&stack, struct pending_dir_move, list);
3639 free_pending_move(sctx, pm);
3640 }
3641 return ret;
3642}
3643
3644/*
3645 * We might need to delay a directory rename even when no ancestor directory
3646 * (in the send root) with a higher inode number than ours (sctx->cur_ino) was
3647 * renamed. This happens when we rename a directory to the old name (the name
3648 * in the parent root) of some other unrelated directory that got its rename
3649 * delayed due to some ancestor with higher number that got renamed.
3650 *
3651 * Example:
3652 *
3653 * Parent snapshot:
3654 * . (ino 256)
3655 * |---- a/ (ino 257)
3656 * | |---- file (ino 260)
3657 * |
3658 * |---- b/ (ino 258)
3659 * |---- c/ (ino 259)
3660 *
3661 * Send snapshot:
3662 * . (ino 256)
3663 * |---- a/ (ino 258)
3664 * |---- x/ (ino 259)
3665 * |---- y/ (ino 257)
3666 * |----- file (ino 260)
3667 *
3668 * Here we can not rename 258 from 'b' to 'a' without the rename of inode 257
3669 * from 'a' to 'x/y' happening first, which in turn depends on the rename of
3670 * inode 259 from 'c' to 'x'. So the order of rename commands the send stream
3671 * must issue is:
3672 *
3673 * 1 - rename 259 from 'c' to 'x'
3674 * 2 - rename 257 from 'a' to 'x/y'
3675 * 3 - rename 258 from 'b' to 'a'
3676 *
3677 * Returns 1 if the rename of sctx->cur_ino needs to be delayed, 0 if it can
3678 * be done right away and < 0 on error.
3679 */
3680static int wait_for_dest_dir_move(struct send_ctx *sctx,
3681 struct recorded_ref *parent_ref,
3682 const bool is_orphan)
3683{
3684 BTRFS_PATH_AUTO_FREE(path);
3685 struct btrfs_key key;
3686 struct btrfs_key di_key;
3687 struct btrfs_dir_item *di;
3688 u64 left_gen;
3689 u64 right_gen;
3690 int ret = 0;
3691 struct waiting_dir_move *wdm;
3692
3693 if (RB_EMPTY_ROOT(&sctx->waiting_dir_moves))
3694 return 0;
3695
3696 path = alloc_path_for_send();
3697 if (!path)
3698 return -ENOMEM;
3699
3700 key.objectid = parent_ref->dir;
3701 key.type = BTRFS_DIR_ITEM_KEY;
3702 key.offset = btrfs_name_hash(parent_ref->name, parent_ref->name_len);
3703
3704 ret = btrfs_search_slot(NULL, sctx->parent_root, &key, path, 0, 0);
3705 if (ret < 0)
3706 return ret;
3707 if (ret > 0)
3708 return 0;
3709
3710 di = btrfs_match_dir_item_name(path, parent_ref->name,
3711 parent_ref->name_len);
3712 if (!di)
3713 return 0;
3714 /*
3715 * di_key.objectid has the number of the inode that has a dentry in the
3716 * parent directory with the same name that sctx->cur_ino is being
3717 * renamed to. We need to check if that inode is in the send root as
3718 * well and if it is currently marked as an inode with a pending rename,
3719 * if it is, we need to delay the rename of sctx->cur_ino as well, so
3720 * that it happens after that other inode is renamed.
3721 */
3722 btrfs_dir_item_key_to_cpu(path->nodes[0], di, &di_key);
3723 if (di_key.type != BTRFS_INODE_ITEM_KEY)
3724 return 0;
3725
3726 ret = get_inode_gen(sctx->parent_root, di_key.objectid, &left_gen);
3727 if (ret < 0)
3728 return ret;
3729 ret = get_inode_gen(sctx->send_root, di_key.objectid, &right_gen);
3730 if (ret < 0) {
3731 if (ret == -ENOENT)
3732 ret = 0;
3733 return ret;
3734 }
3735
3736 /* Different inode, no need to delay the rename of sctx->cur_ino */
3737 if (right_gen != left_gen)
3738 return 0;
3739
3740 wdm = get_waiting_dir_move(sctx, di_key.objectid);
3741 if (wdm && !wdm->orphanized) {
3742 ret = add_pending_dir_move(sctx,
3743 sctx->cur_ino,
3744 sctx->cur_inode_gen,
3745 di_key.objectid,
3746 &sctx->new_refs,
3747 &sctx->deleted_refs,
3748 is_orphan);
3749 if (!ret)
3750 ret = 1;
3751 }
3752 return ret;
3753}
3754
3755/*
3756 * Check if inode ino2, or any of its ancestors, is inode ino1.
3757 * Return 1 if true, 0 if false and < 0 on error.
3758 */
3759static int check_ino_in_path(struct btrfs_root *root,
3760 const u64 ino1,
3761 const u64 ino1_gen,
3762 const u64 ino2,
3763 const u64 ino2_gen,
3764 struct fs_path *fs_path)
3765{
3766 u64 ino = ino2;
3767
3768 if (ino1 == ino2)
3769 return ino1_gen == ino2_gen;
3770
3771 while (ino > BTRFS_FIRST_FREE_OBJECTID) {
3772 u64 parent;
3773 u64 parent_gen;
3774 int ret;
3775
3776 fs_path_reset(fs_path);
3777 ret = get_first_ref(root, ino, &parent, &parent_gen, fs_path);
3778 if (ret < 0)
3779 return ret;
3780 if (parent == ino1)
3781 return parent_gen == ino1_gen;
3782 ino = parent;
3783 }
3784 return 0;
3785}
3786
3787/*
3788 * Check if inode ino1 is an ancestor of inode ino2 in the given root for any
3789 * possible path (in case ino2 is not a directory and has multiple hard links).
3790 * Return 1 if true, 0 if false and < 0 on error.
3791 */
3792static int is_ancestor(struct btrfs_root *root,
3793 const u64 ino1,
3794 const u64 ino1_gen,
3795 const u64 ino2,
3796 struct fs_path *fs_path)
3797{
3798 bool free_fs_path = false;
3799 int ret = 0;
3800 int iter_ret = 0;
3801 BTRFS_PATH_AUTO_FREE(path);
3802 struct btrfs_key key;
3803
3804 if (!fs_path) {
3805 fs_path = fs_path_alloc();
3806 if (!fs_path)
3807 return -ENOMEM;
3808 free_fs_path = true;
3809 }
3810
3811 path = alloc_path_for_send();
3812 if (!path) {
3813 ret = -ENOMEM;
3814 goto out;
3815 }
3816
3817 key.objectid = ino2;
3818 key.type = BTRFS_INODE_REF_KEY;
3819 key.offset = 0;
3820
3821 btrfs_for_each_slot(root, &key, &key, path, iter_ret) {
3822 struct extent_buffer *leaf = path->nodes[0];
3823 int slot = path->slots[0];
3824 u32 cur_offset = 0;
3825 u32 item_size;
3826
3827 if (key.objectid != ino2)
3828 break;
3829 if (key.type != BTRFS_INODE_REF_KEY &&
3830 key.type != BTRFS_INODE_EXTREF_KEY)
3831 break;
3832
3833 item_size = btrfs_item_size(leaf, slot);
3834 while (cur_offset < item_size) {
3835 u64 parent;
3836 u64 parent_gen;
3837
3838 if (key.type == BTRFS_INODE_EXTREF_KEY) {
3839 unsigned long ptr;
3840 struct btrfs_inode_extref *extref;
3841
3842 ptr = btrfs_item_ptr_offset(leaf, slot);
3843 extref = (struct btrfs_inode_extref *)
3844 (ptr + cur_offset);
3845 parent = btrfs_inode_extref_parent(leaf,
3846 extref);
3847 cur_offset += sizeof(*extref);
3848 cur_offset += btrfs_inode_extref_name_len(leaf,
3849 extref);
3850 } else {
3851 parent = key.offset;
3852 cur_offset = item_size;
3853 }
3854
3855 ret = get_inode_gen(root, parent, &parent_gen);
3856 if (ret < 0)
3857 goto out;
3858 ret = check_ino_in_path(root, ino1, ino1_gen,
3859 parent, parent_gen, fs_path);
3860 if (ret)
3861 goto out;
3862 }
3863 }
3864 ret = 0;
3865 if (iter_ret < 0)
3866 ret = iter_ret;
3867
3868out:
3869 if (free_fs_path)
3870 fs_path_free(fs_path);
3871 return ret;
3872}
3873
3874static int wait_for_parent_move(struct send_ctx *sctx,
3875 struct recorded_ref *parent_ref,
3876 const bool is_orphan)
3877{
3878 int ret = 0;
3879 u64 ino = parent_ref->dir;
3880 u64 ino_gen = parent_ref->dir_gen;
3881 u64 parent_ino_before, parent_ino_after;
3882 struct fs_path *path_before = NULL;
3883 struct fs_path *path_after = NULL;
3884 int len1, len2;
3885
3886 path_after = fs_path_alloc();
3887 path_before = fs_path_alloc();
3888 if (!path_after || !path_before) {
3889 ret = -ENOMEM;
3890 goto out;
3891 }
3892
3893 /*
3894 * Our current directory inode may not yet be renamed/moved because some
3895 * ancestor (immediate or not) has to be renamed/moved first. So find if
3896 * such ancestor exists and make sure our own rename/move happens after
3897 * that ancestor is processed to avoid path build infinite loops (done
3898 * at get_cur_path()).
3899 */
3900 while (ino > BTRFS_FIRST_FREE_OBJECTID) {
3901 u64 parent_ino_after_gen;
3902
3903 if (is_waiting_for_move(sctx, ino)) {
3904 /*
3905 * If the current inode is an ancestor of ino in the
3906 * parent root, we need to delay the rename of the
3907 * current inode, otherwise don't delayed the rename
3908 * because we can end up with a circular dependency
3909 * of renames, resulting in some directories never
3910 * getting the respective rename operations issued in
3911 * the send stream or getting into infinite path build
3912 * loops.
3913 */
3914 ret = is_ancestor(sctx->parent_root,
3915 sctx->cur_ino, sctx->cur_inode_gen,
3916 ino, path_before);
3917 if (ret)
3918 break;
3919 }
3920
3921 fs_path_reset(path_before);
3922 fs_path_reset(path_after);
3923
3924 ret = get_first_ref(sctx->send_root, ino, &parent_ino_after,
3925 &parent_ino_after_gen, path_after);
3926 if (ret < 0)
3927 goto out;
3928 ret = get_first_ref(sctx->parent_root, ino, &parent_ino_before,
3929 NULL, path_before);
3930 if (ret < 0 && ret != -ENOENT) {
3931 goto out;
3932 } else if (ret == -ENOENT) {
3933 ret = 0;
3934 break;
3935 }
3936
3937 len1 = fs_path_len(path_before);
3938 len2 = fs_path_len(path_after);
3939 if (ino > sctx->cur_ino &&
3940 (parent_ino_before != parent_ino_after || len1 != len2 ||
3941 memcmp(path_before->start, path_after->start, len1))) {
3942 u64 parent_ino_gen;
3943
3944 ret = get_inode_gen(sctx->parent_root, ino, &parent_ino_gen);
3945 if (ret < 0)
3946 goto out;
3947 if (ino_gen == parent_ino_gen) {
3948 ret = 1;
3949 break;
3950 }
3951 }
3952 ino = parent_ino_after;
3953 ino_gen = parent_ino_after_gen;
3954 }
3955
3956out:
3957 fs_path_free(path_before);
3958 fs_path_free(path_after);
3959
3960 if (ret == 1) {
3961 ret = add_pending_dir_move(sctx,
3962 sctx->cur_ino,
3963 sctx->cur_inode_gen,
3964 ino,
3965 &sctx->new_refs,
3966 &sctx->deleted_refs,
3967 is_orphan);
3968 if (!ret)
3969 ret = 1;
3970 }
3971
3972 return ret;
3973}
3974
3975static int update_ref_path(struct send_ctx *sctx, struct recorded_ref *ref)
3976{
3977 int ret;
3978 struct fs_path *new_path;
3979
3980 /*
3981 * Our reference's name member points to its full_path member string, so
3982 * we use here a new path.
3983 */
3984 new_path = fs_path_alloc();
3985 if (!new_path)
3986 return -ENOMEM;
3987
3988 ret = get_cur_path(sctx, ref->dir, ref->dir_gen, new_path);
3989 if (ret < 0) {
3990 fs_path_free(new_path);
3991 return ret;
3992 }
3993 ret = fs_path_add(new_path, ref->name, ref->name_len);
3994 if (ret < 0) {
3995 fs_path_free(new_path);
3996 return ret;
3997 }
3998
3999 fs_path_free(ref->full_path);
4000 set_ref_path(ref, new_path);
4001
4002 return 0;
4003}
4004
4005/*
4006 * When processing the new references for an inode we may orphanize an existing
4007 * directory inode because its old name conflicts with one of the new references
4008 * of the current inode. Later, when processing another new reference of our
4009 * inode, we might need to orphanize another inode, but the path we have in the
4010 * reference reflects the pre-orphanization name of the directory we previously
4011 * orphanized. For example:
4012 *
4013 * parent snapshot looks like:
4014 *
4015 * . (ino 256)
4016 * |----- f1 (ino 257)
4017 * |----- f2 (ino 258)
4018 * |----- d1/ (ino 259)
4019 * |----- d2/ (ino 260)
4020 *
4021 * send snapshot looks like:
4022 *
4023 * . (ino 256)
4024 * |----- d1 (ino 258)
4025 * |----- f2/ (ino 259)
4026 * |----- f2_link/ (ino 260)
4027 * | |----- f1 (ino 257)
4028 * |
4029 * |----- d2 (ino 258)
4030 *
4031 * When processing inode 257 we compute the name for inode 259 as "d1", and we
4032 * cache it in the name cache. Later when we start processing inode 258, when
4033 * collecting all its new references we set a full path of "d1/d2" for its new
4034 * reference with name "d2". When we start processing the new references we
4035 * start by processing the new reference with name "d1", and this results in
4036 * orphanizing inode 259, since its old reference causes a conflict. Then we
4037 * move on the next new reference, with name "d2", and we find out we must
4038 * orphanize inode 260, as its old reference conflicts with ours - but for the
4039 * orphanization we use a source path corresponding to the path we stored in the
4040 * new reference, which is "d1/d2" and not "o259-6-0/d2" - this makes the
4041 * receiver fail since the path component "d1/" no longer exists, it was renamed
4042 * to "o259-6-0/" when processing the previous new reference. So in this case we
4043 * must recompute the path in the new reference and use it for the new
4044 * orphanization operation.
4045 */
4046static int refresh_ref_path(struct send_ctx *sctx, struct recorded_ref *ref)
4047{
4048 char AUTO_KFREE(name);
4049 int ret;
4050
4051 name = kmemdup(ref->name, ref->name_len, GFP_KERNEL);
4052 if (!name)
4053 return -ENOMEM;
4054
4055 fs_path_reset(ref->full_path);
4056 ret = get_cur_path(sctx, ref->dir, ref->dir_gen, ref->full_path);
4057 if (ret < 0)
4058 return ret;
4059
4060 ret = fs_path_add(ref->full_path, name, ref->name_len);
4061 if (ret < 0)
4062 return ret;
4063
4064 /* Update the reference's base name pointer. */
4065 set_ref_path(ref, ref->full_path);
4066
4067 return 0;
4068}
4069
4070static int rbtree_check_dir_ref_comp(const void *k, const struct rb_node *node)
4071{
4072 const struct recorded_ref *data = k;
4073 const struct recorded_ref *ref = rb_entry(node, struct recorded_ref, node);
4074
4075 if (data->dir > ref->dir)
4076 return 1;
4077 if (data->dir < ref->dir)
4078 return -1;
4079 if (data->dir_gen > ref->dir_gen)
4080 return 1;
4081 if (data->dir_gen < ref->dir_gen)
4082 return -1;
4083 return 0;
4084}
4085
4086static bool rbtree_check_dir_ref_less(struct rb_node *node, const struct rb_node *parent)
4087{
4088 const struct recorded_ref *entry = rb_entry(node, struct recorded_ref, node);
4089
4090 return rbtree_check_dir_ref_comp(entry, parent) < 0;
4091}
4092
4093static int record_check_dir_ref_in_tree(struct rb_root *root,
4094 struct recorded_ref *ref, struct list_head *list)
4095{
4096 struct recorded_ref *tmp_ref;
4097 int ret;
4098
4099 if (rb_find(ref, root, rbtree_check_dir_ref_comp))
4100 return 0;
4101
4102 ret = dup_ref(ref, list);
4103 if (ret < 0)
4104 return ret;
4105
4106 tmp_ref = list_last_entry(list, struct recorded_ref, list);
4107 rb_add(&tmp_ref->node, root, rbtree_check_dir_ref_less);
4108 tmp_ref->root = root;
4109 return 0;
4110}
4111
4112static int rename_current_inode(struct send_ctx *sctx,
4113 struct fs_path *current_path,
4114 struct fs_path *new_path)
4115{
4116 int ret;
4117
4118 ret = send_rename(sctx, current_path, new_path);
4119 if (ret < 0)
4120 return ret;
4121
4122 ret = fs_path_copy(&sctx->cur_inode_path, new_path);
4123 if (ret < 0)
4124 return ret;
4125
4126 return fs_path_copy(current_path, new_path);
4127}
4128
4129/*
4130 * This does all the move/link/unlink/rmdir magic.
4131 */
4132static int process_recorded_refs(struct send_ctx *sctx, bool *pending_move)
4133{
4134 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
4135 int ret = 0;
4136 struct recorded_ref *cur;
4137 struct recorded_ref *cur2;
4138 LIST_HEAD(check_dirs);
4139 struct rb_root rbtree_check_dirs = RB_ROOT;
4140 struct fs_path *valid_path = NULL;
4141 u64 ow_inode = 0;
4142 u64 ow_gen;
4143 u64 ow_mode;
4144 bool did_overwrite = false;
4145 bool is_orphan = false;
4146 bool can_rename = true;
4147 bool orphanized_dir = false;
4148 bool orphanized_ancestor = false;
4149
4150 /*
4151 * This should never happen as the root dir always has the same ref
4152 * which is always '..'
4153 */
4154 if (unlikely(sctx->cur_ino <= BTRFS_FIRST_FREE_OBJECTID)) {
4155 btrfs_err(fs_info,
4156 "send: unexpected inode %llu in process_recorded_refs()",
4157 sctx->cur_ino);
4158 ret = -EINVAL;
4159 goto out;
4160 }
4161
4162 valid_path = fs_path_alloc();
4163 if (!valid_path) {
4164 ret = -ENOMEM;
4165 goto out;
4166 }
4167
4168 /*
4169 * First, check if the first ref of the current inode was overwritten
4170 * before. If yes, we know that the current inode was already orphanized
4171 * and thus use the orphan name. If not, we can use get_cur_path to
4172 * get the path of the first ref as it would like while receiving at
4173 * this point in time.
4174 * New inodes are always orphan at the beginning, so force to use the
4175 * orphan name in this case.
4176 * The first ref is stored in valid_path and will be updated if it
4177 * gets moved around.
4178 */
4179 if (!sctx->cur_inode_new) {
4180 ret = did_overwrite_first_ref(sctx, sctx->cur_ino,
4181 sctx->cur_inode_gen);
4182 if (ret < 0)
4183 goto out;
4184 if (ret)
4185 did_overwrite = true;
4186 }
4187 if (sctx->cur_inode_new || did_overwrite) {
4188 ret = gen_unique_name(sctx, sctx->cur_ino,
4189 sctx->cur_inode_gen, valid_path);
4190 if (ret < 0)
4191 goto out;
4192 is_orphan = true;
4193 } else {
4194 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen,
4195 valid_path);
4196 if (ret < 0)
4197 goto out;
4198 }
4199
4200 /*
4201 * Before doing any rename and link operations, do a first pass on the
4202 * new references to orphanize any unprocessed inodes that may have a
4203 * reference that conflicts with one of the new references of the current
4204 * inode. This needs to happen first because a new reference may conflict
4205 * with the old reference of a parent directory, so we must make sure
4206 * that the path used for link and rename commands don't use an
4207 * orphanized name when an ancestor was not yet orphanized.
4208 *
4209 * Example:
4210 *
4211 * Parent snapshot:
4212 *
4213 * . (ino 256)
4214 * |----- testdir/ (ino 259)
4215 * | |----- a (ino 257)
4216 * |
4217 * |----- b (ino 258)
4218 *
4219 * Send snapshot:
4220 *
4221 * . (ino 256)
4222 * |----- testdir_2/ (ino 259)
4223 * | |----- a (ino 260)
4224 * |
4225 * |----- testdir (ino 257)
4226 * |----- b (ino 257)
4227 * |----- b2 (ino 258)
4228 *
4229 * Processing the new reference for inode 257 with name "b" may happen
4230 * before processing the new reference with name "testdir". If so, we
4231 * must make sure that by the time we send a link command to create the
4232 * hard link "b", inode 259 was already orphanized, since the generated
4233 * path in "valid_path" already contains the orphanized name for 259.
4234 * We are processing inode 257, so only later when processing 259 we do
4235 * the rename operation to change its temporary (orphanized) name to
4236 * "testdir_2".
4237 */
4238 list_for_each_entry(cur, &sctx->new_refs, list) {
4239 ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL);
4240 if (ret < 0)
4241 goto out;
4242 if (ret == inode_state_will_create)
4243 continue;
4244
4245 /*
4246 * Check if this new ref would overwrite the first ref of another
4247 * unprocessed inode. If yes, orphanize the overwritten inode.
4248 * If we find an overwritten ref that is not the first ref,
4249 * simply unlink it.
4250 */
4251 ret = will_overwrite_ref(sctx, cur->dir, cur->dir_gen,
4252 cur->name, cur->name_len,
4253 &ow_inode, &ow_gen, &ow_mode);
4254 if (ret < 0)
4255 goto out;
4256 if (ret) {
4257 ret = is_first_ref(sctx->parent_root,
4258 ow_inode, cur->dir, cur->name,
4259 cur->name_len);
4260 if (ret < 0)
4261 goto out;
4262 if (ret) {
4263 struct name_cache_entry *nce;
4264 struct waiting_dir_move *wdm;
4265
4266 if (orphanized_dir) {
4267 ret = refresh_ref_path(sctx, cur);
4268 if (ret < 0)
4269 goto out;
4270 }
4271
4272 ret = orphanize_inode(sctx, ow_inode, ow_gen,
4273 cur->full_path);
4274 if (ret < 0)
4275 goto out;
4276 if (S_ISDIR(ow_mode))
4277 orphanized_dir = true;
4278
4279 /*
4280 * If ow_inode has its rename operation delayed
4281 * make sure that its orphanized name is used in
4282 * the source path when performing its rename
4283 * operation.
4284 */
4285 wdm = get_waiting_dir_move(sctx, ow_inode);
4286 if (wdm)
4287 wdm->orphanized = true;
4288
4289 /*
4290 * Make sure we clear our orphanized inode's
4291 * name from the name cache. This is because the
4292 * inode ow_inode might be an ancestor of some
4293 * other inode that will be orphanized as well
4294 * later and has an inode number greater than
4295 * sctx->send_progress. We need to prevent
4296 * future name lookups from using the old name
4297 * and get instead the orphan name.
4298 */
4299 nce = name_cache_search(sctx, ow_inode, ow_gen);
4300 if (nce)
4301 btrfs_lru_cache_remove(&sctx->name_cache,
4302 &nce->entry);
4303
4304 /*
4305 * ow_inode might currently be an ancestor of
4306 * cur_ino, therefore compute valid_path (the
4307 * current path of cur_ino) again because it
4308 * might contain the pre-orphanization name of
4309 * ow_inode, which is no longer valid.
4310 */
4311 ret = is_ancestor(sctx->parent_root,
4312 ow_inode, ow_gen,
4313 sctx->cur_ino, NULL);
4314 if (ret > 0) {
4315 orphanized_ancestor = true;
4316 fs_path_reset(valid_path);
4317 fs_path_reset(&sctx->cur_inode_path);
4318 ret = get_cur_path(sctx, sctx->cur_ino,
4319 sctx->cur_inode_gen,
4320 valid_path);
4321 }
4322 if (ret < 0)
4323 goto out;
4324 } else {
4325 /*
4326 * If we previously orphanized a directory that
4327 * collided with a new reference that we already
4328 * processed, recompute the current path because
4329 * that directory may be part of the path.
4330 */
4331 if (orphanized_dir) {
4332 ret = refresh_ref_path(sctx, cur);
4333 if (ret < 0)
4334 goto out;
4335 }
4336 ret = send_unlink(sctx, cur->full_path);
4337 if (ret < 0)
4338 goto out;
4339 }
4340 }
4341
4342 }
4343
4344 list_for_each_entry(cur, &sctx->new_refs, list) {
4345 /*
4346 * We may have refs where the parent directory does not exist
4347 * yet. This happens if the parent directories inum is higher
4348 * than the current inum. To handle this case, we create the
4349 * parent directory out of order. But we need to check if this
4350 * did already happen before due to other refs in the same dir.
4351 */
4352 ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL);
4353 if (ret < 0)
4354 goto out;
4355 if (ret == inode_state_will_create) {
4356 ret = 0;
4357 /*
4358 * First check if any of the current inodes refs did
4359 * already create the dir.
4360 */
4361 list_for_each_entry(cur2, &sctx->new_refs, list) {
4362 if (cur == cur2)
4363 break;
4364 if (cur2->dir == cur->dir) {
4365 ret = 1;
4366 break;
4367 }
4368 }
4369
4370 /*
4371 * If that did not happen, check if a previous inode
4372 * did already create the dir.
4373 */
4374 if (!ret)
4375 ret = did_create_dir(sctx, cur->dir);
4376 if (ret < 0)
4377 goto out;
4378 if (!ret) {
4379 ret = send_create_inode(sctx, cur->dir);
4380 if (ret < 0)
4381 goto out;
4382 cache_dir_created(sctx, cur->dir);
4383 }
4384 }
4385
4386 if (S_ISDIR(sctx->cur_inode_mode) && sctx->parent_root) {
4387 ret = wait_for_dest_dir_move(sctx, cur, is_orphan);
4388 if (ret < 0)
4389 goto out;
4390 if (ret == 1) {
4391 can_rename = false;
4392 *pending_move = true;
4393 }
4394 }
4395
4396 if (S_ISDIR(sctx->cur_inode_mode) && sctx->parent_root &&
4397 can_rename) {
4398 ret = wait_for_parent_move(sctx, cur, is_orphan);
4399 if (ret < 0)
4400 goto out;
4401 if (ret == 1) {
4402 can_rename = false;
4403 *pending_move = true;
4404 }
4405 }
4406
4407 /*
4408 * link/move the ref to the new place. If we have an orphan
4409 * inode, move it and update valid_path. If not, link or move
4410 * it depending on the inode mode.
4411 */
4412 if (is_orphan && can_rename) {
4413 ret = rename_current_inode(sctx, valid_path, cur->full_path);
4414 if (ret < 0)
4415 goto out;
4416 is_orphan = false;
4417 } else if (can_rename) {
4418 if (S_ISDIR(sctx->cur_inode_mode)) {
4419 /*
4420 * Dirs can't be linked, so move it. For moved
4421 * dirs, we always have one new and one deleted
4422 * ref. The deleted ref is ignored later.
4423 */
4424 ret = rename_current_inode(sctx, valid_path,
4425 cur->full_path);
4426 if (ret < 0)
4427 goto out;
4428 } else {
4429 /*
4430 * We might have previously orphanized an inode
4431 * which is an ancestor of our current inode,
4432 * so our reference's full path, which was
4433 * computed before any such orphanizations, must
4434 * be updated.
4435 */
4436 if (orphanized_dir) {
4437 ret = update_ref_path(sctx, cur);
4438 if (ret < 0)
4439 goto out;
4440 }
4441 ret = send_link(sctx, cur->full_path,
4442 valid_path);
4443 if (ret < 0)
4444 goto out;
4445 }
4446 }
4447 ret = record_check_dir_ref_in_tree(&rbtree_check_dirs, cur, &check_dirs);
4448 if (ret < 0)
4449 goto out;
4450 }
4451
4452 if (S_ISDIR(sctx->cur_inode_mode) && sctx->cur_inode_deleted) {
4453 /*
4454 * Check if we can already rmdir the directory. If not,
4455 * orphanize it. For every dir item inside that gets deleted
4456 * later, we do this check again and rmdir it then if possible.
4457 * See the use of check_dirs for more details.
4458 */
4459 ret = can_rmdir(sctx, sctx->cur_ino, sctx->cur_inode_gen);
4460 if (ret < 0)
4461 goto out;
4462 if (ret) {
4463 ret = send_rmdir(sctx, valid_path);
4464 if (ret < 0)
4465 goto out;
4466 } else if (!is_orphan) {
4467 ret = orphanize_inode(sctx, sctx->cur_ino,
4468 sctx->cur_inode_gen, valid_path);
4469 if (ret < 0)
4470 goto out;
4471 is_orphan = true;
4472 }
4473
4474 list_for_each_entry(cur, &sctx->deleted_refs, list) {
4475 ret = record_check_dir_ref_in_tree(&rbtree_check_dirs, cur, &check_dirs);
4476 if (ret < 0)
4477 goto out;
4478 }
4479 } else if (S_ISDIR(sctx->cur_inode_mode) &&
4480 !list_empty(&sctx->deleted_refs)) {
4481 /*
4482 * We have a moved dir. Add the old parent to check_dirs
4483 */
4484 cur = list_first_entry(&sctx->deleted_refs, struct recorded_ref, list);
4485 ret = record_check_dir_ref_in_tree(&rbtree_check_dirs, cur, &check_dirs);
4486 if (ret < 0)
4487 goto out;
4488 } else if (!S_ISDIR(sctx->cur_inode_mode)) {
4489 /*
4490 * We have a non dir inode. Go through all deleted refs and
4491 * unlink them if they were not already overwritten by other
4492 * inodes.
4493 */
4494 list_for_each_entry(cur, &sctx->deleted_refs, list) {
4495 ret = did_overwrite_ref(sctx, cur->dir, cur->dir_gen,
4496 sctx->cur_ino, sctx->cur_inode_gen,
4497 cur->name, cur->name_len);
4498 if (ret < 0)
4499 goto out;
4500 if (!ret) {
4501 /*
4502 * If we orphanized any ancestor before, we need
4503 * to recompute the full path for deleted names,
4504 * since any such path was computed before we
4505 * processed any references and orphanized any
4506 * ancestor inode.
4507 */
4508 if (orphanized_ancestor) {
4509 ret = update_ref_path(sctx, cur);
4510 if (ret < 0)
4511 goto out;
4512 }
4513 ret = send_unlink(sctx, cur->full_path);
4514 if (ret < 0)
4515 goto out;
4516 if (is_current_inode_path(sctx, cur->full_path))
4517 fs_path_reset(&sctx->cur_inode_path);
4518 }
4519 ret = record_check_dir_ref_in_tree(&rbtree_check_dirs, cur, &check_dirs);
4520 if (ret < 0)
4521 goto out;
4522 }
4523 /*
4524 * If the inode is still orphan, unlink the orphan. This may
4525 * happen when a previous inode did overwrite the first ref
4526 * of this inode and no new refs were added for the current
4527 * inode. Unlinking does not mean that the inode is deleted in
4528 * all cases. There may still be links to this inode in other
4529 * places.
4530 */
4531 if (is_orphan) {
4532 ret = send_unlink(sctx, valid_path);
4533 if (ret < 0)
4534 goto out;
4535 }
4536 }
4537
4538 /*
4539 * We did collect all parent dirs where cur_inode was once located. We
4540 * now go through all these dirs and check if they are pending for
4541 * deletion and if it's finally possible to perform the rmdir now.
4542 * We also update the inode stats of the parent dirs here.
4543 */
4544 list_for_each_entry(cur, &check_dirs, list) {
4545 /*
4546 * In case we had refs into dirs that were not processed yet,
4547 * we don't need to do the utime and rmdir logic for these dirs.
4548 * The dir will be processed later.
4549 */
4550 if (cur->dir > sctx->cur_ino)
4551 continue;
4552
4553 ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL);
4554 if (ret < 0)
4555 goto out;
4556
4557 if (ret == inode_state_did_create ||
4558 ret == inode_state_no_change) {
4559 ret = cache_dir_utimes(sctx, cur->dir, cur->dir_gen);
4560 if (ret < 0)
4561 goto out;
4562 } else if (ret == inode_state_did_delete) {
4563 ret = can_rmdir(sctx, cur->dir, cur->dir_gen);
4564 if (ret < 0)
4565 goto out;
4566 if (ret) {
4567 ret = get_cur_path(sctx, cur->dir,
4568 cur->dir_gen, valid_path);
4569 if (ret < 0)
4570 goto out;
4571 ret = send_rmdir(sctx, valid_path);
4572 if (ret < 0)
4573 goto out;
4574 }
4575 }
4576 }
4577
4578 ret = 0;
4579
4580out:
4581 __free_recorded_refs(&check_dirs);
4582 free_recorded_refs(sctx);
4583 fs_path_free(valid_path);
4584 return ret;
4585}
4586
4587static int rbtree_ref_comp(const void *k, const struct rb_node *node)
4588{
4589 const struct recorded_ref *data = k;
4590 const struct recorded_ref *ref = rb_entry(node, struct recorded_ref, node);
4591
4592 if (data->dir > ref->dir)
4593 return 1;
4594 if (data->dir < ref->dir)
4595 return -1;
4596 if (data->dir_gen > ref->dir_gen)
4597 return 1;
4598 if (data->dir_gen < ref->dir_gen)
4599 return -1;
4600 if (data->name_len > ref->name_len)
4601 return 1;
4602 if (data->name_len < ref->name_len)
4603 return -1;
4604 return strcmp(data->name, ref->name);
4605}
4606
4607static bool rbtree_ref_less(struct rb_node *node, const struct rb_node *parent)
4608{
4609 const struct recorded_ref *entry = rb_entry(node, struct recorded_ref, node);
4610
4611 return rbtree_ref_comp(entry, parent) < 0;
4612}
4613
4614static int record_ref_in_tree(struct rb_root *root, struct list_head *refs,
4615 struct fs_path *name, u64 dir, u64 dir_gen,
4616 struct send_ctx *sctx)
4617{
4618 int ret = 0;
4619 struct fs_path *path = NULL;
4620 struct recorded_ref *ref = NULL;
4621
4622 path = fs_path_alloc();
4623 if (!path) {
4624 ret = -ENOMEM;
4625 goto out;
4626 }
4627
4628 ref = recorded_ref_alloc();
4629 if (!ref) {
4630 ret = -ENOMEM;
4631 goto out;
4632 }
4633
4634 ret = get_cur_path(sctx, dir, dir_gen, path);
4635 if (ret < 0)
4636 goto out;
4637 ret = fs_path_add_path(path, name);
4638 if (ret < 0)
4639 goto out;
4640
4641 ref->dir = dir;
4642 ref->dir_gen = dir_gen;
4643 set_ref_path(ref, path);
4644 list_add_tail(&ref->list, refs);
4645 rb_add(&ref->node, root, rbtree_ref_less);
4646 ref->root = root;
4647out:
4648 if (ret) {
4649 if (path && (!ref || !ref->full_path))
4650 fs_path_free(path);
4651 recorded_ref_free(ref);
4652 }
4653 return ret;
4654}
4655
4656static int record_new_ref_if_needed(u64 dir, struct fs_path *name, void *ctx)
4657{
4658 int ret;
4659 struct send_ctx *sctx = ctx;
4660 struct rb_node *node = NULL;
4661 struct recorded_ref data;
4662 struct recorded_ref *ref;
4663 u64 dir_gen;
4664
4665 ret = get_inode_gen(sctx->send_root, dir, &dir_gen);
4666 if (ret < 0)
4667 return ret;
4668
4669 data.dir = dir;
4670 data.dir_gen = dir_gen;
4671 set_ref_path(&data, name);
4672 node = rb_find(&data, &sctx->rbtree_deleted_refs, rbtree_ref_comp);
4673 if (node) {
4674 ref = rb_entry(node, struct recorded_ref, node);
4675 recorded_ref_free(ref);
4676 } else {
4677 ret = record_ref_in_tree(&sctx->rbtree_new_refs,
4678 &sctx->new_refs, name, dir, dir_gen,
4679 sctx);
4680 }
4681
4682 return ret;
4683}
4684
4685static int record_deleted_ref_if_needed(u64 dir, struct fs_path *name, void *ctx)
4686{
4687 int ret;
4688 struct send_ctx *sctx = ctx;
4689 struct rb_node *node = NULL;
4690 struct recorded_ref data;
4691 struct recorded_ref *ref;
4692 u64 dir_gen;
4693
4694 ret = get_inode_gen(sctx->parent_root, dir, &dir_gen);
4695 if (ret < 0)
4696 return ret;
4697
4698 data.dir = dir;
4699 data.dir_gen = dir_gen;
4700 set_ref_path(&data, name);
4701 node = rb_find(&data, &sctx->rbtree_new_refs, rbtree_ref_comp);
4702 if (node) {
4703 ref = rb_entry(node, struct recorded_ref, node);
4704 recorded_ref_free(ref);
4705 } else {
4706 ret = record_ref_in_tree(&sctx->rbtree_deleted_refs,
4707 &sctx->deleted_refs, name, dir,
4708 dir_gen, sctx);
4709 }
4710
4711 return ret;
4712}
4713
4714static int record_new_ref(struct send_ctx *sctx)
4715{
4716 int ret;
4717
4718 ret = iterate_inode_ref(sctx->send_root, sctx->left_path, sctx->cmp_key,
4719 false, record_new_ref_if_needed, sctx);
4720 if (ret < 0)
4721 return ret;
4722
4723 return 0;
4724}
4725
4726static int record_deleted_ref(struct send_ctx *sctx)
4727{
4728 int ret;
4729
4730 ret = iterate_inode_ref(sctx->parent_root, sctx->right_path, sctx->cmp_key,
4731 false, record_deleted_ref_if_needed, sctx);
4732 if (ret < 0)
4733 return ret;
4734
4735 return 0;
4736}
4737
4738static int record_changed_ref(struct send_ctx *sctx)
4739{
4740 int ret;
4741
4742 ret = iterate_inode_ref(sctx->send_root, sctx->left_path, sctx->cmp_key,
4743 false, record_new_ref_if_needed, sctx);
4744 if (ret < 0)
4745 return ret;
4746 ret = iterate_inode_ref(sctx->parent_root, sctx->right_path, sctx->cmp_key,
4747 false, record_deleted_ref_if_needed, sctx);
4748 if (ret < 0)
4749 return ret;
4750
4751 return 0;
4752}
4753
4754/*
4755 * Record and process all refs at once. Needed when an inode changes the
4756 * generation number, which means that it was deleted and recreated.
4757 */
4758static int process_all_refs(struct send_ctx *sctx,
4759 enum btrfs_compare_tree_result cmd)
4760{
4761 int ret = 0;
4762 int iter_ret = 0;
4763 struct btrfs_root *root;
4764 BTRFS_PATH_AUTO_FREE(path);
4765 struct btrfs_key key;
4766 struct btrfs_key found_key;
4767 iterate_inode_ref_t cb;
4768 bool pending_move = false;
4769
4770 path = alloc_path_for_send();
4771 if (!path)
4772 return -ENOMEM;
4773
4774 if (cmd == BTRFS_COMPARE_TREE_NEW) {
4775 root = sctx->send_root;
4776 cb = record_new_ref_if_needed;
4777 } else if (cmd == BTRFS_COMPARE_TREE_DELETED) {
4778 root = sctx->parent_root;
4779 cb = record_deleted_ref_if_needed;
4780 } else {
4781 btrfs_err(sctx->send_root->fs_info,
4782 "Wrong command %d in process_all_refs", cmd);
4783 return -EINVAL;
4784 }
4785
4786 key.objectid = sctx->cmp_key->objectid;
4787 key.type = BTRFS_INODE_REF_KEY;
4788 key.offset = 0;
4789 btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
4790 if (found_key.objectid != key.objectid ||
4791 (found_key.type != BTRFS_INODE_REF_KEY &&
4792 found_key.type != BTRFS_INODE_EXTREF_KEY))
4793 break;
4794
4795 ret = iterate_inode_ref(root, path, &found_key, false, cb, sctx);
4796 if (ret < 0)
4797 return ret;
4798 }
4799 /* Catch error found during iteration */
4800 if (iter_ret < 0)
4801 return iter_ret;
4802
4803 btrfs_release_path(path);
4804
4805 /*
4806 * We don't actually care about pending_move as we are simply
4807 * re-creating this inode and will be rename'ing it into place once we
4808 * rename the parent directory.
4809 */
4810 return process_recorded_refs(sctx, &pending_move);
4811}
4812
4813static int send_set_xattr(struct send_ctx *sctx,
4814 const char *name, int name_len,
4815 const char *data, int data_len)
4816{
4817 struct fs_path *path;
4818 int ret;
4819
4820 path = get_cur_inode_path(sctx);
4821 if (IS_ERR(path))
4822 return PTR_ERR(path);
4823
4824 ret = begin_cmd(sctx, BTRFS_SEND_C_SET_XATTR);
4825 if (ret < 0)
4826 return ret;
4827
4828 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
4829 TLV_PUT_STRING(sctx, BTRFS_SEND_A_XATTR_NAME, name, name_len);
4830 TLV_PUT(sctx, BTRFS_SEND_A_XATTR_DATA, data, data_len);
4831
4832 ret = send_cmd(sctx);
4833
4834tlv_put_failure:
4835 return ret;
4836}
4837
4838static int send_remove_xattr(struct send_ctx *sctx,
4839 struct fs_path *path,
4840 const char *name, int name_len)
4841{
4842 int ret;
4843
4844 ret = begin_cmd(sctx, BTRFS_SEND_C_REMOVE_XATTR);
4845 if (ret < 0)
4846 return ret;
4847
4848 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
4849 TLV_PUT_STRING(sctx, BTRFS_SEND_A_XATTR_NAME, name, name_len);
4850
4851 ret = send_cmd(sctx);
4852
4853tlv_put_failure:
4854 return ret;
4855}
4856
4857static int __process_new_xattr(int num, struct btrfs_key *di_key,
4858 const char *name, int name_len, const char *data,
4859 int data_len, void *ctx)
4860{
4861 struct send_ctx *sctx = ctx;
4862 struct posix_acl_xattr_header dummy_acl;
4863
4864 /* Capabilities are emitted by finish_inode_if_needed */
4865 if (!strncmp(name, XATTR_NAME_CAPS, name_len))
4866 return 0;
4867
4868 /*
4869 * This hack is needed because empty acls are stored as zero byte
4870 * data in xattrs. Problem with that is, that receiving these zero byte
4871 * acls will fail later. To fix this, we send a dummy acl list that
4872 * only contains the version number and no entries.
4873 */
4874 if (!strncmp(name, XATTR_NAME_POSIX_ACL_ACCESS, name_len) ||
4875 !strncmp(name, XATTR_NAME_POSIX_ACL_DEFAULT, name_len)) {
4876 if (data_len == 0) {
4877 dummy_acl.a_version =
4878 cpu_to_le32(POSIX_ACL_XATTR_VERSION);
4879 data = (char *)&dummy_acl;
4880 data_len = sizeof(dummy_acl);
4881 }
4882 }
4883
4884 return send_set_xattr(sctx, name, name_len, data, data_len);
4885}
4886
4887static int __process_deleted_xattr(int num, struct btrfs_key *di_key,
4888 const char *name, int name_len,
4889 const char *data, int data_len, void *ctx)
4890{
4891 struct send_ctx *sctx = ctx;
4892 struct fs_path *p;
4893
4894 p = get_cur_inode_path(sctx);
4895 if (IS_ERR(p))
4896 return PTR_ERR(p);
4897
4898 return send_remove_xattr(sctx, p, name, name_len);
4899}
4900
4901static int process_new_xattr(struct send_ctx *sctx)
4902{
4903 return iterate_dir_item(sctx->send_root, sctx->left_path,
4904 __process_new_xattr, sctx);
4905}
4906
4907static int process_deleted_xattr(struct send_ctx *sctx)
4908{
4909 return iterate_dir_item(sctx->parent_root, sctx->right_path,
4910 __process_deleted_xattr, sctx);
4911}
4912
4913struct find_xattr_ctx {
4914 const char *name;
4915 int name_len;
4916 int found_idx;
4917 char *found_data;
4918 int found_data_len;
4919 bool copy_data;
4920};
4921
4922static int __find_xattr(int num, struct btrfs_key *di_key, const char *name,
4923 int name_len, const char *data, int data_len, void *vctx)
4924{
4925 struct find_xattr_ctx *ctx = vctx;
4926
4927 if (name_len == ctx->name_len &&
4928 strncmp(name, ctx->name, name_len) == 0) {
4929 ctx->found_idx = num;
4930 ctx->found_data_len = data_len;
4931 if (ctx->copy_data) {
4932 ctx->found_data = kmemdup(data, data_len, GFP_KERNEL);
4933 if (!ctx->found_data)
4934 return -ENOMEM;
4935 }
4936 return 1;
4937 }
4938 return 0;
4939}
4940
4941static int find_xattr(struct btrfs_root *root,
4942 struct btrfs_path *path,
4943 struct btrfs_key *key,
4944 const char *name, int name_len,
4945 char **data, int *data_len)
4946{
4947 int ret;
4948 struct find_xattr_ctx ctx;
4949
4950 ctx.name = name;
4951 ctx.name_len = name_len;
4952 ctx.found_idx = -1;
4953 ctx.found_data = NULL;
4954 ctx.found_data_len = 0;
4955 ctx.copy_data = (data != NULL);
4956
4957 ret = iterate_dir_item(root, path, __find_xattr, &ctx);
4958 if (ret < 0)
4959 return ret;
4960
4961 if (ctx.found_idx == -1)
4962 return -ENOENT;
4963 if (data) {
4964 *data = ctx.found_data;
4965 *data_len = ctx.found_data_len;
4966 } else {
4967 ASSERT(ctx.found_data == NULL);
4968 }
4969 return ctx.found_idx;
4970}
4971
4972
4973static int __process_changed_new_xattr(int num, struct btrfs_key *di_key,
4974 const char *name, int name_len,
4975 const char *data, int data_len,
4976 void *ctx)
4977{
4978 int ret;
4979 struct send_ctx *sctx = ctx;
4980 char AUTO_KFREE(found_data);
4981 int found_data_len = 0;
4982
4983 ret = find_xattr(sctx->parent_root, sctx->right_path,
4984 sctx->cmp_key, name, name_len, &found_data,
4985 &found_data_len);
4986 if (ret == -ENOENT) {
4987 ret = __process_new_xattr(num, di_key, name, name_len, data,
4988 data_len, ctx);
4989 } else if (ret >= 0) {
4990 if (data_len != found_data_len ||
4991 memcmp(data, found_data, data_len)) {
4992 ret = __process_new_xattr(num, di_key, name, name_len,
4993 data, data_len, ctx);
4994 } else {
4995 ret = 0;
4996 }
4997 }
4998
4999 return ret;
5000}
5001
5002static int __process_changed_deleted_xattr(int num, struct btrfs_key *di_key,
5003 const char *name, int name_len,
5004 const char *data, int data_len,
5005 void *ctx)
5006{
5007 int ret;
5008 struct send_ctx *sctx = ctx;
5009
5010 ret = find_xattr(sctx->send_root, sctx->left_path, sctx->cmp_key,
5011 name, name_len, NULL, NULL);
5012 if (ret == -ENOENT)
5013 ret = __process_deleted_xattr(num, di_key, name, name_len, data,
5014 data_len, ctx);
5015 else if (ret >= 0)
5016 ret = 0;
5017
5018 return ret;
5019}
5020
5021static int process_changed_xattr(struct send_ctx *sctx)
5022{
5023 int ret;
5024
5025 ret = iterate_dir_item(sctx->send_root, sctx->left_path,
5026 __process_changed_new_xattr, sctx);
5027 if (ret < 0)
5028 return ret;
5029
5030 return iterate_dir_item(sctx->parent_root, sctx->right_path,
5031 __process_changed_deleted_xattr, sctx);
5032}
5033
5034static int process_all_new_xattrs(struct send_ctx *sctx)
5035{
5036 int ret = 0;
5037 int iter_ret = 0;
5038 struct btrfs_root *root;
5039 BTRFS_PATH_AUTO_FREE(path);
5040 struct btrfs_key key;
5041 struct btrfs_key found_key;
5042
5043 path = alloc_path_for_send();
5044 if (!path)
5045 return -ENOMEM;
5046
5047 root = sctx->send_root;
5048
5049 key.objectid = sctx->cmp_key->objectid;
5050 key.type = BTRFS_XATTR_ITEM_KEY;
5051 key.offset = 0;
5052 btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
5053 if (found_key.objectid != key.objectid ||
5054 found_key.type != key.type) {
5055 ret = 0;
5056 break;
5057 }
5058
5059 ret = iterate_dir_item(root, path, __process_new_xattr, sctx);
5060 if (ret < 0)
5061 break;
5062 }
5063 /* Catch error found during iteration */
5064 if (iter_ret < 0)
5065 ret = iter_ret;
5066
5067 return ret;
5068}
5069
5070static int send_verity(struct send_ctx *sctx, struct fs_path *path,
5071 struct fsverity_descriptor *desc)
5072{
5073 int ret;
5074
5075 ret = begin_cmd(sctx, BTRFS_SEND_C_ENABLE_VERITY);
5076 if (ret < 0)
5077 return ret;
5078
5079 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
5080 TLV_PUT_U8(sctx, BTRFS_SEND_A_VERITY_ALGORITHM,
5081 le8_to_cpu(desc->hash_algorithm));
5082 TLV_PUT_U32(sctx, BTRFS_SEND_A_VERITY_BLOCK_SIZE,
5083 1U << le8_to_cpu(desc->log_blocksize));
5084 TLV_PUT(sctx, BTRFS_SEND_A_VERITY_SALT_DATA, desc->salt,
5085 le8_to_cpu(desc->salt_size));
5086 TLV_PUT(sctx, BTRFS_SEND_A_VERITY_SIG_DATA, desc->signature,
5087 le32_to_cpu(desc->sig_size));
5088
5089 ret = send_cmd(sctx);
5090
5091tlv_put_failure:
5092 return ret;
5093}
5094
5095static int process_verity(struct send_ctx *sctx)
5096{
5097 int ret = 0;
5098 struct btrfs_inode *inode;
5099 struct fs_path *p;
5100
5101 inode = btrfs_iget(sctx->cur_ino, sctx->send_root);
5102 if (IS_ERR(inode))
5103 return PTR_ERR(inode);
5104
5105 ret = btrfs_get_verity_descriptor(&inode->vfs_inode, NULL, 0);
5106 if (ret < 0)
5107 goto iput;
5108
5109 if (unlikely(ret > FS_VERITY_MAX_DESCRIPTOR_SIZE)) {
5110 ret = -EMSGSIZE;
5111 goto iput;
5112 }
5113 if (!sctx->verity_descriptor) {
5114 sctx->verity_descriptor = kvmalloc(FS_VERITY_MAX_DESCRIPTOR_SIZE,
5115 GFP_KERNEL);
5116 if (!sctx->verity_descriptor) {
5117 ret = -ENOMEM;
5118 goto iput;
5119 }
5120 }
5121
5122 ret = btrfs_get_verity_descriptor(&inode->vfs_inode, sctx->verity_descriptor, ret);
5123 if (ret < 0)
5124 goto iput;
5125
5126 p = get_cur_inode_path(sctx);
5127 if (IS_ERR(p)) {
5128 ret = PTR_ERR(p);
5129 goto iput;
5130 }
5131
5132 ret = send_verity(sctx, p, sctx->verity_descriptor);
5133iput:
5134 iput(&inode->vfs_inode);
5135 return ret;
5136}
5137
5138static inline u64 max_send_read_size(const struct send_ctx *sctx)
5139{
5140 return sctx->send_max_size - SZ_16K;
5141}
5142
5143static int put_data_header(struct send_ctx *sctx, u32 len)
5144{
5145 if (WARN_ON_ONCE(sctx->put_data))
5146 return -EINVAL;
5147 sctx->put_data = true;
5148 if (sctx->proto >= 2) {
5149 /*
5150 * Since v2, the data attribute header doesn't include a length,
5151 * it is implicitly to the end of the command.
5152 */
5153 if (unlikely(sctx->send_max_size - sctx->send_size < sizeof(__le16) + len))
5154 return -EOVERFLOW;
5155 put_unaligned_le16(BTRFS_SEND_A_DATA, sctx->send_buf + sctx->send_size);
5156 sctx->send_size += sizeof(__le16);
5157 } else {
5158 struct btrfs_tlv_header *hdr;
5159
5160 if (unlikely(sctx->send_max_size - sctx->send_size < sizeof(*hdr) + len))
5161 return -EOVERFLOW;
5162 hdr = (struct btrfs_tlv_header *)(sctx->send_buf + sctx->send_size);
5163 put_unaligned_le16(BTRFS_SEND_A_DATA, &hdr->tlv_type);
5164 put_unaligned_le16(len, &hdr->tlv_len);
5165 sctx->send_size += sizeof(*hdr);
5166 }
5167 return 0;
5168}
5169
5170static int put_file_data(struct send_ctx *sctx, u64 offset, u32 len)
5171{
5172 struct btrfs_root *root = sctx->send_root;
5173 struct btrfs_fs_info *fs_info = root->fs_info;
5174 u64 cur = offset;
5175 const u64 end = offset + len;
5176 const pgoff_t last_index = ((end - 1) >> PAGE_SHIFT);
5177 struct address_space *mapping = sctx->cur_inode->i_mapping;
5178 int ret;
5179
5180 ret = put_data_header(sctx, len);
5181 if (ret)
5182 return ret;
5183
5184 while (cur < end) {
5185 pgoff_t index = (cur >> PAGE_SHIFT);
5186 unsigned int cur_len;
5187 unsigned int pg_offset;
5188 struct folio *folio;
5189
5190 folio = filemap_lock_folio(mapping, index);
5191 if (IS_ERR(folio)) {
5192 page_cache_sync_readahead(mapping,
5193 &sctx->ra, NULL, index,
5194 last_index + 1 - index);
5195
5196 folio = filemap_grab_folio(mapping, index);
5197 if (IS_ERR(folio)) {
5198 ret = PTR_ERR(folio);
5199 break;
5200 }
5201 }
5202 pg_offset = offset_in_folio(folio, cur);
5203 cur_len = min_t(unsigned int, end - cur, folio_size(folio) - pg_offset);
5204
5205 if (folio_test_readahead(folio))
5206 page_cache_async_readahead(mapping, &sctx->ra, NULL, folio,
5207 last_index + 1 - index);
5208
5209 if (!folio_test_uptodate(folio)) {
5210 btrfs_read_folio(NULL, folio);
5211 folio_lock(folio);
5212 if (unlikely(!folio_test_uptodate(folio))) {
5213 folio_unlock(folio);
5214 btrfs_err(fs_info,
5215 "send: IO error at offset %llu for inode %llu root %llu",
5216 folio_pos(folio), sctx->cur_ino,
5217 btrfs_root_id(sctx->send_root));
5218 folio_put(folio);
5219 ret = -EIO;
5220 break;
5221 }
5222 if (folio->mapping != mapping) {
5223 folio_unlock(folio);
5224 folio_put(folio);
5225 continue;
5226 }
5227 }
5228
5229 memcpy_from_folio(sctx->send_buf + sctx->send_size, folio,
5230 pg_offset, cur_len);
5231 folio_unlock(folio);
5232 folio_put(folio);
5233 cur += cur_len;
5234 sctx->send_size += cur_len;
5235 }
5236
5237 return ret;
5238}
5239
5240/*
5241 * Read some bytes from the current inode/file and send a write command to
5242 * user space.
5243 */
5244static int send_write(struct send_ctx *sctx, u64 offset, u32 len)
5245{
5246 int ret = 0;
5247 struct fs_path *p;
5248
5249 p = get_cur_inode_path(sctx);
5250 if (IS_ERR(p))
5251 return PTR_ERR(p);
5252
5253 ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE);
5254 if (ret < 0)
5255 return ret;
5256
5257 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5258 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5259 ret = put_file_data(sctx, offset, len);
5260 if (ret < 0)
5261 return ret;
5262
5263 ret = send_cmd(sctx);
5264
5265tlv_put_failure:
5266 return ret;
5267}
5268
5269/*
5270 * Send a clone command to user space.
5271 */
5272static int send_clone(struct send_ctx *sctx,
5273 u64 offset, u32 len,
5274 struct clone_root *clone_root)
5275{
5276 int ret = 0;
5277 struct fs_path *p;
5278 struct fs_path *cur_inode_path;
5279 u64 gen;
5280
5281 cur_inode_path = get_cur_inode_path(sctx);
5282 if (IS_ERR(cur_inode_path))
5283 return PTR_ERR(cur_inode_path);
5284
5285 p = fs_path_alloc();
5286 if (!p)
5287 return -ENOMEM;
5288
5289 ret = begin_cmd(sctx, BTRFS_SEND_C_CLONE);
5290 if (ret < 0)
5291 goto out;
5292
5293 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5294 TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_LEN, len);
5295 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, cur_inode_path);
5296
5297 if (clone_root->root == sctx->send_root) {
5298 ret = get_inode_gen(sctx->send_root, clone_root->ino, &gen);
5299 if (ret < 0)
5300 goto out;
5301 ret = get_cur_path(sctx, clone_root->ino, gen, p);
5302 } else {
5303 ret = get_inode_path(clone_root->root, clone_root->ino, p);
5304 }
5305 if (ret < 0)
5306 goto out;
5307
5308 /*
5309 * If the parent we're using has a received_uuid set then use that as
5310 * our clone source as that is what we will look for when doing a
5311 * receive.
5312 *
5313 * This covers the case that we create a snapshot off of a received
5314 * subvolume and then use that as the parent and try to receive on a
5315 * different host.
5316 */
5317 if (!btrfs_is_empty_uuid(clone_root->root->root_item.received_uuid))
5318 TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
5319 clone_root->root->root_item.received_uuid);
5320 else
5321 TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID,
5322 clone_root->root->root_item.uuid);
5323 TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_CTRANSID,
5324 btrfs_root_ctransid(&clone_root->root->root_item));
5325 TLV_PUT_PATH(sctx, BTRFS_SEND_A_CLONE_PATH, p);
5326 TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_OFFSET,
5327 clone_root->offset);
5328
5329 ret = send_cmd(sctx);
5330
5331tlv_put_failure:
5332out:
5333 fs_path_free(p);
5334 return ret;
5335}
5336
5337/*
5338 * Send an update extent command to user space.
5339 */
5340static int send_update_extent(struct send_ctx *sctx,
5341 u64 offset, u32 len)
5342{
5343 int ret = 0;
5344 struct fs_path *p;
5345
5346 p = get_cur_inode_path(sctx);
5347 if (IS_ERR(p))
5348 return PTR_ERR(p);
5349
5350 ret = begin_cmd(sctx, BTRFS_SEND_C_UPDATE_EXTENT);
5351 if (ret < 0)
5352 return ret;
5353
5354 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5355 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5356 TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, len);
5357
5358 ret = send_cmd(sctx);
5359
5360tlv_put_failure:
5361 return ret;
5362}
5363
5364static int send_fallocate(struct send_ctx *sctx, u32 mode, u64 offset, u64 len)
5365{
5366 struct fs_path *path;
5367 int ret;
5368
5369 path = get_cur_inode_path(sctx);
5370 if (IS_ERR(path))
5371 return PTR_ERR(path);
5372
5373 ret = begin_cmd(sctx, BTRFS_SEND_C_FALLOCATE);
5374 if (ret < 0)
5375 return ret;
5376
5377 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path);
5378 TLV_PUT_U32(sctx, BTRFS_SEND_A_FALLOCATE_MODE, mode);
5379 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5380 TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, len);
5381
5382 ret = send_cmd(sctx);
5383
5384tlv_put_failure:
5385 return ret;
5386}
5387
5388static int send_hole(struct send_ctx *sctx, u64 end)
5389{
5390 struct fs_path *p = NULL;
5391 u64 read_size = max_send_read_size(sctx);
5392 u64 offset = sctx->cur_inode_last_extent;
5393 int ret = 0;
5394
5395 /*
5396 * Starting with send stream v2 we have fallocate and can use it to
5397 * punch holes instead of sending writes full of zeroes.
5398 */
5399 if (proto_cmd_ok(sctx, BTRFS_SEND_C_FALLOCATE))
5400 return send_fallocate(sctx, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
5401 offset, end - offset);
5402
5403 /*
5404 * A hole that starts at EOF or beyond it. Since we do not yet support
5405 * fallocate (for extent preallocation and hole punching), sending a
5406 * write of zeroes starting at EOF or beyond would later require issuing
5407 * a truncate operation which would undo the write and achieve nothing.
5408 */
5409 if (offset >= sctx->cur_inode_size)
5410 return 0;
5411
5412 /*
5413 * Don't go beyond the inode's i_size due to prealloc extents that start
5414 * after the i_size.
5415 */
5416 end = min_t(u64, end, sctx->cur_inode_size);
5417
5418 if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA)
5419 return send_update_extent(sctx, offset, end - offset);
5420
5421 p = get_cur_inode_path(sctx);
5422 if (IS_ERR(p))
5423 return PTR_ERR(p);
5424
5425 while (offset < end) {
5426 u64 len = min(end - offset, read_size);
5427
5428 ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE);
5429 if (ret < 0)
5430 break;
5431 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
5432 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5433 ret = put_data_header(sctx, len);
5434 if (ret < 0)
5435 break;
5436 memset(sctx->send_buf + sctx->send_size, 0, len);
5437 sctx->send_size += len;
5438 ret = send_cmd(sctx);
5439 if (ret < 0)
5440 break;
5441 offset += len;
5442 }
5443 sctx->cur_inode_next_write_offset = offset;
5444tlv_put_failure:
5445 return ret;
5446}
5447
5448static int send_encoded_inline_extent(struct send_ctx *sctx,
5449 struct btrfs_path *path, u64 offset,
5450 u64 len)
5451{
5452 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
5453 struct fs_path *fspath;
5454 struct extent_buffer *leaf = path->nodes[0];
5455 struct btrfs_key key;
5456 struct btrfs_file_extent_item *ei;
5457 u64 ram_bytes;
5458 size_t inline_size;
5459 int ret;
5460
5461 fspath = get_cur_inode_path(sctx);
5462 if (IS_ERR(fspath))
5463 return PTR_ERR(fspath);
5464
5465 ret = begin_cmd(sctx, BTRFS_SEND_C_ENCODED_WRITE);
5466 if (ret < 0)
5467 return ret;
5468
5469 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
5470 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
5471 ram_bytes = btrfs_file_extent_ram_bytes(leaf, ei);
5472 inline_size = btrfs_file_extent_inline_item_len(leaf, path->slots[0]);
5473
5474 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, fspath);
5475 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5476 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN,
5477 min(key.offset + ram_bytes - offset, len));
5478 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN, ram_bytes);
5479 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET, offset - key.offset);
5480 ret = btrfs_encoded_io_compression_from_extent(fs_info,
5481 btrfs_file_extent_compression(leaf, ei));
5482 if (ret < 0)
5483 return ret;
5484 TLV_PUT_U32(sctx, BTRFS_SEND_A_COMPRESSION, ret);
5485
5486 ret = put_data_header(sctx, inline_size);
5487 if (ret < 0)
5488 return ret;
5489 read_extent_buffer(leaf, sctx->send_buf + sctx->send_size,
5490 btrfs_file_extent_inline_start(ei), inline_size);
5491 sctx->send_size += inline_size;
5492
5493 ret = send_cmd(sctx);
5494
5495tlv_put_failure:
5496 return ret;
5497}
5498
5499static int send_encoded_extent(struct send_ctx *sctx, struct btrfs_path *path,
5500 u64 offset, u64 len)
5501{
5502 struct btrfs_root *root = sctx->send_root;
5503 struct btrfs_fs_info *fs_info = root->fs_info;
5504 struct btrfs_inode *inode;
5505 struct fs_path *fspath;
5506 struct extent_buffer *leaf = path->nodes[0];
5507 struct btrfs_key key;
5508 struct btrfs_file_extent_item *ei;
5509 u64 disk_bytenr, disk_num_bytes;
5510 u32 data_offset;
5511 struct btrfs_cmd_header *hdr;
5512 u32 crc;
5513 int ret;
5514
5515 inode = btrfs_iget(sctx->cur_ino, root);
5516 if (IS_ERR(inode))
5517 return PTR_ERR(inode);
5518
5519 fspath = get_cur_inode_path(sctx);
5520 if (IS_ERR(fspath)) {
5521 ret = PTR_ERR(fspath);
5522 goto out;
5523 }
5524
5525 ret = begin_cmd(sctx, BTRFS_SEND_C_ENCODED_WRITE);
5526 if (ret < 0)
5527 goto out;
5528
5529 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
5530 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
5531 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei);
5532 disk_num_bytes = btrfs_file_extent_disk_num_bytes(leaf, ei);
5533
5534 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, fspath);
5535 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
5536 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN,
5537 min(key.offset + btrfs_file_extent_num_bytes(leaf, ei) - offset,
5538 len));
5539 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN,
5540 btrfs_file_extent_ram_bytes(leaf, ei));
5541 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET,
5542 offset - key.offset + btrfs_file_extent_offset(leaf, ei));
5543 ret = btrfs_encoded_io_compression_from_extent(fs_info,
5544 btrfs_file_extent_compression(leaf, ei));
5545 if (ret < 0)
5546 goto out;
5547 TLV_PUT_U32(sctx, BTRFS_SEND_A_COMPRESSION, ret);
5548 TLV_PUT_U32(sctx, BTRFS_SEND_A_ENCRYPTION, 0);
5549
5550 ret = put_data_header(sctx, disk_num_bytes);
5551 if (ret < 0)
5552 goto out;
5553
5554 /*
5555 * We want to do I/O directly into the send buffer, so get the next page
5556 * boundary in the send buffer. This means that there may be a gap
5557 * between the beginning of the command and the file data.
5558 */
5559 data_offset = PAGE_ALIGN(sctx->send_size);
5560 if (unlikely(data_offset > sctx->send_max_size ||
5561 sctx->send_max_size - data_offset < disk_num_bytes)) {
5562 ret = -EOVERFLOW;
5563 goto out;
5564 }
5565
5566 /*
5567 * Note that send_buf is a mapping of send_buf_pages, so this is really
5568 * reading into send_buf.
5569 */
5570 ret = btrfs_encoded_read_regular_fill_pages(inode,
5571 disk_bytenr, disk_num_bytes,
5572 sctx->send_buf_pages +
5573 (data_offset >> PAGE_SHIFT),
5574 NULL);
5575 if (ret)
5576 goto out;
5577
5578 hdr = (struct btrfs_cmd_header *)sctx->send_buf;
5579 hdr->len = cpu_to_le32(sctx->send_size + disk_num_bytes - sizeof(*hdr));
5580 hdr->crc = 0;
5581 crc = crc32c(0, sctx->send_buf, sctx->send_size);
5582 crc = crc32c(crc, sctx->send_buf + data_offset, disk_num_bytes);
5583 hdr->crc = cpu_to_le32(crc);
5584
5585 ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size,
5586 &sctx->send_off);
5587 if (!ret) {
5588 ret = write_buf(sctx->send_filp, sctx->send_buf + data_offset,
5589 disk_num_bytes, &sctx->send_off);
5590 }
5591 sctx->send_size = 0;
5592 sctx->put_data = false;
5593
5594tlv_put_failure:
5595out:
5596 iput(&inode->vfs_inode);
5597 return ret;
5598}
5599
5600static int send_extent_data(struct send_ctx *sctx, struct btrfs_path *path,
5601 const u64 offset, const u64 len)
5602{
5603 const u64 end = offset + len;
5604 struct extent_buffer *leaf = path->nodes[0];
5605 struct btrfs_file_extent_item *ei;
5606 u64 read_size = max_send_read_size(sctx);
5607 u64 sent = 0;
5608
5609 if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA)
5610 return send_update_extent(sctx, offset, len);
5611
5612 ei = btrfs_item_ptr(leaf, path->slots[0],
5613 struct btrfs_file_extent_item);
5614 if ((sctx->flags & BTRFS_SEND_FLAG_COMPRESSED) &&
5615 btrfs_file_extent_compression(leaf, ei) != BTRFS_COMPRESS_NONE) {
5616 bool is_inline = (btrfs_file_extent_type(leaf, ei) ==
5617 BTRFS_FILE_EXTENT_INLINE);
5618
5619 /*
5620 * Send the compressed extent unless the compressed data is
5621 * larger than the decompressed data. This can happen if we're
5622 * not sending the entire extent, either because it has been
5623 * partially overwritten/truncated or because this is a part of
5624 * the extent that we couldn't clone in clone_range().
5625 */
5626 if (is_inline &&
5627 btrfs_file_extent_inline_item_len(leaf,
5628 path->slots[0]) <= len) {
5629 return send_encoded_inline_extent(sctx, path, offset,
5630 len);
5631 } else if (!is_inline &&
5632 btrfs_file_extent_disk_num_bytes(leaf, ei) <= len) {
5633 return send_encoded_extent(sctx, path, offset, len);
5634 }
5635 }
5636
5637 if (sctx->cur_inode == NULL) {
5638 struct btrfs_inode *btrfs_inode;
5639 struct btrfs_root *root = sctx->send_root;
5640
5641 btrfs_inode = btrfs_iget(sctx->cur_ino, root);
5642 if (IS_ERR(btrfs_inode))
5643 return PTR_ERR(btrfs_inode);
5644
5645 sctx->cur_inode = &btrfs_inode->vfs_inode;
5646 memset(&sctx->ra, 0, sizeof(struct file_ra_state));
5647 file_ra_state_init(&sctx->ra, sctx->cur_inode->i_mapping);
5648
5649 /*
5650 * It's very likely there are no pages from this inode in the page
5651 * cache, so after reading extents and sending their data, we clean
5652 * the page cache to avoid trashing the page cache (adding pressure
5653 * to the page cache and forcing eviction of other data more useful
5654 * for applications).
5655 *
5656 * We decide if we should clean the page cache simply by checking
5657 * if the inode's mapping nrpages is 0 when we first open it, and
5658 * not by using something like filemap_range_has_page() before
5659 * reading an extent because when we ask the readahead code to
5660 * read a given file range, it may (and almost always does) read
5661 * pages from beyond that range (see the documentation for
5662 * page_cache_sync_readahead()), so it would not be reliable,
5663 * because after reading the first extent future calls to
5664 * filemap_range_has_page() would return true because the readahead
5665 * on the previous extent resulted in reading pages of the current
5666 * extent as well.
5667 */
5668 sctx->clean_page_cache = (sctx->cur_inode->i_mapping->nrpages == 0);
5669 sctx->page_cache_clear_start = round_down(offset, PAGE_SIZE);
5670 }
5671
5672 while (sent < len) {
5673 u64 size = min(len - sent, read_size);
5674 int ret;
5675
5676 ret = send_write(sctx, offset + sent, size);
5677 if (ret < 0)
5678 return ret;
5679 sent += size;
5680 }
5681
5682 if (sctx->clean_page_cache && PAGE_ALIGNED(end)) {
5683 /*
5684 * Always operate only on ranges that are a multiple of the page
5685 * size. This is not only to prevent zeroing parts of a page in
5686 * the case of subpage sector size, but also to guarantee we evict
5687 * pages, as passing a range that is smaller than page size does
5688 * not evict the respective page (only zeroes part of its content).
5689 *
5690 * Always start from the end offset of the last range cleared.
5691 * This is because the readahead code may (and very often does)
5692 * reads pages beyond the range we request for readahead. So if
5693 * we have an extent layout like this:
5694 *
5695 * [ extent A ] [ extent B ] [ extent C ]
5696 *
5697 * When we ask page_cache_sync_readahead() to read extent A, it
5698 * may also trigger reads for pages of extent B. If we are doing
5699 * an incremental send and extent B has not changed between the
5700 * parent and send snapshots, some or all of its pages may end
5701 * up being read and placed in the page cache. So when truncating
5702 * the page cache we always start from the end offset of the
5703 * previously processed extent up to the end of the current
5704 * extent.
5705 */
5706 truncate_inode_pages_range(&sctx->cur_inode->i_data,
5707 sctx->page_cache_clear_start,
5708 end - 1);
5709 sctx->page_cache_clear_start = end;
5710 }
5711
5712 return 0;
5713}
5714
5715/*
5716 * Search for a capability xattr related to sctx->cur_ino. If the capability is
5717 * found, call send_set_xattr function to emit it.
5718 *
5719 * Return 0 if there isn't a capability, or when the capability was emitted
5720 * successfully, or < 0 if an error occurred.
5721 */
5722static int send_capabilities(struct send_ctx *sctx)
5723{
5724 BTRFS_PATH_AUTO_FREE(path);
5725 struct btrfs_dir_item *di;
5726 struct extent_buffer *leaf;
5727 unsigned long data_ptr;
5728 char AUTO_KFREE(buf);
5729 int buf_len;
5730 int ret = 0;
5731
5732 path = alloc_path_for_send();
5733 if (!path)
5734 return -ENOMEM;
5735
5736 di = btrfs_lookup_xattr(NULL, sctx->send_root, path, sctx->cur_ino,
5737 XATTR_NAME_CAPS, strlen(XATTR_NAME_CAPS), 0);
5738 if (!di) {
5739 /* There is no xattr for this inode */
5740 return 0;
5741 } else if (IS_ERR(di)) {
5742 return PTR_ERR(di);
5743 }
5744
5745 leaf = path->nodes[0];
5746 buf_len = btrfs_dir_data_len(leaf, di);
5747
5748 buf = kmalloc(buf_len, GFP_KERNEL);
5749 if (!buf)
5750 return -ENOMEM;
5751
5752 data_ptr = (unsigned long)(di + 1) + btrfs_dir_name_len(leaf, di);
5753 read_extent_buffer(leaf, buf, data_ptr, buf_len);
5754
5755 ret = send_set_xattr(sctx, XATTR_NAME_CAPS,
5756 strlen(XATTR_NAME_CAPS), buf, buf_len);
5757 return ret;
5758}
5759
5760static int clone_range(struct send_ctx *sctx, struct btrfs_path *dst_path,
5761 struct clone_root *clone_root, const u64 disk_byte,
5762 u64 data_offset, u64 offset, u64 len)
5763{
5764 BTRFS_PATH_AUTO_FREE(path);
5765 struct btrfs_key key;
5766 int ret;
5767 struct btrfs_inode_info info;
5768 u64 clone_src_i_size = 0;
5769
5770 /*
5771 * Prevent cloning from a zero offset with a length matching the sector
5772 * size because in some scenarios this will make the receiver fail.
5773 *
5774 * For example, if in the source filesystem the extent at offset 0
5775 * has a length of sectorsize and it was written using direct IO, then
5776 * it can never be an inline extent (even if compression is enabled).
5777 * Then this extent can be cloned in the original filesystem to a non
5778 * zero file offset, but it may not be possible to clone in the
5779 * destination filesystem because it can be inlined due to compression
5780 * on the destination filesystem (as the receiver's write operations are
5781 * always done using buffered IO). The same happens when the original
5782 * filesystem does not have compression enabled but the destination
5783 * filesystem has.
5784 */
5785 if (clone_root->offset == 0 &&
5786 len == sctx->send_root->fs_info->sectorsize)
5787 return send_extent_data(sctx, dst_path, offset, len);
5788
5789 path = alloc_path_for_send();
5790 if (!path)
5791 return -ENOMEM;
5792
5793 /*
5794 * There are inodes that have extents that lie behind its i_size. Don't
5795 * accept clones from these extents.
5796 */
5797 ret = get_inode_info(clone_root->root, clone_root->ino, &info);
5798 btrfs_release_path(path);
5799 if (ret < 0)
5800 return ret;
5801 clone_src_i_size = info.size;
5802
5803 /*
5804 * We can't send a clone operation for the entire range if we find
5805 * extent items in the respective range in the source file that
5806 * refer to different extents or if we find holes.
5807 * So check for that and do a mix of clone and regular write/copy
5808 * operations if needed.
5809 *
5810 * Example:
5811 *
5812 * mkfs.btrfs -f /dev/sda
5813 * mount /dev/sda /mnt
5814 * xfs_io -f -c "pwrite -S 0xaa 0K 100K" /mnt/foo
5815 * cp --reflink=always /mnt/foo /mnt/bar
5816 * xfs_io -c "pwrite -S 0xbb 50K 50K" /mnt/foo
5817 * btrfs subvolume snapshot -r /mnt /mnt/snap
5818 *
5819 * If when we send the snapshot and we are processing file bar (which
5820 * has a higher inode number than foo) we blindly send a clone operation
5821 * for the [0, 100K[ range from foo to bar, the receiver ends up getting
5822 * a file bar that matches the content of file foo - iow, doesn't match
5823 * the content from bar in the original filesystem.
5824 */
5825 key.objectid = clone_root->ino;
5826 key.type = BTRFS_EXTENT_DATA_KEY;
5827 key.offset = clone_root->offset;
5828 ret = btrfs_search_slot(NULL, clone_root->root, &key, path, 0, 0);
5829 if (ret < 0)
5830 return ret;
5831 if (ret > 0 && path->slots[0] > 0) {
5832 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1);
5833 if (key.objectid == clone_root->ino &&
5834 key.type == BTRFS_EXTENT_DATA_KEY)
5835 path->slots[0]--;
5836 }
5837
5838 while (true) {
5839 struct extent_buffer *leaf = path->nodes[0];
5840 int slot = path->slots[0];
5841 struct btrfs_file_extent_item *ei;
5842 u8 type;
5843 u64 ext_len;
5844 u64 clone_len;
5845 u64 clone_data_offset;
5846 bool crossed_src_i_size = false;
5847
5848 if (slot >= btrfs_header_nritems(leaf)) {
5849 ret = btrfs_next_leaf(clone_root->root, path);
5850 if (ret < 0)
5851 return ret;
5852 else if (ret > 0)
5853 break;
5854 continue;
5855 }
5856
5857 btrfs_item_key_to_cpu(leaf, &key, slot);
5858
5859 /*
5860 * We might have an implicit trailing hole (NO_HOLES feature
5861 * enabled). We deal with it after leaving this loop.
5862 */
5863 if (key.objectid != clone_root->ino ||
5864 key.type != BTRFS_EXTENT_DATA_KEY)
5865 break;
5866
5867 ei = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
5868 type = btrfs_file_extent_type(leaf, ei);
5869 if (type == BTRFS_FILE_EXTENT_INLINE) {
5870 ext_len = btrfs_file_extent_ram_bytes(leaf, ei);
5871 ext_len = PAGE_ALIGN(ext_len);
5872 } else {
5873 ext_len = btrfs_file_extent_num_bytes(leaf, ei);
5874 }
5875
5876 if (key.offset + ext_len <= clone_root->offset)
5877 goto next;
5878
5879 if (key.offset > clone_root->offset) {
5880 /* Implicit hole, NO_HOLES feature enabled. */
5881 u64 hole_len = key.offset - clone_root->offset;
5882
5883 if (hole_len > len)
5884 hole_len = len;
5885 ret = send_extent_data(sctx, dst_path, offset,
5886 hole_len);
5887 if (ret < 0)
5888 return ret;
5889
5890 len -= hole_len;
5891 if (len == 0)
5892 break;
5893 offset += hole_len;
5894 clone_root->offset += hole_len;
5895 data_offset += hole_len;
5896 }
5897
5898 if (key.offset >= clone_root->offset + len)
5899 break;
5900
5901 if (key.offset >= clone_src_i_size)
5902 break;
5903
5904 if (key.offset + ext_len > clone_src_i_size) {
5905 ext_len = clone_src_i_size - key.offset;
5906 crossed_src_i_size = true;
5907 }
5908
5909 clone_data_offset = btrfs_file_extent_offset(leaf, ei);
5910 if (btrfs_file_extent_disk_bytenr(leaf, ei) == disk_byte) {
5911 clone_root->offset = key.offset;
5912 if (clone_data_offset < data_offset &&
5913 clone_data_offset + ext_len > data_offset) {
5914 u64 extent_offset;
5915
5916 extent_offset = data_offset - clone_data_offset;
5917 ext_len -= extent_offset;
5918 clone_data_offset += extent_offset;
5919 clone_root->offset += extent_offset;
5920 }
5921 }
5922
5923 clone_len = min_t(u64, ext_len, len);
5924
5925 if (btrfs_file_extent_disk_bytenr(leaf, ei) == disk_byte &&
5926 clone_data_offset == data_offset) {
5927 const u64 src_end = clone_root->offset + clone_len;
5928 const u64 sectorsize = SZ_64K;
5929
5930 /*
5931 * We can't clone the last block, when its size is not
5932 * sector size aligned, into the middle of a file. If we
5933 * do so, the receiver will get a failure (-EINVAL) when
5934 * trying to clone or will silently corrupt the data in
5935 * the destination file if it's on a kernel without the
5936 * fix introduced by commit ac765f83f1397646
5937 * ("Btrfs: fix data corruption due to cloning of eof
5938 * block).
5939 *
5940 * So issue a clone of the aligned down range plus a
5941 * regular write for the eof block, if we hit that case.
5942 *
5943 * Also, we use the maximum possible sector size, 64K,
5944 * because we don't know what's the sector size of the
5945 * filesystem that receives the stream, so we have to
5946 * assume the largest possible sector size.
5947 */
5948 if (src_end == clone_src_i_size &&
5949 !IS_ALIGNED(src_end, sectorsize) &&
5950 offset + clone_len < sctx->cur_inode_size) {
5951 u64 slen;
5952
5953 slen = ALIGN_DOWN(src_end - clone_root->offset,
5954 sectorsize);
5955 if (slen > 0) {
5956 ret = send_clone(sctx, offset, slen,
5957 clone_root);
5958 if (ret < 0)
5959 return ret;
5960 }
5961 ret = send_extent_data(sctx, dst_path,
5962 offset + slen,
5963 clone_len - slen);
5964 } else {
5965 ret = send_clone(sctx, offset, clone_len,
5966 clone_root);
5967 }
5968 } else if (crossed_src_i_size && clone_len < len) {
5969 /*
5970 * If we are at i_size of the clone source inode and we
5971 * can not clone from it, terminate the loop. This is
5972 * to avoid sending two write operations, one with a
5973 * length matching clone_len and the final one after
5974 * this loop with a length of len - clone_len.
5975 *
5976 * When using encoded writes (BTRFS_SEND_FLAG_COMPRESSED
5977 * was passed to the send ioctl), this helps avoid
5978 * sending an encoded write for an offset that is not
5979 * sector size aligned, in case the i_size of the source
5980 * inode is not sector size aligned. That will make the
5981 * receiver fallback to decompression of the data and
5982 * writing it using regular buffered IO, therefore while
5983 * not incorrect, it's not optimal due decompression and
5984 * possible re-compression at the receiver.
5985 */
5986 break;
5987 } else {
5988 ret = send_extent_data(sctx, dst_path, offset,
5989 clone_len);
5990 }
5991
5992 if (ret < 0)
5993 return ret;
5994
5995 len -= clone_len;
5996 if (len == 0)
5997 break;
5998 offset += clone_len;
5999 clone_root->offset += clone_len;
6000
6001 /*
6002 * If we are cloning from the file we are currently processing,
6003 * and using the send root as the clone root, we must stop once
6004 * the current clone offset reaches the current eof of the file
6005 * at the receiver, otherwise we would issue an invalid clone
6006 * operation (source range going beyond eof) and cause the
6007 * receiver to fail. So if we reach the current eof, bail out
6008 * and fallback to a regular write.
6009 */
6010 if (clone_root->root == sctx->send_root &&
6011 clone_root->ino == sctx->cur_ino &&
6012 clone_root->offset >= sctx->cur_inode_next_write_offset)
6013 break;
6014
6015 data_offset += clone_len;
6016next:
6017 path->slots[0]++;
6018 }
6019
6020 if (len > 0)
6021 ret = send_extent_data(sctx, dst_path, offset, len);
6022 else
6023 ret = 0;
6024 return ret;
6025}
6026
6027static int send_write_or_clone(struct send_ctx *sctx,
6028 struct btrfs_path *path,
6029 struct btrfs_key *key,
6030 struct clone_root *clone_root)
6031{
6032 int ret = 0;
6033 u64 offset = key->offset;
6034 u64 end;
6035 const u32 bs = sctx->send_root->fs_info->sectorsize;
6036 struct btrfs_file_extent_item *ei;
6037 u64 disk_byte;
6038 u64 data_offset;
6039 u64 num_bytes;
6040 struct btrfs_inode_info info = { 0 };
6041
6042 end = min_t(u64, btrfs_file_extent_end(path), sctx->cur_inode_size);
6043 if (offset >= end)
6044 return 0;
6045
6046 num_bytes = end - offset;
6047
6048 if (!clone_root)
6049 goto write_data;
6050
6051 if (IS_ALIGNED(end, bs))
6052 goto clone_data;
6053
6054 /*
6055 * If the extent end is not aligned, we can clone if the extent ends at
6056 * the i_size of the inode and the clone range ends at the i_size of the
6057 * source inode, otherwise the clone operation fails with -EINVAL.
6058 */
6059 if (end != sctx->cur_inode_size)
6060 goto write_data;
6061
6062 ret = get_inode_info(clone_root->root, clone_root->ino, &info);
6063 if (ret < 0)
6064 return ret;
6065
6066 if (clone_root->offset + num_bytes == info.size) {
6067 /*
6068 * The final size of our file matches the end offset, but it may
6069 * be that its current size is larger, so we have to truncate it
6070 * to any value between the start offset of the range and the
6071 * final i_size, otherwise the clone operation is invalid
6072 * because it's unaligned and it ends before the current EOF.
6073 * We do this truncate to the final i_size when we finish
6074 * processing the inode, but it's too late by then. And here we
6075 * truncate to the start offset of the range because it's always
6076 * sector size aligned while if it were the final i_size it
6077 * would result in dirtying part of a page, filling part of a
6078 * page with zeroes and then having the clone operation at the
6079 * receiver trigger IO and wait for it due to the dirty page.
6080 */
6081 if (sctx->parent_root != NULL) {
6082 ret = send_truncate(sctx, sctx->cur_ino,
6083 sctx->cur_inode_gen, offset);
6084 if (ret < 0)
6085 return ret;
6086 }
6087 goto clone_data;
6088 }
6089
6090write_data:
6091 ret = send_extent_data(sctx, path, offset, num_bytes);
6092 sctx->cur_inode_next_write_offset = end;
6093 return ret;
6094
6095clone_data:
6096 ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
6097 struct btrfs_file_extent_item);
6098 disk_byte = btrfs_file_extent_disk_bytenr(path->nodes[0], ei);
6099 data_offset = btrfs_file_extent_offset(path->nodes[0], ei);
6100 ret = clone_range(sctx, path, clone_root, disk_byte, data_offset, offset,
6101 num_bytes);
6102 sctx->cur_inode_next_write_offset = end;
6103 return ret;
6104}
6105
6106static int is_extent_unchanged(struct send_ctx *sctx,
6107 struct btrfs_path *left_path,
6108 struct btrfs_key *ekey)
6109{
6110 int ret = 0;
6111 struct btrfs_key key;
6112 BTRFS_PATH_AUTO_FREE(path);
6113 struct extent_buffer *eb;
6114 int slot;
6115 struct btrfs_key found_key;
6116 struct btrfs_file_extent_item *ei;
6117 u64 left_disknr;
6118 u64 right_disknr;
6119 u64 left_offset;
6120 u64 right_offset;
6121 u64 left_offset_fixed;
6122 u64 left_len;
6123 u64 right_len;
6124 u64 left_gen;
6125 u64 right_gen;
6126 u8 left_type;
6127 u8 right_type;
6128
6129 path = alloc_path_for_send();
6130 if (!path)
6131 return -ENOMEM;
6132
6133 eb = left_path->nodes[0];
6134 slot = left_path->slots[0];
6135 ei = btrfs_item_ptr(eb, slot, struct btrfs_file_extent_item);
6136 left_type = btrfs_file_extent_type(eb, ei);
6137
6138 if (left_type != BTRFS_FILE_EXTENT_REG)
6139 return 0;
6140
6141 left_disknr = btrfs_file_extent_disk_bytenr(eb, ei);
6142 left_len = btrfs_file_extent_num_bytes(eb, ei);
6143 left_offset = btrfs_file_extent_offset(eb, ei);
6144 left_gen = btrfs_file_extent_generation(eb, ei);
6145
6146 /*
6147 * Following comments will refer to these graphics. L is the left
6148 * extents which we are checking at the moment. 1-8 are the right
6149 * extents that we iterate.
6150 *
6151 * |-----L-----|
6152 * |-1-|-2a-|-3-|-4-|-5-|-6-|
6153 *
6154 * |-----L-----|
6155 * |--1--|-2b-|...(same as above)
6156 *
6157 * Alternative situation. Happens on files where extents got split.
6158 * |-----L-----|
6159 * |-----------7-----------|-6-|
6160 *
6161 * Alternative situation. Happens on files which got larger.
6162 * |-----L-----|
6163 * |-8-|
6164 * Nothing follows after 8.
6165 */
6166
6167 key.objectid = ekey->objectid;
6168 key.type = BTRFS_EXTENT_DATA_KEY;
6169 key.offset = ekey->offset;
6170 ret = btrfs_search_slot_for_read(sctx->parent_root, &key, path, 0, 0);
6171 if (ret < 0)
6172 return ret;
6173 if (ret)
6174 return 0;
6175
6176 /*
6177 * Handle special case where the right side has no extents at all.
6178 */
6179 eb = path->nodes[0];
6180 slot = path->slots[0];
6181 btrfs_item_key_to_cpu(eb, &found_key, slot);
6182 if (found_key.objectid != key.objectid ||
6183 found_key.type != key.type)
6184 /* If we're a hole then just pretend nothing changed */
6185 return (left_disknr ? 0 : 1);
6186
6187 /*
6188 * We're now on 2a, 2b or 7.
6189 */
6190 key = found_key;
6191 while (key.offset < ekey->offset + left_len) {
6192 ei = btrfs_item_ptr(eb, slot, struct btrfs_file_extent_item);
6193 right_type = btrfs_file_extent_type(eb, ei);
6194 if (right_type != BTRFS_FILE_EXTENT_REG &&
6195 right_type != BTRFS_FILE_EXTENT_INLINE)
6196 return 0;
6197
6198 if (right_type == BTRFS_FILE_EXTENT_INLINE) {
6199 right_len = btrfs_file_extent_ram_bytes(eb, ei);
6200 right_len = PAGE_ALIGN(right_len);
6201 } else {
6202 right_len = btrfs_file_extent_num_bytes(eb, ei);
6203 }
6204
6205 /*
6206 * Are we at extent 8? If yes, we know the extent is changed.
6207 * This may only happen on the first iteration.
6208 */
6209 if (found_key.offset + right_len <= ekey->offset)
6210 /* If we're a hole just pretend nothing changed */
6211 return (left_disknr ? 0 : 1);
6212
6213 /*
6214 * We just wanted to see if when we have an inline extent, what
6215 * follows it is a regular extent (wanted to check the above
6216 * condition for inline extents too). This should normally not
6217 * happen but it's possible for example when we have an inline
6218 * compressed extent representing data with a size matching
6219 * the page size (currently the same as sector size).
6220 */
6221 if (right_type == BTRFS_FILE_EXTENT_INLINE)
6222 return 0;
6223
6224 right_disknr = btrfs_file_extent_disk_bytenr(eb, ei);
6225 right_offset = btrfs_file_extent_offset(eb, ei);
6226 right_gen = btrfs_file_extent_generation(eb, ei);
6227
6228 left_offset_fixed = left_offset;
6229 if (key.offset < ekey->offset) {
6230 /* Fix the right offset for 2a and 7. */
6231 right_offset += ekey->offset - key.offset;
6232 } else {
6233 /* Fix the left offset for all behind 2a and 2b */
6234 left_offset_fixed += key.offset - ekey->offset;
6235 }
6236
6237 /*
6238 * Check if we have the same extent.
6239 */
6240 if (left_disknr != right_disknr ||
6241 left_offset_fixed != right_offset ||
6242 left_gen != right_gen)
6243 return 0;
6244
6245 /*
6246 * Go to the next extent.
6247 */
6248 ret = btrfs_next_item(sctx->parent_root, path);
6249 if (ret < 0)
6250 return ret;
6251 if (!ret) {
6252 eb = path->nodes[0];
6253 slot = path->slots[0];
6254 btrfs_item_key_to_cpu(eb, &found_key, slot);
6255 }
6256 if (ret || found_key.objectid != key.objectid ||
6257 found_key.type != key.type) {
6258 key.offset += right_len;
6259 break;
6260 }
6261 if (found_key.offset != key.offset + right_len)
6262 return 0;
6263
6264 key = found_key;
6265 }
6266
6267 /*
6268 * We're now behind the left extent (treat as unchanged) or at the end
6269 * of the right side (treat as changed).
6270 */
6271 if (key.offset >= ekey->offset + left_len)
6272 ret = 1;
6273 else
6274 ret = 0;
6275
6276 return ret;
6277}
6278
6279static int get_last_extent(struct send_ctx *sctx, u64 offset)
6280{
6281 BTRFS_PATH_AUTO_FREE(path);
6282 struct btrfs_root *root = sctx->send_root;
6283 struct btrfs_key key;
6284 int ret;
6285
6286 path = alloc_path_for_send();
6287 if (!path)
6288 return -ENOMEM;
6289
6290 sctx->cur_inode_last_extent = 0;
6291
6292 key.objectid = sctx->cur_ino;
6293 key.type = BTRFS_EXTENT_DATA_KEY;
6294 key.offset = offset;
6295 ret = btrfs_search_slot_for_read(root, &key, path, 0, 1);
6296 if (ret < 0)
6297 return ret;
6298 ret = 0;
6299 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
6300 if (key.objectid != sctx->cur_ino || key.type != BTRFS_EXTENT_DATA_KEY)
6301 return ret;
6302
6303 sctx->cur_inode_last_extent = btrfs_file_extent_end(path);
6304 return ret;
6305}
6306
6307static int range_is_hole_in_parent(struct send_ctx *sctx,
6308 const u64 start,
6309 const u64 end)
6310{
6311 BTRFS_PATH_AUTO_FREE(path);
6312 struct btrfs_key key;
6313 struct btrfs_root *root = sctx->parent_root;
6314 u64 search_start = start;
6315 int ret;
6316
6317 path = alloc_path_for_send();
6318 if (!path)
6319 return -ENOMEM;
6320
6321 key.objectid = sctx->cur_ino;
6322 key.type = BTRFS_EXTENT_DATA_KEY;
6323 key.offset = search_start;
6324 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
6325 if (ret < 0)
6326 return ret;
6327 if (ret > 0 && path->slots[0] > 0)
6328 path->slots[0]--;
6329
6330 while (search_start < end) {
6331 struct extent_buffer *leaf = path->nodes[0];
6332 int slot = path->slots[0];
6333 struct btrfs_file_extent_item *fi;
6334 u64 extent_end;
6335
6336 if (slot >= btrfs_header_nritems(leaf)) {
6337 ret = btrfs_next_leaf(root, path);
6338 if (ret < 0)
6339 return ret;
6340 if (ret > 0)
6341 break;
6342 continue;
6343 }
6344
6345 btrfs_item_key_to_cpu(leaf, &key, slot);
6346 if (key.objectid < sctx->cur_ino ||
6347 key.type < BTRFS_EXTENT_DATA_KEY)
6348 goto next;
6349 if (key.objectid > sctx->cur_ino ||
6350 key.type > BTRFS_EXTENT_DATA_KEY ||
6351 key.offset >= end)
6352 break;
6353
6354 fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
6355 extent_end = btrfs_file_extent_end(path);
6356 if (extent_end <= start)
6357 goto next;
6358 if (btrfs_file_extent_type(leaf, fi) == BTRFS_FILE_EXTENT_INLINE)
6359 return 0;
6360 if (btrfs_file_extent_disk_bytenr(leaf, fi) == 0) {
6361 search_start = extent_end;
6362 goto next;
6363 }
6364 return 0;
6365next:
6366 path->slots[0]++;
6367 }
6368 return 1;
6369}
6370
6371static int maybe_send_hole(struct send_ctx *sctx, struct btrfs_path *path,
6372 struct btrfs_key *key)
6373{
6374 int ret = 0;
6375
6376 if (sctx->cur_ino != key->objectid || !need_send_hole(sctx))
6377 return 0;
6378
6379 /*
6380 * Get last extent's end offset (exclusive) if we haven't determined it
6381 * yet (we're processing the first file extent item that is new), or if
6382 * we're at the first slot of a leaf and the last extent's end is less
6383 * than the current extent's offset, because we might have skipped
6384 * entire leaves that contained only file extent items for our current
6385 * inode. These leaves have a generation number smaller (older) than the
6386 * one in the current leaf and the leaf our last extent came from, and
6387 * are located between these 2 leaves.
6388 */
6389 if ((sctx->cur_inode_last_extent == (u64)-1) ||
6390 (path->slots[0] == 0 && sctx->cur_inode_last_extent < key->offset)) {
6391 ret = get_last_extent(sctx, key->offset - 1);
6392 if (ret)
6393 return ret;
6394 }
6395
6396 if (sctx->cur_inode_last_extent < key->offset) {
6397 ret = range_is_hole_in_parent(sctx,
6398 sctx->cur_inode_last_extent,
6399 key->offset);
6400 if (ret < 0)
6401 return ret;
6402 else if (ret == 0)
6403 ret = send_hole(sctx, key->offset);
6404 else
6405 ret = 0;
6406 }
6407 sctx->cur_inode_last_extent = btrfs_file_extent_end(path);
6408 return ret;
6409}
6410
6411static int process_extent(struct send_ctx *sctx,
6412 struct btrfs_path *path,
6413 struct btrfs_key *key)
6414{
6415 struct clone_root *found_clone = NULL;
6416 int ret = 0;
6417
6418 if (S_ISLNK(sctx->cur_inode_mode))
6419 return 0;
6420 if (unlikely(!S_ISREG(sctx->cur_inode_mode))) {
6421 btrfs_crit(sctx->send_root->fs_info,
6422 "send: extent for non-regular inode %llu root %llu mode 0%llo",
6423 key->objectid, btrfs_root_id(sctx->send_root),
6424 sctx->cur_inode_mode & S_IFMT);
6425 return -EUCLEAN;
6426 }
6427
6428 if (sctx->parent_root && !sctx->cur_inode_new) {
6429 ret = is_extent_unchanged(sctx, path, key);
6430 if (ret < 0)
6431 return ret;
6432 if (ret)
6433 goto out_hole;
6434 } else {
6435 struct btrfs_file_extent_item *ei;
6436 u8 type;
6437
6438 ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
6439 struct btrfs_file_extent_item);
6440 type = btrfs_file_extent_type(path->nodes[0], ei);
6441 if (type == BTRFS_FILE_EXTENT_PREALLOC ||
6442 type == BTRFS_FILE_EXTENT_REG) {
6443 /*
6444 * The send spec does not have a prealloc command yet,
6445 * so just leave a hole for prealloc'ed extents until
6446 * we have enough commands queued up to justify rev'ing
6447 * the send spec.
6448 */
6449 if (type == BTRFS_FILE_EXTENT_PREALLOC)
6450 return 0;
6451
6452 /* Have a hole, just skip it. */
6453 if (btrfs_file_extent_disk_bytenr(path->nodes[0], ei) == 0)
6454 return 0;
6455 }
6456 }
6457
6458 ret = find_extent_clone(sctx, path, key->objectid, key->offset,
6459 sctx->cur_inode_size, &found_clone);
6460 if (ret != -ENOENT && ret < 0)
6461 return ret;
6462
6463 ret = send_write_or_clone(sctx, path, key, found_clone);
6464 if (ret)
6465 return ret;
6466out_hole:
6467 return maybe_send_hole(sctx, path, key);
6468}
6469
6470static int process_all_extents(struct send_ctx *sctx)
6471{
6472 int ret = 0;
6473 int iter_ret = 0;
6474 struct btrfs_root *root;
6475 BTRFS_PATH_AUTO_FREE(path);
6476 struct btrfs_key key;
6477 struct btrfs_key found_key;
6478
6479 root = sctx->send_root;
6480 path = alloc_path_for_send();
6481 if (!path)
6482 return -ENOMEM;
6483
6484 key.objectid = sctx->cmp_key->objectid;
6485 key.type = BTRFS_EXTENT_DATA_KEY;
6486 key.offset = 0;
6487 btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
6488 if (found_key.objectid != key.objectid ||
6489 found_key.type != key.type) {
6490 ret = 0;
6491 break;
6492 }
6493
6494 ret = process_extent(sctx, path, &found_key);
6495 if (ret < 0)
6496 break;
6497 }
6498 /* Catch error found during iteration */
6499 if (iter_ret < 0)
6500 ret = iter_ret;
6501
6502 return ret;
6503}
6504
6505static int process_recorded_refs_if_needed(struct send_ctx *sctx, bool at_end,
6506 bool *pending_move, bool *refs_processed)
6507{
6508 int ret;
6509
6510 if (sctx->cur_ino == 0)
6511 return 0;
6512
6513 if (!at_end && sctx->cur_ino == sctx->cmp_key->objectid &&
6514 sctx->cmp_key->type <= BTRFS_INODE_EXTREF_KEY)
6515 return 0;
6516
6517 if (list_empty(&sctx->new_refs) && list_empty(&sctx->deleted_refs))
6518 return 0;
6519
6520 ret = process_recorded_refs(sctx, pending_move);
6521 if (ret < 0)
6522 return ret;
6523
6524 *refs_processed = true;
6525 return 0;
6526}
6527
6528static int finish_inode_if_needed(struct send_ctx *sctx, bool at_end)
6529{
6530 int ret = 0;
6531 struct btrfs_inode_info info;
6532 u64 left_mode;
6533 u64 left_uid;
6534 u64 left_gid;
6535 u64 left_fileattr;
6536 u64 right_mode;
6537 u64 right_uid;
6538 u64 right_gid;
6539 u64 right_fileattr;
6540 bool need_chmod = false;
6541 bool need_chown = false;
6542 bool need_fileattr = false;
6543 bool need_truncate = true;
6544 bool pending_move = false;
6545 bool refs_processed = false;
6546
6547 if (sctx->ignore_cur_inode)
6548 return 0;
6549
6550 ret = process_recorded_refs_if_needed(sctx, at_end, &pending_move,
6551 &refs_processed);
6552 if (ret < 0)
6553 goto out;
6554
6555 /*
6556 * We have processed the refs and thus need to advance send_progress.
6557 * Now, calls to get_cur_xxx will take the updated refs of the current
6558 * inode into account.
6559 *
6560 * On the other hand, if our current inode is a directory and couldn't
6561 * be moved/renamed because its parent was renamed/moved too and it has
6562 * a higher inode number, we can only move/rename our current inode
6563 * after we moved/renamed its parent. Therefore in this case operate on
6564 * the old path (pre move/rename) of our current inode, and the
6565 * move/rename will be performed later.
6566 */
6567 if (refs_processed && !pending_move)
6568 sctx->send_progress = sctx->cur_ino + 1;
6569
6570 if (sctx->cur_ino == 0 || sctx->cur_inode_deleted)
6571 goto out;
6572 if (!at_end && sctx->cmp_key->objectid == sctx->cur_ino)
6573 goto out;
6574 ret = get_inode_info(sctx->send_root, sctx->cur_ino, &info);
6575 if (ret < 0)
6576 goto out;
6577 left_mode = info.mode;
6578 left_uid = info.uid;
6579 left_gid = info.gid;
6580 left_fileattr = info.fileattr;
6581
6582 if (!sctx->parent_root || sctx->cur_inode_new) {
6583 need_chown = true;
6584 if (!S_ISLNK(sctx->cur_inode_mode))
6585 need_chmod = true;
6586 if (sctx->cur_inode_next_write_offset == sctx->cur_inode_size)
6587 need_truncate = false;
6588 } else {
6589 u64 old_size;
6590
6591 ret = get_inode_info(sctx->parent_root, sctx->cur_ino, &info);
6592 if (ret < 0)
6593 goto out;
6594 old_size = info.size;
6595 right_mode = info.mode;
6596 right_uid = info.uid;
6597 right_gid = info.gid;
6598 right_fileattr = info.fileattr;
6599
6600 if (left_uid != right_uid || left_gid != right_gid)
6601 need_chown = true;
6602 if (!S_ISLNK(sctx->cur_inode_mode) && left_mode != right_mode)
6603 need_chmod = true;
6604 if (!S_ISLNK(sctx->cur_inode_mode) && left_fileattr != right_fileattr)
6605 need_fileattr = true;
6606 if ((old_size == sctx->cur_inode_size) ||
6607 (sctx->cur_inode_size > old_size &&
6608 sctx->cur_inode_next_write_offset == sctx->cur_inode_size))
6609 need_truncate = false;
6610 }
6611
6612 if (S_ISREG(sctx->cur_inode_mode)) {
6613 if (need_send_hole(sctx)) {
6614 if (sctx->cur_inode_last_extent == (u64)-1 ||
6615 sctx->cur_inode_last_extent <
6616 sctx->cur_inode_size) {
6617 ret = get_last_extent(sctx, (u64)-1);
6618 if (ret)
6619 goto out;
6620 }
6621 if (sctx->cur_inode_last_extent < sctx->cur_inode_size) {
6622 ret = range_is_hole_in_parent(sctx,
6623 sctx->cur_inode_last_extent,
6624 sctx->cur_inode_size);
6625 if (ret < 0) {
6626 goto out;
6627 } else if (ret == 0) {
6628 ret = send_hole(sctx, sctx->cur_inode_size);
6629 if (ret < 0)
6630 goto out;
6631 } else {
6632 /* Range is already a hole, skip. */
6633 ret = 0;
6634 }
6635 }
6636 }
6637 if (need_truncate) {
6638 ret = send_truncate(sctx, sctx->cur_ino,
6639 sctx->cur_inode_gen,
6640 sctx->cur_inode_size);
6641 if (ret < 0)
6642 goto out;
6643 }
6644 }
6645
6646 if (need_chown) {
6647 ret = send_chown(sctx, sctx->cur_ino, sctx->cur_inode_gen,
6648 left_uid, left_gid);
6649 if (ret < 0)
6650 goto out;
6651 }
6652 if (need_chmod) {
6653 ret = send_chmod(sctx, sctx->cur_ino, sctx->cur_inode_gen,
6654 left_mode);
6655 if (ret < 0)
6656 goto out;
6657 }
6658 if (need_fileattr) {
6659 ret = send_fileattr(sctx, sctx->cur_ino, sctx->cur_inode_gen,
6660 left_fileattr);
6661 if (ret < 0)
6662 goto out;
6663 }
6664
6665 if (proto_cmd_ok(sctx, BTRFS_SEND_C_ENABLE_VERITY)
6666 && sctx->cur_inode_needs_verity) {
6667 ret = process_verity(sctx);
6668 if (ret < 0)
6669 goto out;
6670 }
6671
6672 ret = send_capabilities(sctx);
6673 if (ret < 0)
6674 goto out;
6675
6676 /*
6677 * If other directory inodes depended on our current directory
6678 * inode's move/rename, now do their move/rename operations.
6679 */
6680 if (!is_waiting_for_move(sctx, sctx->cur_ino)) {
6681 ret = apply_children_dir_moves(sctx);
6682 if (ret)
6683 goto out;
6684 /*
6685 * Need to send that every time, no matter if it actually
6686 * changed between the two trees as we have done changes to
6687 * the inode before. If our inode is a directory and it's
6688 * waiting to be moved/renamed, we will send its utimes when
6689 * it's moved/renamed, therefore we don't need to do it here.
6690 */
6691 sctx->send_progress = sctx->cur_ino + 1;
6692
6693 /*
6694 * If the current inode is a non-empty directory, delay issuing
6695 * the utimes command for it, as it's very likely we have inodes
6696 * with an higher number inside it. We want to issue the utimes
6697 * command only after adding all dentries to it.
6698 */
6699 if (S_ISDIR(sctx->cur_inode_mode) && sctx->cur_inode_size > 0)
6700 ret = cache_dir_utimes(sctx, sctx->cur_ino, sctx->cur_inode_gen);
6701 else
6702 ret = send_utimes(sctx, sctx->cur_ino, sctx->cur_inode_gen);
6703
6704 if (ret < 0)
6705 goto out;
6706 }
6707
6708out:
6709 if (!ret)
6710 ret = trim_dir_utimes_cache(sctx);
6711
6712 return ret;
6713}
6714
6715static void close_current_inode(struct send_ctx *sctx)
6716{
6717 u64 i_size;
6718
6719 if (sctx->cur_inode == NULL)
6720 return;
6721
6722 i_size = i_size_read(sctx->cur_inode);
6723
6724 /*
6725 * If we are doing an incremental send, we may have extents between the
6726 * last processed extent and the i_size that have not been processed
6727 * because they haven't changed but we may have read some of their pages
6728 * through readahead, see the comments at send_extent_data().
6729 */
6730 if (sctx->clean_page_cache && sctx->page_cache_clear_start < i_size)
6731 truncate_inode_pages_range(&sctx->cur_inode->i_data,
6732 sctx->page_cache_clear_start,
6733 round_up(i_size, PAGE_SIZE) - 1);
6734
6735 iput(sctx->cur_inode);
6736 sctx->cur_inode = NULL;
6737}
6738
6739static int changed_inode(struct send_ctx *sctx,
6740 enum btrfs_compare_tree_result result)
6741{
6742 int ret;
6743 struct btrfs_key *key = sctx->cmp_key;
6744 struct btrfs_inode_item *left_ii = NULL;
6745 struct btrfs_inode_item *right_ii = NULL;
6746 u64 left_gen = 0;
6747 u64 right_gen = 0;
6748
6749 close_current_inode(sctx);
6750
6751 sctx->cur_ino = key->objectid;
6752 sctx->cur_inode_new_gen = false;
6753 sctx->cur_inode_last_extent = (u64)-1;
6754 sctx->cur_inode_next_write_offset = 0;
6755 sctx->ignore_cur_inode = false;
6756 fs_path_reset(&sctx->cur_inode_path);
6757
6758 /*
6759 * Set send_progress to current inode. This will tell all get_cur_xxx
6760 * functions that the current inode's refs are not updated yet. Later,
6761 * when process_recorded_refs is finished, it is set to cur_ino + 1.
6762 */
6763 sctx->send_progress = sctx->cur_ino;
6764
6765 if (result == BTRFS_COMPARE_TREE_NEW ||
6766 result == BTRFS_COMPARE_TREE_CHANGED) {
6767 left_ii = btrfs_item_ptr(sctx->left_path->nodes[0],
6768 sctx->left_path->slots[0],
6769 struct btrfs_inode_item);
6770 left_gen = btrfs_inode_generation(sctx->left_path->nodes[0],
6771 left_ii);
6772 } else {
6773 right_ii = btrfs_item_ptr(sctx->right_path->nodes[0],
6774 sctx->right_path->slots[0],
6775 struct btrfs_inode_item);
6776 right_gen = btrfs_inode_generation(sctx->right_path->nodes[0],
6777 right_ii);
6778 }
6779 if (result == BTRFS_COMPARE_TREE_CHANGED) {
6780 right_ii = btrfs_item_ptr(sctx->right_path->nodes[0],
6781 sctx->right_path->slots[0],
6782 struct btrfs_inode_item);
6783
6784 right_gen = btrfs_inode_generation(sctx->right_path->nodes[0],
6785 right_ii);
6786
6787 /*
6788 * The cur_ino = root dir case is special here. We can't treat
6789 * the inode as deleted+reused because it would generate a
6790 * stream that tries to delete/mkdir the root dir.
6791 */
6792 if (left_gen != right_gen &&
6793 sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID)
6794 sctx->cur_inode_new_gen = true;
6795 }
6796
6797 /*
6798 * Normally we do not find inodes with a link count of zero (orphans)
6799 * because the most common case is to create a snapshot and use it
6800 * for a send operation. However other less common use cases involve
6801 * using a subvolume and send it after turning it to RO mode just
6802 * after deleting all hard links of a file while holding an open
6803 * file descriptor against it or turning a RO snapshot into RW mode,
6804 * keep an open file descriptor against a file, delete it and then
6805 * turn the snapshot back to RO mode before using it for a send
6806 * operation. The former is what the receiver operation does.
6807 * Therefore, if we want to send these snapshots soon after they're
6808 * received, we need to handle orphan inodes as well. Moreover, orphans
6809 * can appear not only in the send snapshot but also in the parent
6810 * snapshot. Here are several cases:
6811 *
6812 * Case 1: BTRFS_COMPARE_TREE_NEW
6813 * | send snapshot | action
6814 * --------------------------------
6815 * nlink | 0 | ignore
6816 *
6817 * Case 2: BTRFS_COMPARE_TREE_DELETED
6818 * | parent snapshot | action
6819 * ----------------------------------
6820 * nlink | 0 | as usual
6821 * Note: No unlinks will be sent because there're no paths for it.
6822 *
6823 * Case 3: BTRFS_COMPARE_TREE_CHANGED
6824 * | | parent snapshot | send snapshot | action
6825 * -----------------------------------------------------------------------
6826 * subcase 1 | nlink | 0 | 0 | ignore
6827 * subcase 2 | nlink | >0 | 0 | new_gen(deletion)
6828 * subcase 3 | nlink | 0 | >0 | new_gen(creation)
6829 *
6830 */
6831 if (result == BTRFS_COMPARE_TREE_NEW) {
6832 if (btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii) == 0) {
6833 sctx->ignore_cur_inode = true;
6834 return 0;
6835 }
6836 sctx->cur_inode_gen = left_gen;
6837 sctx->cur_inode_new = true;
6838 sctx->cur_inode_deleted = false;
6839 sctx->cur_inode_size = btrfs_inode_size(
6840 sctx->left_path->nodes[0], left_ii);
6841 sctx->cur_inode_mode = btrfs_inode_mode(
6842 sctx->left_path->nodes[0], left_ii);
6843 sctx->cur_inode_rdev = btrfs_inode_rdev(
6844 sctx->left_path->nodes[0], left_ii);
6845 if (sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID)
6846 ret = send_create_inode_if_needed(sctx);
6847 } else if (result == BTRFS_COMPARE_TREE_DELETED) {
6848 sctx->cur_inode_gen = right_gen;
6849 sctx->cur_inode_new = false;
6850 sctx->cur_inode_deleted = true;
6851 sctx->cur_inode_size = btrfs_inode_size(
6852 sctx->right_path->nodes[0], right_ii);
6853 sctx->cur_inode_mode = btrfs_inode_mode(
6854 sctx->right_path->nodes[0], right_ii);
6855 } else if (result == BTRFS_COMPARE_TREE_CHANGED) {
6856 u32 new_nlinks, old_nlinks;
6857
6858 new_nlinks = btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii);
6859 old_nlinks = btrfs_inode_nlink(sctx->right_path->nodes[0], right_ii);
6860 if (new_nlinks == 0 && old_nlinks == 0) {
6861 sctx->ignore_cur_inode = true;
6862 return 0;
6863 } else if (new_nlinks == 0 || old_nlinks == 0) {
6864 sctx->cur_inode_new_gen = 1;
6865 }
6866 /*
6867 * We need to do some special handling in case the inode was
6868 * reported as changed with a changed generation number. This
6869 * means that the original inode was deleted and new inode
6870 * reused the same inum. So we have to treat the old inode as
6871 * deleted and the new one as new.
6872 */
6873 if (sctx->cur_inode_new_gen) {
6874 /*
6875 * First, process the inode as if it was deleted.
6876 */
6877 if (old_nlinks > 0) {
6878 sctx->cur_inode_gen = right_gen;
6879 sctx->cur_inode_new = false;
6880 sctx->cur_inode_deleted = true;
6881 sctx->cur_inode_size = btrfs_inode_size(
6882 sctx->right_path->nodes[0], right_ii);
6883 sctx->cur_inode_mode = btrfs_inode_mode(
6884 sctx->right_path->nodes[0], right_ii);
6885 ret = process_all_refs(sctx,
6886 BTRFS_COMPARE_TREE_DELETED);
6887 if (ret < 0)
6888 return ret;
6889 }
6890
6891 /*
6892 * Now process the inode as if it was new.
6893 */
6894 if (new_nlinks > 0) {
6895 sctx->cur_inode_gen = left_gen;
6896 sctx->cur_inode_new = true;
6897 sctx->cur_inode_deleted = false;
6898 sctx->cur_inode_size = btrfs_inode_size(
6899 sctx->left_path->nodes[0],
6900 left_ii);
6901 sctx->cur_inode_mode = btrfs_inode_mode(
6902 sctx->left_path->nodes[0],
6903 left_ii);
6904 sctx->cur_inode_rdev = btrfs_inode_rdev(
6905 sctx->left_path->nodes[0],
6906 left_ii);
6907 ret = send_create_inode_if_needed(sctx);
6908 if (ret < 0)
6909 return ret;
6910
6911 ret = process_all_refs(sctx, BTRFS_COMPARE_TREE_NEW);
6912 if (ret < 0)
6913 return ret;
6914 /*
6915 * Advance send_progress now as we did not get
6916 * into process_recorded_refs_if_needed in the
6917 * new_gen case.
6918 */
6919 sctx->send_progress = sctx->cur_ino + 1;
6920
6921 /*
6922 * Now process all extents and xattrs of the
6923 * inode as if they were all new.
6924 */
6925 ret = process_all_extents(sctx);
6926 if (ret < 0)
6927 return ret;
6928 ret = process_all_new_xattrs(sctx);
6929 if (ret < 0)
6930 return ret;
6931 }
6932 } else {
6933 sctx->cur_inode_gen = left_gen;
6934 sctx->cur_inode_new = false;
6935 sctx->cur_inode_new_gen = false;
6936 sctx->cur_inode_deleted = false;
6937 sctx->cur_inode_size = btrfs_inode_size(
6938 sctx->left_path->nodes[0], left_ii);
6939 sctx->cur_inode_mode = btrfs_inode_mode(
6940 sctx->left_path->nodes[0], left_ii);
6941 }
6942 }
6943
6944 return 0;
6945}
6946
6947/*
6948 * We have to process new refs before deleted refs, but compare_trees gives us
6949 * the new and deleted refs mixed. To fix this, we record the new/deleted refs
6950 * first and later process them in process_recorded_refs.
6951 * For the cur_inode_new_gen case, we skip recording completely because
6952 * changed_inode did already initiate processing of refs. The reason for this is
6953 * that in this case, compare_tree actually compares the refs of 2 different
6954 * inodes. To fix this, process_all_refs is used in changed_inode to handle all
6955 * refs of the right tree as deleted and all refs of the left tree as new.
6956 */
6957static int changed_ref(struct send_ctx *sctx,
6958 enum btrfs_compare_tree_result result)
6959{
6960 int ret = 0;
6961
6962 if (unlikely(sctx->cur_ino != sctx->cmp_key->objectid)) {
6963 inconsistent_snapshot_error(sctx, result, "reference");
6964 return -EIO;
6965 }
6966
6967 if (!sctx->cur_inode_new_gen &&
6968 sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID) {
6969 if (result == BTRFS_COMPARE_TREE_NEW)
6970 ret = record_new_ref(sctx);
6971 else if (result == BTRFS_COMPARE_TREE_DELETED)
6972 ret = record_deleted_ref(sctx);
6973 else if (result == BTRFS_COMPARE_TREE_CHANGED)
6974 ret = record_changed_ref(sctx);
6975 }
6976
6977 return ret;
6978}
6979
6980/*
6981 * Process new/deleted/changed xattrs. We skip processing in the
6982 * cur_inode_new_gen case because changed_inode did already initiate processing
6983 * of xattrs. The reason is the same as in changed_ref
6984 */
6985static int changed_xattr(struct send_ctx *sctx,
6986 enum btrfs_compare_tree_result result)
6987{
6988 int ret = 0;
6989
6990 if (unlikely(sctx->cur_ino != sctx->cmp_key->objectid)) {
6991 inconsistent_snapshot_error(sctx, result, "xattr");
6992 return -EIO;
6993 }
6994
6995 if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) {
6996 if (result == BTRFS_COMPARE_TREE_NEW)
6997 ret = process_new_xattr(sctx);
6998 else if (result == BTRFS_COMPARE_TREE_DELETED)
6999 ret = process_deleted_xattr(sctx);
7000 else if (result == BTRFS_COMPARE_TREE_CHANGED)
7001 ret = process_changed_xattr(sctx);
7002 }
7003
7004 return ret;
7005}
7006
7007/*
7008 * Process new/deleted/changed extents. We skip processing in the
7009 * cur_inode_new_gen case because changed_inode did already initiate processing
7010 * of extents. The reason is the same as in changed_ref
7011 */
7012static int changed_extent(struct send_ctx *sctx,
7013 enum btrfs_compare_tree_result result)
7014{
7015 int ret = 0;
7016
7017 /*
7018 * We have found an extent item that changed without the inode item
7019 * having changed. This can happen either after relocation (where the
7020 * disk_bytenr of an extent item is replaced at
7021 * relocation.c:replace_file_extents()) or after deduplication into a
7022 * file in both the parent and send snapshots (where an extent item can
7023 * get modified or replaced with a new one). Note that deduplication
7024 * updates the inode item, but it only changes the iversion (sequence
7025 * field in the inode item) of the inode, so if a file is deduplicated
7026 * the same number of times in both the parent and send snapshots, its
7027 * iversion becomes the same in both snapshots, whence the inode item is
7028 * the same on both snapshots.
7029 */
7030 if (sctx->cur_ino != sctx->cmp_key->objectid)
7031 return 0;
7032
7033 if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) {
7034 if (result != BTRFS_COMPARE_TREE_DELETED)
7035 ret = process_extent(sctx, sctx->left_path,
7036 sctx->cmp_key);
7037 }
7038
7039 return ret;
7040}
7041
7042static int changed_verity(struct send_ctx *sctx, enum btrfs_compare_tree_result result)
7043{
7044 if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) {
7045 if (result == BTRFS_COMPARE_TREE_NEW)
7046 sctx->cur_inode_needs_verity = true;
7047 }
7048 return 0;
7049}
7050
7051static int dir_changed(struct send_ctx *sctx, u64 dir)
7052{
7053 u64 orig_gen, new_gen;
7054 int ret;
7055
7056 ret = get_inode_gen(sctx->send_root, dir, &new_gen);
7057 if (ret)
7058 return ret;
7059
7060 ret = get_inode_gen(sctx->parent_root, dir, &orig_gen);
7061 if (ret)
7062 return ret;
7063
7064 return (orig_gen != new_gen) ? 1 : 0;
7065}
7066
7067static int compare_refs(struct send_ctx *sctx, struct btrfs_path *path,
7068 struct btrfs_key *key)
7069{
7070 struct btrfs_inode_extref *extref;
7071 struct extent_buffer *leaf;
7072 u64 dirid = 0, last_dirid = 0;
7073 unsigned long ptr;
7074 u32 item_size;
7075 u32 cur_offset = 0;
7076 int ref_name_len;
7077
7078 /* Easy case, just check this one dirid */
7079 if (key->type == BTRFS_INODE_REF_KEY) {
7080 dirid = key->offset;
7081
7082 return dir_changed(sctx, dirid);
7083 }
7084
7085 leaf = path->nodes[0];
7086 item_size = btrfs_item_size(leaf, path->slots[0]);
7087 ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
7088 while (cur_offset < item_size) {
7089 int ret;
7090
7091 extref = (struct btrfs_inode_extref *)(ptr +
7092 cur_offset);
7093 dirid = btrfs_inode_extref_parent(leaf, extref);
7094 ref_name_len = btrfs_inode_extref_name_len(leaf, extref);
7095 cur_offset += ref_name_len + sizeof(*extref);
7096 if (dirid == last_dirid)
7097 continue;
7098 ret = dir_changed(sctx, dirid);
7099 if (ret)
7100 return ret;
7101 last_dirid = dirid;
7102 }
7103 return 0;
7104}
7105
7106/*
7107 * Updates compare related fields in sctx and simply forwards to the actual
7108 * changed_xxx functions.
7109 */
7110static int changed_cb(struct btrfs_path *left_path,
7111 struct btrfs_path *right_path,
7112 struct btrfs_key *key,
7113 enum btrfs_compare_tree_result result,
7114 struct send_ctx *sctx)
7115{
7116 int ret;
7117
7118 /*
7119 * We can not hold the commit root semaphore here. This is because in
7120 * the case of sending and receiving to the same filesystem, using a
7121 * pipe, could result in a deadlock:
7122 *
7123 * 1) The task running send blocks on the pipe because it's full;
7124 *
7125 * 2) The task running receive, which is the only consumer of the pipe,
7126 * is waiting for a transaction commit (for example due to a space
7127 * reservation when doing a write or triggering a transaction commit
7128 * when creating a subvolume);
7129 *
7130 * 3) The transaction is waiting to write lock the commit root semaphore,
7131 * but can not acquire it since it's being held at 1).
7132 *
7133 * Down this call chain we write to the pipe through kernel_write().
7134 * The same type of problem can also happen when sending to a file that
7135 * is stored in the same filesystem - when reserving space for a write
7136 * into the file, we can trigger a transaction commit.
7137 *
7138 * Our caller has supplied us with clones of leaves from the send and
7139 * parent roots, so we're safe here from a concurrent relocation and
7140 * further reallocation of metadata extents while we are here. Below we
7141 * also assert that the leaves are clones.
7142 */
7143 lockdep_assert_not_held(&sctx->send_root->fs_info->commit_root_sem);
7144
7145 /*
7146 * We always have a send root, so left_path is never NULL. We will not
7147 * have a leaf when we have reached the end of the send root but have
7148 * not yet reached the end of the parent root.
7149 */
7150 if (left_path->nodes[0])
7151 ASSERT(test_bit(EXTENT_BUFFER_UNMAPPED,
7152 &left_path->nodes[0]->bflags));
7153 /*
7154 * When doing a full send we don't have a parent root, so right_path is
7155 * NULL. When doing an incremental send, we may have reached the end of
7156 * the parent root already, so we don't have a leaf at right_path.
7157 */
7158 if (right_path && right_path->nodes[0])
7159 ASSERT(test_bit(EXTENT_BUFFER_UNMAPPED,
7160 &right_path->nodes[0]->bflags));
7161
7162 if (result == BTRFS_COMPARE_TREE_SAME) {
7163 if (key->type == BTRFS_INODE_REF_KEY ||
7164 key->type == BTRFS_INODE_EXTREF_KEY) {
7165 ret = compare_refs(sctx, left_path, key);
7166 if (!ret)
7167 return 0;
7168 if (ret < 0)
7169 return ret;
7170 } else if (key->type == BTRFS_EXTENT_DATA_KEY) {
7171 return maybe_send_hole(sctx, left_path, key);
7172 } else {
7173 return 0;
7174 }
7175 result = BTRFS_COMPARE_TREE_CHANGED;
7176 }
7177
7178 sctx->left_path = left_path;
7179 sctx->right_path = right_path;
7180 sctx->cmp_key = key;
7181
7182 ret = finish_inode_if_needed(sctx, false);
7183 if (ret < 0)
7184 return ret;
7185
7186 /* Ignore non-FS objects */
7187 if (key->objectid == BTRFS_FREE_INO_OBJECTID ||
7188 key->objectid == BTRFS_FREE_SPACE_OBJECTID)
7189 return 0;
7190
7191 if (key->type == BTRFS_INODE_ITEM_KEY) {
7192 ret = changed_inode(sctx, result);
7193 } else if (!sctx->ignore_cur_inode) {
7194 if (key->type == BTRFS_INODE_REF_KEY ||
7195 key->type == BTRFS_INODE_EXTREF_KEY)
7196 ret = changed_ref(sctx, result);
7197 else if (key->type == BTRFS_XATTR_ITEM_KEY)
7198 ret = changed_xattr(sctx, result);
7199 else if (key->type == BTRFS_EXTENT_DATA_KEY)
7200 ret = changed_extent(sctx, result);
7201 else if (key->type == BTRFS_VERITY_DESC_ITEM_KEY &&
7202 key->offset == 0)
7203 ret = changed_verity(sctx, result);
7204 }
7205
7206 return ret;
7207}
7208
7209static int search_key_again(const struct send_ctx *sctx,
7210 struct btrfs_root *root,
7211 struct btrfs_path *path,
7212 const struct btrfs_key *key)
7213{
7214 int ret;
7215
7216 if (!path->need_commit_sem)
7217 lockdep_assert_held_read(&root->fs_info->commit_root_sem);
7218
7219 /*
7220 * Roots used for send operations are readonly and no one can add,
7221 * update or remove keys from them, so we should be able to find our
7222 * key again. The only exception is deduplication, which can operate on
7223 * readonly roots and add, update or remove keys to/from them - but at
7224 * the moment we don't allow it to run in parallel with send.
7225 */
7226 ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
7227 ASSERT(ret <= 0);
7228 if (unlikely(ret > 0)) {
7229 btrfs_print_tree(path->nodes[path->lowest_level], false);
7230 btrfs_err(root->fs_info,
7231"send: key " BTRFS_KEY_FMT" not found in %s root %llu, lowest_level %d, slot %d",
7232 BTRFS_KEY_FMT_VALUE(key),
7233 (root == sctx->parent_root ? "parent" : "send"),
7234 btrfs_root_id(root), path->lowest_level,
7235 path->slots[path->lowest_level]);
7236 return -EUCLEAN;
7237 }
7238
7239 return ret;
7240}
7241
7242static int full_send_tree(struct send_ctx *sctx)
7243{
7244 int ret;
7245 struct btrfs_root *send_root = sctx->send_root;
7246 struct btrfs_key key;
7247 struct btrfs_fs_info *fs_info = send_root->fs_info;
7248 BTRFS_PATH_AUTO_FREE(path);
7249
7250 path = alloc_path_for_send();
7251 if (!path)
7252 return -ENOMEM;
7253 path->reada = READA_FORWARD_ALWAYS;
7254
7255 key.objectid = BTRFS_FIRST_FREE_OBJECTID;
7256 key.type = BTRFS_INODE_ITEM_KEY;
7257 key.offset = 0;
7258
7259 down_read(&fs_info->commit_root_sem);
7260 sctx->last_reloc_trans = fs_info->last_reloc_trans;
7261 up_read(&fs_info->commit_root_sem);
7262
7263 ret = btrfs_search_slot_for_read(send_root, &key, path, 1, 0);
7264 if (ret < 0)
7265 return ret;
7266 if (ret)
7267 goto out_finish;
7268
7269 while (1) {
7270 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
7271
7272 ret = changed_cb(path, NULL, &key,
7273 BTRFS_COMPARE_TREE_NEW, sctx);
7274 if (ret < 0)
7275 return ret;
7276
7277 down_read(&fs_info->commit_root_sem);
7278 if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
7279 sctx->last_reloc_trans = fs_info->last_reloc_trans;
7280 up_read(&fs_info->commit_root_sem);
7281 /*
7282 * A transaction used for relocating a block group was
7283 * committed or is about to finish its commit. Release
7284 * our path (leaf) and restart the search, so that we
7285 * avoid operating on any file extent items that are
7286 * stale, with a disk_bytenr that reflects a pre
7287 * relocation value. This way we avoid as much as
7288 * possible to fallback to regular writes when checking
7289 * if we can clone file ranges.
7290 */
7291 btrfs_release_path(path);
7292 ret = search_key_again(sctx, send_root, path, &key);
7293 if (ret < 0)
7294 return ret;
7295 } else {
7296 up_read(&fs_info->commit_root_sem);
7297 }
7298
7299 ret = btrfs_next_item(send_root, path);
7300 if (ret < 0)
7301 return ret;
7302 if (ret) {
7303 ret = 0;
7304 break;
7305 }
7306 }
7307
7308out_finish:
7309 return finish_inode_if_needed(sctx, true);
7310}
7311
7312static int replace_node_with_clone(struct btrfs_path *path, int level)
7313{
7314 struct extent_buffer *clone;
7315
7316 clone = btrfs_clone_extent_buffer(path->nodes[level]);
7317 if (!clone)
7318 return -ENOMEM;
7319
7320 free_extent_buffer(path->nodes[level]);
7321 path->nodes[level] = clone;
7322
7323 return 0;
7324}
7325
7326static int tree_move_down(struct btrfs_path *path, int *level, u64 reada_min_gen)
7327{
7328 struct extent_buffer *eb;
7329 struct extent_buffer *parent = path->nodes[*level];
7330 int slot = path->slots[*level];
7331 const int nritems = btrfs_header_nritems(parent);
7332 u64 reada_max;
7333 u64 reada_done = 0;
7334
7335 lockdep_assert_held_read(&parent->fs_info->commit_root_sem);
7336 ASSERT(*level != 0);
7337
7338 eb = btrfs_read_node_slot(parent, slot);
7339 if (IS_ERR(eb))
7340 return PTR_ERR(eb);
7341
7342 /*
7343 * Trigger readahead for the next leaves we will process, so that it is
7344 * very likely that when we need them they are already in memory and we
7345 * will not block on disk IO. For nodes we only do readahead for one,
7346 * since the time window between processing nodes is typically larger.
7347 */
7348 reada_max = (*level == 1 ? SZ_128K : eb->fs_info->nodesize);
7349
7350 for (slot++; slot < nritems && reada_done < reada_max; slot++) {
7351 if (btrfs_node_ptr_generation(parent, slot) > reada_min_gen) {
7352 btrfs_readahead_node_child(parent, slot);
7353 reada_done += eb->fs_info->nodesize;
7354 }
7355 }
7356
7357 path->nodes[*level - 1] = eb;
7358 path->slots[*level - 1] = 0;
7359 (*level)--;
7360
7361 if (*level == 0)
7362 return replace_node_with_clone(path, 0);
7363
7364 return 0;
7365}
7366
7367static int tree_move_next_or_upnext(struct btrfs_path *path,
7368 int *level, int root_level)
7369{
7370 int ret = 0;
7371 int nritems;
7372 nritems = btrfs_header_nritems(path->nodes[*level]);
7373
7374 path->slots[*level]++;
7375
7376 while (path->slots[*level] >= nritems) {
7377 if (*level == root_level) {
7378 path->slots[*level] = nritems - 1;
7379 return -1;
7380 }
7381
7382 /* move upnext */
7383 path->slots[*level] = 0;
7384 free_extent_buffer(path->nodes[*level]);
7385 path->nodes[*level] = NULL;
7386 (*level)++;
7387 path->slots[*level]++;
7388
7389 nritems = btrfs_header_nritems(path->nodes[*level]);
7390 ret = 1;
7391 }
7392 return ret;
7393}
7394
7395/*
7396 * Returns 1 if it had to move up and next. 0 is returned if it moved only next
7397 * or down.
7398 */
7399static int tree_advance(struct btrfs_path *path,
7400 int *level, int root_level,
7401 int allow_down,
7402 struct btrfs_key *key,
7403 u64 reada_min_gen)
7404{
7405 int ret;
7406
7407 if (*level == 0 || !allow_down) {
7408 ret = tree_move_next_or_upnext(path, level, root_level);
7409 } else {
7410 ret = tree_move_down(path, level, reada_min_gen);
7411 }
7412
7413 /*
7414 * Even if we have reached the end of a tree, ret is -1, update the key
7415 * anyway, so that in case we need to restart due to a block group
7416 * relocation, we can assert that the last key of the root node still
7417 * exists in the tree.
7418 */
7419 if (*level == 0)
7420 btrfs_item_key_to_cpu(path->nodes[*level], key,
7421 path->slots[*level]);
7422 else
7423 btrfs_node_key_to_cpu(path->nodes[*level], key,
7424 path->slots[*level]);
7425
7426 return ret;
7427}
7428
7429static int tree_compare_item(struct btrfs_path *left_path,
7430 struct btrfs_path *right_path,
7431 char *tmp_buf)
7432{
7433 int cmp;
7434 int len1, len2;
7435 unsigned long off1, off2;
7436
7437 len1 = btrfs_item_size(left_path->nodes[0], left_path->slots[0]);
7438 len2 = btrfs_item_size(right_path->nodes[0], right_path->slots[0]);
7439 if (len1 != len2)
7440 return 1;
7441
7442 off1 = btrfs_item_ptr_offset(left_path->nodes[0], left_path->slots[0]);
7443 off2 = btrfs_item_ptr_offset(right_path->nodes[0],
7444 right_path->slots[0]);
7445
7446 read_extent_buffer(left_path->nodes[0], tmp_buf, off1, len1);
7447
7448 cmp = memcmp_extent_buffer(right_path->nodes[0], tmp_buf, off2, len1);
7449 if (cmp)
7450 return 1;
7451 return 0;
7452}
7453
7454/*
7455 * A transaction used for relocating a block group was committed or is about to
7456 * finish its commit. Release our paths and restart the search, so that we are
7457 * not using stale extent buffers:
7458 *
7459 * 1) For levels > 0, we are only holding references of extent buffers, without
7460 * any locks on them, which does not prevent them from having been relocated
7461 * and reallocated after the last time we released the commit root semaphore.
7462 * The exception are the root nodes, for which we always have a clone, see
7463 * the comment at btrfs_compare_trees();
7464 *
7465 * 2) For leaves, level 0, we are holding copies (clones) of extent buffers, so
7466 * we are safe from the concurrent relocation and reallocation. However they
7467 * can have file extent items with a pre relocation disk_bytenr value, so we
7468 * restart the start from the current commit roots and clone the new leaves so
7469 * that we get the post relocation disk_bytenr values. Not doing so, could
7470 * make us clone the wrong data in case there are new extents using the old
7471 * disk_bytenr that happen to be shared.
7472 */
7473static int restart_after_relocation(struct btrfs_path *left_path,
7474 struct btrfs_path *right_path,
7475 const struct btrfs_key *left_key,
7476 const struct btrfs_key *right_key,
7477 int left_level,
7478 int right_level,
7479 const struct send_ctx *sctx)
7480{
7481 int root_level;
7482 int ret;
7483
7484 lockdep_assert_held_read(&sctx->send_root->fs_info->commit_root_sem);
7485
7486 btrfs_release_path(left_path);
7487 btrfs_release_path(right_path);
7488
7489 /*
7490 * Since keys can not be added or removed to/from our roots because they
7491 * are readonly and we do not allow deduplication to run in parallel
7492 * (which can add, remove or change keys), the layout of the trees should
7493 * not change.
7494 */
7495 left_path->lowest_level = left_level;
7496 ret = search_key_again(sctx, sctx->send_root, left_path, left_key);
7497 if (ret < 0)
7498 return ret;
7499
7500 right_path->lowest_level = right_level;
7501 ret = search_key_again(sctx, sctx->parent_root, right_path, right_key);
7502 if (ret < 0)
7503 return ret;
7504
7505 /*
7506 * If the lowest level nodes are leaves, clone them so that they can be
7507 * safely used by changed_cb() while not under the protection of the
7508 * commit root semaphore, even if relocation and reallocation happens in
7509 * parallel.
7510 */
7511 if (left_level == 0) {
7512 ret = replace_node_with_clone(left_path, 0);
7513 if (ret < 0)
7514 return ret;
7515 }
7516
7517 if (right_level == 0) {
7518 ret = replace_node_with_clone(right_path, 0);
7519 if (ret < 0)
7520 return ret;
7521 }
7522
7523 /*
7524 * Now clone the root nodes (unless they happen to be the leaves we have
7525 * already cloned). This is to protect against concurrent snapshotting of
7526 * the send and parent roots (see the comment at btrfs_compare_trees()).
7527 */
7528 root_level = btrfs_header_level(sctx->send_root->commit_root);
7529 if (root_level > 0) {
7530 ret = replace_node_with_clone(left_path, root_level);
7531 if (ret < 0)
7532 return ret;
7533 }
7534
7535 root_level = btrfs_header_level(sctx->parent_root->commit_root);
7536 if (root_level > 0) {
7537 ret = replace_node_with_clone(right_path, root_level);
7538 if (ret < 0)
7539 return ret;
7540 }
7541
7542 return 0;
7543}
7544
7545/*
7546 * This function compares two trees and calls the provided callback for
7547 * every changed/new/deleted item it finds.
7548 * If shared tree blocks are encountered, whole subtrees are skipped, making
7549 * the compare pretty fast on snapshotted subvolumes.
7550 *
7551 * This currently works on commit roots only. As commit roots are read only,
7552 * we don't do any locking. The commit roots are protected with transactions.
7553 * Transactions are ended and rejoined when a commit is tried in between.
7554 *
7555 * This function checks for modifications done to the trees while comparing.
7556 * If it detects a change, it aborts immediately.
7557 */
7558static int btrfs_compare_trees(struct btrfs_root *left_root,
7559 struct btrfs_root *right_root, struct send_ctx *sctx)
7560{
7561 struct btrfs_fs_info *fs_info = left_root->fs_info;
7562 int ret;
7563 int cmp;
7564 BTRFS_PATH_AUTO_FREE(left_path);
7565 BTRFS_PATH_AUTO_FREE(right_path);
7566 struct btrfs_key left_key;
7567 struct btrfs_key right_key;
7568 char *tmp_buf = NULL;
7569 int left_root_level;
7570 int right_root_level;
7571 int left_level;
7572 int right_level;
7573 int left_end_reached = 0;
7574 int right_end_reached = 0;
7575 int advance_left = 0;
7576 int advance_right = 0;
7577 u64 left_blockptr;
7578 u64 right_blockptr;
7579 u64 left_gen;
7580 u64 right_gen;
7581 u64 reada_min_gen;
7582
7583 left_path = btrfs_alloc_path();
7584 if (!left_path) {
7585 ret = -ENOMEM;
7586 goto out;
7587 }
7588 right_path = btrfs_alloc_path();
7589 if (!right_path) {
7590 ret = -ENOMEM;
7591 goto out;
7592 }
7593
7594 tmp_buf = kvmalloc(fs_info->nodesize, GFP_KERNEL);
7595 if (!tmp_buf) {
7596 ret = -ENOMEM;
7597 goto out;
7598 }
7599
7600 left_path->search_commit_root = true;
7601 left_path->skip_locking = true;
7602 right_path->search_commit_root = true;
7603 right_path->skip_locking = true;
7604
7605 /*
7606 * Strategy: Go to the first items of both trees. Then do
7607 *
7608 * If both trees are at level 0
7609 * Compare keys of current items
7610 * If left < right treat left item as new, advance left tree
7611 * and repeat
7612 * If left > right treat right item as deleted, advance right tree
7613 * and repeat
7614 * If left == right do deep compare of items, treat as changed if
7615 * needed, advance both trees and repeat
7616 * If both trees are at the same level but not at level 0
7617 * Compare keys of current nodes/leafs
7618 * If left < right advance left tree and repeat
7619 * If left > right advance right tree and repeat
7620 * If left == right compare blockptrs of the next nodes/leafs
7621 * If they match advance both trees but stay at the same level
7622 * and repeat
7623 * If they don't match advance both trees while allowing to go
7624 * deeper and repeat
7625 * If tree levels are different
7626 * Advance the tree that needs it and repeat
7627 *
7628 * Advancing a tree means:
7629 * If we are at level 0, try to go to the next slot. If that's not
7630 * possible, go one level up and repeat. Stop when we found a level
7631 * where we could go to the next slot. We may at this point be on a
7632 * node or a leaf.
7633 *
7634 * If we are not at level 0 and not on shared tree blocks, go one
7635 * level deeper.
7636 *
7637 * If we are not at level 0 and on shared tree blocks, go one slot to
7638 * the right if possible or go up and right.
7639 */
7640
7641 down_read(&fs_info->commit_root_sem);
7642 left_level = btrfs_header_level(left_root->commit_root);
7643 left_root_level = left_level;
7644 /*
7645 * We clone the root node of the send and parent roots to prevent races
7646 * with snapshot creation of these roots. Snapshot creation COWs the
7647 * root node of a tree, so after the transaction is committed the old
7648 * extent can be reallocated while this send operation is still ongoing.
7649 * So we clone them, under the commit root semaphore, to be race free.
7650 */
7651 left_path->nodes[left_level] =
7652 btrfs_clone_extent_buffer(left_root->commit_root);
7653 if (!left_path->nodes[left_level]) {
7654 ret = -ENOMEM;
7655 goto out_unlock;
7656 }
7657
7658 right_level = btrfs_header_level(right_root->commit_root);
7659 right_root_level = right_level;
7660 right_path->nodes[right_level] =
7661 btrfs_clone_extent_buffer(right_root->commit_root);
7662 if (!right_path->nodes[right_level]) {
7663 ret = -ENOMEM;
7664 goto out_unlock;
7665 }
7666 /*
7667 * Our right root is the parent root, while the left root is the "send"
7668 * root. We know that all new nodes/leaves in the left root must have
7669 * a generation greater than the right root's generation, so we trigger
7670 * readahead for those nodes and leaves of the left root, as we know we
7671 * will need to read them at some point.
7672 */
7673 reada_min_gen = btrfs_header_generation(right_root->commit_root);
7674
7675 if (left_level == 0)
7676 btrfs_item_key_to_cpu(left_path->nodes[left_level],
7677 &left_key, left_path->slots[left_level]);
7678 else
7679 btrfs_node_key_to_cpu(left_path->nodes[left_level],
7680 &left_key, left_path->slots[left_level]);
7681 if (right_level == 0)
7682 btrfs_item_key_to_cpu(right_path->nodes[right_level],
7683 &right_key, right_path->slots[right_level]);
7684 else
7685 btrfs_node_key_to_cpu(right_path->nodes[right_level],
7686 &right_key, right_path->slots[right_level]);
7687
7688 sctx->last_reloc_trans = fs_info->last_reloc_trans;
7689
7690 while (1) {
7691 if (need_resched() ||
7692 rwsem_is_contended(&fs_info->commit_root_sem)) {
7693 up_read(&fs_info->commit_root_sem);
7694 cond_resched();
7695 down_read(&fs_info->commit_root_sem);
7696 }
7697
7698 if (fs_info->last_reloc_trans > sctx->last_reloc_trans) {
7699 ret = restart_after_relocation(left_path, right_path,
7700 &left_key, &right_key,
7701 left_level, right_level,
7702 sctx);
7703 if (ret < 0)
7704 goto out_unlock;
7705 sctx->last_reloc_trans = fs_info->last_reloc_trans;
7706 }
7707
7708 if (advance_left && !left_end_reached) {
7709 ret = tree_advance(left_path, &left_level,
7710 left_root_level,
7711 advance_left != ADVANCE_ONLY_NEXT,
7712 &left_key, reada_min_gen);
7713 if (ret == -1)
7714 left_end_reached = ADVANCE;
7715 else if (ret < 0)
7716 goto out_unlock;
7717 advance_left = 0;
7718 }
7719 if (advance_right && !right_end_reached) {
7720 ret = tree_advance(right_path, &right_level,
7721 right_root_level,
7722 advance_right != ADVANCE_ONLY_NEXT,
7723 &right_key, reada_min_gen);
7724 if (ret == -1)
7725 right_end_reached = ADVANCE;
7726 else if (ret < 0)
7727 goto out_unlock;
7728 advance_right = 0;
7729 }
7730
7731 if (left_end_reached && right_end_reached) {
7732 ret = 0;
7733 goto out_unlock;
7734 } else if (left_end_reached) {
7735 if (right_level == 0) {
7736 up_read(&fs_info->commit_root_sem);
7737 ret = changed_cb(left_path, right_path,
7738 &right_key,
7739 BTRFS_COMPARE_TREE_DELETED,
7740 sctx);
7741 if (ret < 0)
7742 goto out;
7743 down_read(&fs_info->commit_root_sem);
7744 }
7745 advance_right = ADVANCE;
7746 continue;
7747 } else if (right_end_reached) {
7748 if (left_level == 0) {
7749 up_read(&fs_info->commit_root_sem);
7750 ret = changed_cb(left_path, right_path,
7751 &left_key,
7752 BTRFS_COMPARE_TREE_NEW,
7753 sctx);
7754 if (ret < 0)
7755 goto out;
7756 down_read(&fs_info->commit_root_sem);
7757 }
7758 advance_left = ADVANCE;
7759 continue;
7760 }
7761
7762 if (left_level == 0 && right_level == 0) {
7763 up_read(&fs_info->commit_root_sem);
7764 cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
7765 if (cmp < 0) {
7766 ret = changed_cb(left_path, right_path,
7767 &left_key,
7768 BTRFS_COMPARE_TREE_NEW,
7769 sctx);
7770 advance_left = ADVANCE;
7771 } else if (cmp > 0) {
7772 ret = changed_cb(left_path, right_path,
7773 &right_key,
7774 BTRFS_COMPARE_TREE_DELETED,
7775 sctx);
7776 advance_right = ADVANCE;
7777 } else {
7778 enum btrfs_compare_tree_result result;
7779
7780 WARN_ON(!extent_buffer_uptodate(left_path->nodes[0]));
7781 ret = tree_compare_item(left_path, right_path,
7782 tmp_buf);
7783 if (ret)
7784 result = BTRFS_COMPARE_TREE_CHANGED;
7785 else
7786 result = BTRFS_COMPARE_TREE_SAME;
7787 ret = changed_cb(left_path, right_path,
7788 &left_key, result, sctx);
7789 advance_left = ADVANCE;
7790 advance_right = ADVANCE;
7791 }
7792
7793 if (ret < 0)
7794 goto out;
7795 down_read(&fs_info->commit_root_sem);
7796 } else if (left_level == right_level) {
7797 cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
7798 if (cmp < 0) {
7799 advance_left = ADVANCE;
7800 } else if (cmp > 0) {
7801 advance_right = ADVANCE;
7802 } else {
7803 left_blockptr = btrfs_node_blockptr(
7804 left_path->nodes[left_level],
7805 left_path->slots[left_level]);
7806 right_blockptr = btrfs_node_blockptr(
7807 right_path->nodes[right_level],
7808 right_path->slots[right_level]);
7809 left_gen = btrfs_node_ptr_generation(
7810 left_path->nodes[left_level],
7811 left_path->slots[left_level]);
7812 right_gen = btrfs_node_ptr_generation(
7813 right_path->nodes[right_level],
7814 right_path->slots[right_level]);
7815 if (left_blockptr == right_blockptr &&
7816 left_gen == right_gen) {
7817 /*
7818 * As we're on a shared block, don't
7819 * allow to go deeper.
7820 */
7821 advance_left = ADVANCE_ONLY_NEXT;
7822 advance_right = ADVANCE_ONLY_NEXT;
7823 } else {
7824 advance_left = ADVANCE;
7825 advance_right = ADVANCE;
7826 }
7827 }
7828 } else if (left_level < right_level) {
7829 advance_right = ADVANCE;
7830 } else {
7831 advance_left = ADVANCE;
7832 }
7833 }
7834
7835out_unlock:
7836 up_read(&fs_info->commit_root_sem);
7837out:
7838 kvfree(tmp_buf);
7839 return ret;
7840}
7841
7842static int send_subvol(struct send_ctx *sctx)
7843{
7844 int ret;
7845
7846 if (!(sctx->flags & BTRFS_SEND_FLAG_OMIT_STREAM_HEADER)) {
7847 ret = send_header(sctx);
7848 if (ret < 0)
7849 goto out;
7850 }
7851
7852 ret = send_subvol_begin(sctx);
7853 if (ret < 0)
7854 goto out;
7855
7856 if (sctx->parent_root) {
7857 ret = btrfs_compare_trees(sctx->send_root, sctx->parent_root, sctx);
7858 if (ret < 0)
7859 goto out;
7860 ret = finish_inode_if_needed(sctx, true);
7861 if (ret < 0)
7862 goto out;
7863 } else {
7864 ret = full_send_tree(sctx);
7865 if (ret < 0)
7866 goto out;
7867 }
7868
7869out:
7870 free_recorded_refs(sctx);
7871 return ret;
7872}
7873
7874/*
7875 * If orphan cleanup did remove any orphans from a root, it means the tree
7876 * was modified and therefore the commit root is not the same as the current
7877 * root anymore. This is a problem, because send uses the commit root and
7878 * therefore can see inode items that don't exist in the current root anymore,
7879 * and for example make calls to btrfs_iget, which will do tree lookups based
7880 * on the current root and not on the commit root. Those lookups will fail,
7881 * returning a -ESTALE error, and making send fail with that error. So make
7882 * sure a send does not see any orphans we have just removed, and that it will
7883 * see the same inodes regardless of whether a transaction commit happened
7884 * before it started (meaning that the commit root will be the same as the
7885 * current root) or not.
7886 */
7887static int ensure_commit_roots_uptodate(struct send_ctx *sctx)
7888{
7889 struct btrfs_root *root = sctx->parent_root;
7890
7891 if (root && root->node != root->commit_root)
7892 return btrfs_commit_current_transaction(root);
7893
7894 for (int i = 0; i < sctx->clone_roots_cnt; i++) {
7895 root = sctx->clone_roots[i].root;
7896 if (root->node != root->commit_root)
7897 return btrfs_commit_current_transaction(root);
7898 }
7899
7900 return 0;
7901}
7902
7903/*
7904 * Make sure any existing delalloc is flushed for any root used by a send
7905 * operation so that we do not miss any data and we do not race with writeback
7906 * finishing and changing a tree while send is using the tree. This could
7907 * happen if a subvolume is in RW mode, has delalloc, is turned to RO mode and
7908 * a send operation then uses the subvolume.
7909 * After flushing delalloc ensure_commit_roots_uptodate() must be called.
7910 */
7911static int flush_delalloc_roots(struct send_ctx *sctx)
7912{
7913 struct btrfs_root *root = sctx->parent_root;
7914 int ret;
7915 int i;
7916
7917 if (root) {
7918 ret = btrfs_start_delalloc_snapshot(root, false);
7919 if (ret)
7920 return ret;
7921 btrfs_wait_ordered_extents(root, U64_MAX, NULL);
7922 }
7923
7924 for (i = 0; i < sctx->clone_roots_cnt; i++) {
7925 root = sctx->clone_roots[i].root;
7926 ret = btrfs_start_delalloc_snapshot(root, false);
7927 if (ret)
7928 return ret;
7929 btrfs_wait_ordered_extents(root, U64_MAX, NULL);
7930 }
7931
7932 return 0;
7933}
7934
7935static void btrfs_root_dec_send_in_progress(struct btrfs_root* root)
7936{
7937 spin_lock(&root->root_item_lock);
7938 root->send_in_progress--;
7939 /*
7940 * Not much left to do, we don't know why it's unbalanced and
7941 * can't blindly reset it to 0.
7942 */
7943 if (root->send_in_progress < 0)
7944 btrfs_err(root->fs_info,
7945 "send_in_progress unbalanced %d root %llu",
7946 root->send_in_progress, btrfs_root_id(root));
7947 spin_unlock(&root->root_item_lock);
7948}
7949
7950static void dedupe_in_progress_warn(const struct btrfs_root *root)
7951{
7952 btrfs_warn_rl(root->fs_info,
7953"cannot use root %llu for send while deduplications on it are in progress (%d in progress)",
7954 btrfs_root_id(root), root->dedupe_in_progress);
7955}
7956
7957long btrfs_ioctl_send(struct btrfs_root *send_root, const struct btrfs_ioctl_send_args *arg)
7958{
7959 int ret = 0;
7960 struct btrfs_fs_info *fs_info = send_root->fs_info;
7961 struct btrfs_root *clone_root;
7962 struct send_ctx *sctx = NULL;
7963 u32 i;
7964 u64 *clone_sources_tmp = NULL;
7965 int clone_sources_to_rollback = 0;
7966 size_t alloc_size;
7967 bool sort_clone_roots = false;
7968 struct btrfs_lru_cache_entry *entry;
7969 struct btrfs_lru_cache_entry *tmp;
7970
7971 if (!capable(CAP_SYS_ADMIN))
7972 return -EPERM;
7973
7974 /*
7975 * The subvolume must remain read-only during send, protect against
7976 * making it RW. This also protects against deletion.
7977 */
7978 spin_lock(&send_root->root_item_lock);
7979 /*
7980 * Unlikely but possible, if the subvolume is marked for deletion but
7981 * is slow to remove the directory entry, send can still be started.
7982 */
7983 if (btrfs_root_dead(send_root)) {
7984 spin_unlock(&send_root->root_item_lock);
7985 return -EPERM;
7986 }
7987 /* Userspace tools do the checks and warn the user if it's not RO. */
7988 if (!btrfs_root_readonly(send_root)) {
7989 spin_unlock(&send_root->root_item_lock);
7990 return -EPERM;
7991 }
7992 if (send_root->dedupe_in_progress) {
7993 dedupe_in_progress_warn(send_root);
7994 spin_unlock(&send_root->root_item_lock);
7995 return -EAGAIN;
7996 }
7997 send_root->send_in_progress++;
7998 spin_unlock(&send_root->root_item_lock);
7999
8000 /*
8001 * Check that we don't overflow at later allocations, we request
8002 * clone_sources_count + 1 items, and compare to unsigned long inside
8003 * access_ok. Also set an upper limit for allocation size so this can't
8004 * easily exhaust memory. Max number of clone sources is about 200K.
8005 */
8006 if (arg->clone_sources_count > SZ_8M / sizeof(struct clone_root)) {
8007 ret = -EINVAL;
8008 goto out;
8009 }
8010
8011 if (arg->flags & ~BTRFS_SEND_FLAG_MASK) {
8012 ret = -EOPNOTSUPP;
8013 goto out;
8014 }
8015
8016 sctx = kzalloc_obj(struct send_ctx);
8017 if (!sctx) {
8018 ret = -ENOMEM;
8019 goto out;
8020 }
8021
8022 init_path(&sctx->cur_inode_path);
8023 INIT_LIST_HEAD(&sctx->new_refs);
8024 INIT_LIST_HEAD(&sctx->deleted_refs);
8025
8026 btrfs_lru_cache_init(&sctx->name_cache, SEND_MAX_NAME_CACHE_SIZE);
8027 btrfs_lru_cache_init(&sctx->backref_cache, SEND_MAX_BACKREF_CACHE_SIZE);
8028 btrfs_lru_cache_init(&sctx->dir_created_cache,
8029 SEND_MAX_DIR_CREATED_CACHE_SIZE);
8030 /*
8031 * This cache is periodically trimmed to a fixed size elsewhere, see
8032 * cache_dir_utimes() and trim_dir_utimes_cache().
8033 */
8034 btrfs_lru_cache_init(&sctx->dir_utimes_cache, 0);
8035
8036 sctx->pending_dir_moves = RB_ROOT;
8037 sctx->waiting_dir_moves = RB_ROOT;
8038 sctx->orphan_dirs = RB_ROOT;
8039 sctx->rbtree_new_refs = RB_ROOT;
8040 sctx->rbtree_deleted_refs = RB_ROOT;
8041
8042 sctx->flags = arg->flags;
8043
8044 if (arg->flags & BTRFS_SEND_FLAG_VERSION) {
8045 if (arg->version > BTRFS_SEND_STREAM_VERSION) {
8046 ret = -EPROTO;
8047 goto out;
8048 }
8049 /* Zero means "use the highest version" */
8050 sctx->proto = arg->version ?: BTRFS_SEND_STREAM_VERSION;
8051 } else {
8052 sctx->proto = 1;
8053 }
8054 if ((arg->flags & BTRFS_SEND_FLAG_COMPRESSED) && sctx->proto < 2) {
8055 ret = -EINVAL;
8056 goto out;
8057 }
8058
8059 sctx->send_filp = fget(arg->send_fd);
8060 if (!sctx->send_filp || !(sctx->send_filp->f_mode & FMODE_WRITE)) {
8061 ret = -EBADF;
8062 goto out;
8063 }
8064
8065 sctx->send_root = send_root;
8066 sctx->clone_roots_cnt = arg->clone_sources_count;
8067
8068 if (sctx->proto >= 2) {
8069 u32 send_buf_num_pages;
8070
8071 sctx->send_max_size = BTRFS_SEND_BUF_SIZE_V2;
8072 sctx->send_buf = vmalloc(sctx->send_max_size);
8073 if (!sctx->send_buf) {
8074 ret = -ENOMEM;
8075 goto out;
8076 }
8077 send_buf_num_pages = sctx->send_max_size >> PAGE_SHIFT;
8078 sctx->send_buf_pages = kzalloc_objs(*sctx->send_buf_pages,
8079 send_buf_num_pages);
8080 if (!sctx->send_buf_pages) {
8081 ret = -ENOMEM;
8082 goto out;
8083 }
8084 for (i = 0; i < send_buf_num_pages; i++) {
8085 sctx->send_buf_pages[i] =
8086 vmalloc_to_page(sctx->send_buf + (i << PAGE_SHIFT));
8087 }
8088 } else {
8089 sctx->send_max_size = BTRFS_SEND_BUF_SIZE_V1;
8090 sctx->send_buf = kvmalloc(sctx->send_max_size, GFP_KERNEL);
8091 }
8092 if (!sctx->send_buf) {
8093 ret = -ENOMEM;
8094 goto out;
8095 }
8096
8097 sctx->clone_roots = kvzalloc_objs(*sctx->clone_roots,
8098 arg->clone_sources_count + 1);
8099 if (!sctx->clone_roots) {
8100 ret = -ENOMEM;
8101 goto out;
8102 }
8103
8104 alloc_size = array_size(sizeof(*arg->clone_sources),
8105 arg->clone_sources_count);
8106
8107 if (arg->clone_sources_count) {
8108 clone_sources_tmp = kvmalloc(alloc_size, GFP_KERNEL);
8109 if (!clone_sources_tmp) {
8110 ret = -ENOMEM;
8111 goto out;
8112 }
8113
8114 ret = copy_from_user(clone_sources_tmp, arg->clone_sources,
8115 alloc_size);
8116 if (ret) {
8117 ret = -EFAULT;
8118 goto out;
8119 }
8120
8121 for (i = 0; i < arg->clone_sources_count; i++) {
8122 clone_root = btrfs_get_fs_root(fs_info,
8123 clone_sources_tmp[i], true);
8124 if (IS_ERR(clone_root)) {
8125 ret = PTR_ERR(clone_root);
8126 goto out;
8127 }
8128 spin_lock(&clone_root->root_item_lock);
8129 if (!btrfs_root_readonly(clone_root) ||
8130 btrfs_root_dead(clone_root)) {
8131 spin_unlock(&clone_root->root_item_lock);
8132 btrfs_put_root(clone_root);
8133 ret = -EPERM;
8134 goto out;
8135 }
8136 if (clone_root->dedupe_in_progress) {
8137 dedupe_in_progress_warn(clone_root);
8138 spin_unlock(&clone_root->root_item_lock);
8139 btrfs_put_root(clone_root);
8140 ret = -EAGAIN;
8141 goto out;
8142 }
8143 clone_root->send_in_progress++;
8144 spin_unlock(&clone_root->root_item_lock);
8145
8146 sctx->clone_roots[i].root = clone_root;
8147 clone_sources_to_rollback = i + 1;
8148 }
8149 kvfree(clone_sources_tmp);
8150 clone_sources_tmp = NULL;
8151 }
8152
8153 if (arg->parent_root) {
8154 sctx->parent_root = btrfs_get_fs_root(fs_info, arg->parent_root,
8155 true);
8156 if (IS_ERR(sctx->parent_root)) {
8157 ret = PTR_ERR(sctx->parent_root);
8158 goto out;
8159 }
8160
8161 spin_lock(&sctx->parent_root->root_item_lock);
8162 sctx->parent_root->send_in_progress++;
8163 if (!btrfs_root_readonly(sctx->parent_root) ||
8164 btrfs_root_dead(sctx->parent_root)) {
8165 spin_unlock(&sctx->parent_root->root_item_lock);
8166 ret = -EPERM;
8167 goto out;
8168 }
8169 if (sctx->parent_root->dedupe_in_progress) {
8170 dedupe_in_progress_warn(sctx->parent_root);
8171 spin_unlock(&sctx->parent_root->root_item_lock);
8172 ret = -EAGAIN;
8173 goto out;
8174 }
8175 spin_unlock(&sctx->parent_root->root_item_lock);
8176 }
8177
8178 /*
8179 * Clones from send_root are allowed, but only if the clone source
8180 * is behind the current send position. This is checked while searching
8181 * for possible clone sources.
8182 */
8183 sctx->clone_roots[sctx->clone_roots_cnt++].root =
8184 btrfs_grab_root(sctx->send_root);
8185
8186 /* We do a bsearch later */
8187 sort(sctx->clone_roots, sctx->clone_roots_cnt,
8188 sizeof(*sctx->clone_roots), __clone_root_cmp_sort,
8189 NULL);
8190 sort_clone_roots = true;
8191
8192 ret = flush_delalloc_roots(sctx);
8193 if (ret)
8194 goto out;
8195
8196 ret = ensure_commit_roots_uptodate(sctx);
8197 if (ret)
8198 goto out;
8199
8200 ret = send_subvol(sctx);
8201 if (ret < 0)
8202 goto out;
8203
8204 btrfs_lru_cache_for_each_entry_safe(&sctx->dir_utimes_cache, entry, tmp) {
8205 ret = send_utimes(sctx, entry->key, entry->gen);
8206 if (ret < 0)
8207 goto out;
8208 btrfs_lru_cache_remove(&sctx->dir_utimes_cache, entry);
8209 }
8210
8211 if (!(sctx->flags & BTRFS_SEND_FLAG_OMIT_END_CMD)) {
8212 ret = begin_cmd(sctx, BTRFS_SEND_C_END);
8213 if (ret < 0)
8214 goto out;
8215 ret = send_cmd(sctx);
8216 if (ret < 0)
8217 goto out;
8218 }
8219
8220out:
8221 WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->pending_dir_moves));
8222 while (sctx && !RB_EMPTY_ROOT(&sctx->pending_dir_moves)) {
8223 struct rb_node *n;
8224 struct pending_dir_move *pm;
8225
8226 n = rb_first(&sctx->pending_dir_moves);
8227 pm = rb_entry(n, struct pending_dir_move, node);
8228 while (!list_empty(&pm->list)) {
8229 struct pending_dir_move *pm2;
8230
8231 pm2 = list_first_entry(&pm->list,
8232 struct pending_dir_move, list);
8233 free_pending_move(sctx, pm2);
8234 }
8235 free_pending_move(sctx, pm);
8236 }
8237
8238 WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->waiting_dir_moves));
8239 while (sctx && !RB_EMPTY_ROOT(&sctx->waiting_dir_moves)) {
8240 struct rb_node *n;
8241 struct waiting_dir_move *dm;
8242
8243 n = rb_first(&sctx->waiting_dir_moves);
8244 dm = rb_entry(n, struct waiting_dir_move, node);
8245 rb_erase(&dm->node, &sctx->waiting_dir_moves);
8246 kfree(dm);
8247 }
8248
8249 WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->orphan_dirs));
8250 while (sctx && !RB_EMPTY_ROOT(&sctx->orphan_dirs)) {
8251 struct rb_node *n;
8252 struct orphan_dir_info *odi;
8253
8254 n = rb_first(&sctx->orphan_dirs);
8255 odi = rb_entry(n, struct orphan_dir_info, node);
8256 free_orphan_dir_info(sctx, odi);
8257 }
8258
8259 if (sort_clone_roots) {
8260 for (i = 0; sctx && i < sctx->clone_roots_cnt; i++) {
8261 btrfs_root_dec_send_in_progress(
8262 sctx->clone_roots[i].root);
8263 btrfs_put_root(sctx->clone_roots[i].root);
8264 }
8265 } else {
8266 for (i = 0; sctx && i < clone_sources_to_rollback; i++) {
8267 btrfs_root_dec_send_in_progress(
8268 sctx->clone_roots[i].root);
8269 btrfs_put_root(sctx->clone_roots[i].root);
8270 }
8271
8272 btrfs_root_dec_send_in_progress(send_root);
8273 }
8274 if (sctx && !IS_ERR_OR_NULL(sctx->parent_root)) {
8275 btrfs_root_dec_send_in_progress(sctx->parent_root);
8276 btrfs_put_root(sctx->parent_root);
8277 }
8278
8279 kvfree(clone_sources_tmp);
8280
8281 if (sctx) {
8282 if (sctx->send_filp)
8283 fput(sctx->send_filp);
8284
8285 kvfree(sctx->clone_roots);
8286 kfree(sctx->send_buf_pages);
8287 kvfree(sctx->send_buf);
8288 kvfree(sctx->verity_descriptor);
8289
8290 close_current_inode(sctx);
8291
8292 btrfs_lru_cache_clear(&sctx->name_cache);
8293 btrfs_lru_cache_clear(&sctx->backref_cache);
8294 btrfs_lru_cache_clear(&sctx->dir_created_cache);
8295 btrfs_lru_cache_clear(&sctx->dir_utimes_cache);
8296
8297 if (sctx->cur_inode_path.buf != sctx->cur_inode_path.inline_buf)
8298 kfree(sctx->cur_inode_path.buf);
8299
8300 kfree(sctx);
8301 }
8302
8303 return ret;
8304}