| 1 | |
| 2 | |
| 3 | * Copyright (C) 2007 Oracle. All rights reserved.
|
| 4 |
|
| 5 |
|
| 6 | #include <linux/fs.h>
|
| 7 | #include <linux/pagemap.h>
|
| 8 | #include <linux/time.h>
|
| 9 | #include <linux/init.h>
|
| 10 | #include <linux/string.h>
|
| 11 | #include <linux/backing-dev.h>
|
| 12 | #include <linux/falloc.h>
|
| 13 | #include <linux/filelock.h>
|
| 14 | #include <linux/writeback.h>
|
| 15 | #include <linux/compat.h>
|
| 16 | #include <linux/slab.h>
|
| 17 | #include <linux/btrfs.h>
|
| 18 | #include <linux/uio.h>
|
| 19 | #include <linux/iversion.h>
|
| 20 | #include <linux/fsverity.h>
|
| 21 | #include "ctree.h"
|
| 22 | #include "direct-io.h"
|
| 23 | #include "disk-io.h"
|
| 24 | #include "transaction.h"
|
| 25 | #include "btrfs_inode.h"
|
| 26 | #include "tree-log.h"
|
| 27 | #include "locking.h"
|
| 28 | #include "qgroup.h"
|
| 29 | #include "compression.h"
|
| 30 | #include "delalloc-space.h"
|
| 31 | #include "reflink.h"
|
| 32 | #include "subpage.h"
|
| 33 | #include "fs.h"
|
| 34 | #include "accessors.h"
|
| 35 | #include "extent-tree.h"
|
| 36 | #include "file-item.h"
|
| 37 | #include "ioctl.h"
|
| 38 | #include "file.h"
|
| 39 | #include "super.h"
|
| 40 | #include "print-tree.h"
|
| 41 |
|
| 42 | |
| 43 | * Unlock folio after btrfs_file_write() is done with it.
|
| 44 |
|
| 45 | static void btrfs_drop_folio(struct btrfs_fs_info *fs_info, struct folio *folio,
|
| 46 | u64 pos, u64 copied)
|
| 47 | {
|
| 48 | u64 block_start = round_down(pos, fs_info->sectorsize);
|
| 49 | u64 block_len = round_up(pos + copied, fs_info->sectorsize) - block_start;
|
| 50 |
|
| 51 | ASSERT(block_len <= U32_MAX);
|
| 52 | folio_unlock(folio);
|
| 53 | folio_put(folio);
|
| 54 | }
|
| 55 |
|
| 56 | |
| 57 | * After copy_folio_from_iter_atomic(), update the following things for delalloc:
|
| 58 | * - Mark newly dirtied folio as DELALLOC in the io tree.
|
| 59 | * Used to advise which range is to be written back.
|
| 60 | * - Mark modified folio as Uptodate/Dirty
|
| 61 | * - Update inode size for past EOF write
|
| 62 |
|
| 63 | int btrfs_dirty_folio(struct btrfs_inode *inode, struct folio *folio, loff_t pos,
|
| 64 | size_t write_bytes, struct extent_state **cached, bool noreserve)
|
| 65 | {
|
| 66 | struct btrfs_fs_info *fs_info = inode->root->fs_info;
|
| 67 | int ret = 0;
|
| 68 | u64 num_bytes;
|
| 69 | u64 start_pos;
|
| 70 | u64 end_of_last_block;
|
| 71 | const u64 end_pos = pos + write_bytes;
|
| 72 | loff_t isize = i_size_read(&inode->vfs_inode);
|
| 73 | unsigned int extra_bits = 0;
|
| 74 |
|
| 75 | if (write_bytes == 0)
|
| 76 | return 0;
|
| 77 |
|
| 78 | if (noreserve)
|
| 79 | extra_bits |= EXTENT_NORESERVE;
|
| 80 |
|
| 81 | start_pos = round_down(pos, fs_info->sectorsize);
|
| 82 | num_bytes = round_up(end_pos - start_pos, fs_info->sectorsize);
|
| 83 | ASSERT(num_bytes <= U32_MAX);
|
| 84 | ASSERT(folio_pos(folio) <= pos && folio_next_pos(folio) >= end_pos);
|
| 85 |
|
| 86 | end_of_last_block = start_pos + num_bytes - 1;
|
| 87 |
|
| 88 | ret = btrfs_reset_extent_delalloc(inode, start_pos, end_of_last_block,
|
| 89 | extra_bits, cached);
|
| 90 | if (ret)
|
| 91 | return ret;
|
| 92 |
|
| 93 | btrfs_folio_clamp_set_uptodate(fs_info, folio, start_pos, num_bytes);
|
| 94 | btrfs_folio_clamp_set_dirty(fs_info, folio, start_pos, num_bytes);
|
| 95 |
|
| 96 | |
| 97 | * we've only changed i_size in ram, and we haven't updated
|
| 98 | * the disk i_size. There is no need to log the inode
|
| 99 | * at this time.
|
| 100 |
|
| 101 | if (end_pos > isize)
|
| 102 | i_size_write(&inode->vfs_inode, end_pos);
|
| 103 | return 0;
|
| 104 | }
|
| 105 |
|
| 106 | |
| 107 | * this is very complex, but the basic idea is to drop all extents
|
| 108 | * in the range start - end. hint_block is filled in with a block number
|
| 109 | * that would be a good hint to the block allocator for this file.
|
| 110 | *
|
| 111 | * If an extent intersects the range but is not entirely inside the range
|
| 112 | * it is either truncated or split. Anything entirely inside the range
|
| 113 | * is deleted from the tree.
|
| 114 | *
|
| 115 | * Note: the VFS' inode number of bytes is not updated, it's up to the caller
|
| 116 | * to deal with that. We set the field 'bytes_found' of the arguments structure
|
| 117 | * with the number of allocated bytes found in the target range, so that the
|
| 118 | * caller can update the inode's number of bytes in an atomic way when
|
| 119 | * replacing extents in a range to avoid races with stat(2).
|
| 120 |
|
| 121 | int btrfs_drop_extents(struct btrfs_trans_handle *trans,
|
| 122 | struct btrfs_root *root, struct btrfs_inode *inode,
|
| 123 | struct btrfs_drop_extents_args *args)
|
| 124 | {
|
| 125 | struct btrfs_fs_info *fs_info = root->fs_info;
|
| 126 | struct extent_buffer *leaf;
|
| 127 | struct btrfs_file_extent_item *fi;
|
| 128 | struct btrfs_key key;
|
| 129 | struct btrfs_key new_key;
|
| 130 | u64 ino = btrfs_ino(inode);
|
| 131 | u64 search_start = args->start;
|
| 132 | u64 disk_bytenr = 0;
|
| 133 | u64 num_bytes = 0;
|
| 134 | u64 extent_offset = 0;
|
| 135 | u64 extent_end = 0;
|
| 136 | u64 last_end = args->start;
|
| 137 | int del_nr = 0;
|
| 138 | int del_slot = 0;
|
| 139 | int extent_type;
|
| 140 | int recow;
|
| 141 | int ret;
|
| 142 | int modify_tree = -1;
|
| 143 | int update_refs;
|
| 144 | bool found = false;
|
| 145 | struct btrfs_path *path = args->path;
|
| 146 |
|
| 147 | args->bytes_found = 0;
|
| 148 | args->extent_inserted = false;
|
| 149 |
|
| 150 |
|
| 151 | ASSERT(!(args->replace_extent && !args->path));
|
| 152 |
|
| 153 | if (!path) {
|
| 154 | path = btrfs_alloc_path();
|
| 155 | if (!path) {
|
| 156 | ret = -ENOMEM;
|
| 157 | goto out;
|
| 158 | }
|
| 159 | }
|
| 160 |
|
| 161 | if (args->drop_cache)
|
| 162 | btrfs_drop_extent_map_range(inode, args->start, args->end - 1, false);
|
| 163 |
|
| 164 | if (data_race(args->start >= inode->disk_i_size) && !args->replace_extent)
|
| 165 | modify_tree = 0;
|
| 166 |
|
| 167 | update_refs = (btrfs_root_id(root) != BTRFS_TREE_LOG_OBJECTID);
|
| 168 | while (1) {
|
| 169 | recow = 0;
|
| 170 | ret = btrfs_lookup_file_extent(trans, root, path, ino,
|
| 171 | search_start, modify_tree);
|
| 172 | if (ret < 0)
|
| 173 | break;
|
| 174 | if (ret > 0 && path->slots[0] > 0 && search_start == args->start) {
|
| 175 | leaf = path->nodes[0];
|
| 176 | btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1);
|
| 177 | if (key.objectid == ino &&
|
| 178 | key.type == BTRFS_EXTENT_DATA_KEY)
|
| 179 | path->slots[0]--;
|
| 180 | }
|
| 181 | ret = 0;
|
| 182 | next_slot:
|
| 183 | leaf = path->nodes[0];
|
| 184 | if (path->slots[0] >= btrfs_header_nritems(leaf)) {
|
| 185 | if (WARN_ON(del_nr > 0)) {
|
| 186 | btrfs_print_leaf(leaf);
|
| 187 | ret = -EINVAL;
|
| 188 | break;
|
| 189 | }
|
| 190 | ret = btrfs_next_leaf(root, path);
|
| 191 | if (ret < 0)
|
| 192 | break;
|
| 193 | if (ret > 0) {
|
| 194 | ret = 0;
|
| 195 | break;
|
| 196 | }
|
| 197 | leaf = path->nodes[0];
|
| 198 | recow = 1;
|
| 199 | }
|
| 200 |
|
| 201 | btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
|
| 202 |
|
| 203 | if (key.objectid > ino)
|
| 204 | break;
|
| 205 | if (WARN_ON_ONCE(key.objectid < ino) ||
|
| 206 | key.type < BTRFS_EXTENT_DATA_KEY) {
|
| 207 | ASSERT(del_nr == 0);
|
| 208 | path->slots[0]++;
|
| 209 | goto next_slot;
|
| 210 | }
|
| 211 | if (key.type > BTRFS_EXTENT_DATA_KEY || key.offset >= args->end)
|
| 212 | break;
|
| 213 |
|
| 214 | fi = btrfs_item_ptr(leaf, path->slots[0],
|
| 215 | struct btrfs_file_extent_item);
|
| 216 | extent_type = btrfs_file_extent_type(leaf, fi);
|
| 217 |
|
| 218 | if (extent_type == BTRFS_FILE_EXTENT_REG ||
|
| 219 | extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
|
| 220 | disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
|
| 221 | num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
|
| 222 | extent_offset = btrfs_file_extent_offset(leaf, fi);
|
| 223 | extent_end = key.offset +
|
| 224 | btrfs_file_extent_num_bytes(leaf, fi);
|
| 225 | } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
|
| 226 | extent_end = key.offset +
|
| 227 | btrfs_file_extent_ram_bytes(leaf, fi);
|
| 228 | } else {
|
| 229 |
|
| 230 | BUG();
|
| 231 | }
|
| 232 |
|
| 233 | |
| 234 | * Don't skip extent items representing 0 byte lengths. They
|
| 235 | * used to be created (bug) if while punching holes we hit
|
| 236 | * -ENOSPC condition. So if we find one here, just ensure we
|
| 237 | * delete it, otherwise we would insert a new file extent item
|
| 238 | * with the same key (offset) as that 0 bytes length file
|
| 239 | * extent item in the call to setup_items_for_insert() later
|
| 240 | * in this function.
|
| 241 |
|
| 242 | if (extent_end == key.offset && extent_end >= search_start) {
|
| 243 | last_end = extent_end;
|
| 244 | goto delete_extent_item;
|
| 245 | }
|
| 246 |
|
| 247 | if (extent_end <= search_start) {
|
| 248 | path->slots[0]++;
|
| 249 | goto next_slot;
|
| 250 | }
|
| 251 |
|
| 252 | found = true;
|
| 253 | search_start = max(key.offset, args->start);
|
| 254 | if (recow || !modify_tree) {
|
| 255 | modify_tree = -1;
|
| 256 | btrfs_release_path(path);
|
| 257 | continue;
|
| 258 | }
|
| 259 |
|
| 260 | |
| 261 | * | - range to drop - |
|
| 262 | * | -------- extent -------- |
|
| 263 |
|
| 264 | if (args->start > key.offset && args->end < extent_end) {
|
| 265 | if (WARN_ON(del_nr > 0)) {
|
| 266 | btrfs_print_leaf(leaf);
|
| 267 | ret = -EINVAL;
|
| 268 | break;
|
| 269 | }
|
| 270 | if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
|
| 271 | ret = -EOPNOTSUPP;
|
| 272 | break;
|
| 273 | }
|
| 274 |
|
| 275 | memcpy(&new_key, &key, sizeof(new_key));
|
| 276 | new_key.offset = args->start;
|
| 277 | ret = btrfs_duplicate_item(trans, root, path,
|
| 278 | &new_key);
|
| 279 | if (ret == -EAGAIN) {
|
| 280 | btrfs_release_path(path);
|
| 281 | continue;
|
| 282 | }
|
| 283 | if (ret < 0)
|
| 284 | break;
|
| 285 |
|
| 286 | leaf = path->nodes[0];
|
| 287 | fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
|
| 288 | struct btrfs_file_extent_item);
|
| 289 | btrfs_set_file_extent_num_bytes(leaf, fi,
|
| 290 | args->start - key.offset);
|
| 291 |
|
| 292 | fi = btrfs_item_ptr(leaf, path->slots[0],
|
| 293 | struct btrfs_file_extent_item);
|
| 294 |
|
| 295 | extent_offset += args->start - key.offset;
|
| 296 | btrfs_set_file_extent_offset(leaf, fi, extent_offset);
|
| 297 | btrfs_set_file_extent_num_bytes(leaf, fi,
|
| 298 | extent_end - args->start);
|
| 299 |
|
| 300 | if (update_refs && disk_bytenr > 0) {
|
| 301 | struct btrfs_ref ref = {
|
| 302 | .action = BTRFS_ADD_DELAYED_REF,
|
| 303 | .bytenr = disk_bytenr,
|
| 304 | .num_bytes = num_bytes,
|
| 305 | .parent = 0,
|
| 306 | .owning_root = btrfs_root_id(root),
|
| 307 | .ref_root = btrfs_root_id(root),
|
| 308 | };
|
| 309 | btrfs_init_data_ref(&ref, new_key.objectid,
|
| 310 | args->start - extent_offset,
|
| 311 | 0, false);
|
| 312 | ret = btrfs_inc_extent_ref(trans, &ref);
|
| 313 | if (unlikely(ret)) {
|
| 314 | btrfs_abort_transaction(trans, ret);
|
| 315 | break;
|
| 316 | }
|
| 317 | }
|
| 318 | key.offset = args->start;
|
| 319 | }
|
| 320 | |
| 321 | * From here on out we will have actually dropped something, so
|
| 322 | * last_end can be updated.
|
| 323 |
|
| 324 | last_end = extent_end;
|
| 325 |
|
| 326 | |
| 327 | * | ---- range to drop ----- |
|
| 328 | * | -------- extent -------- |
|
| 329 |
|
| 330 | if (args->start <= key.offset && args->end < extent_end) {
|
| 331 | if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
|
| 332 | ret = -EOPNOTSUPP;
|
| 333 | break;
|
| 334 | }
|
| 335 |
|
| 336 | memcpy(&new_key, &key, sizeof(new_key));
|
| 337 | new_key.offset = args->end;
|
| 338 | btrfs_set_item_key_safe(trans, path, &new_key);
|
| 339 |
|
| 340 | extent_offset += args->end - key.offset;
|
| 341 | btrfs_set_file_extent_offset(leaf, fi, extent_offset);
|
| 342 | btrfs_set_file_extent_num_bytes(leaf, fi,
|
| 343 | extent_end - args->end);
|
| 344 | if (update_refs && disk_bytenr > 0)
|
| 345 | args->bytes_found += args->end - key.offset;
|
| 346 | break;
|
| 347 | }
|
| 348 |
|
| 349 | search_start = extent_end;
|
| 350 | |
| 351 | * | ---- range to drop ----- |
|
| 352 | * | -------- extent -------- |
|
| 353 |
|
| 354 | if (args->start > key.offset && args->end >= extent_end) {
|
| 355 | if (WARN_ON(del_nr > 0)) {
|
| 356 | btrfs_print_leaf(leaf);
|
| 357 | ret = -EINVAL;
|
| 358 | break;
|
| 359 | }
|
| 360 | if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
|
| 361 | ret = -EOPNOTSUPP;
|
| 362 | break;
|
| 363 | }
|
| 364 |
|
| 365 | btrfs_set_file_extent_num_bytes(leaf, fi,
|
| 366 | args->start - key.offset);
|
| 367 | if (update_refs && disk_bytenr > 0)
|
| 368 | args->bytes_found += extent_end - args->start;
|
| 369 | if (args->end == extent_end)
|
| 370 | break;
|
| 371 |
|
| 372 | path->slots[0]++;
|
| 373 | goto next_slot;
|
| 374 | }
|
| 375 |
|
| 376 | |
| 377 | * | ---- range to drop ----- |
|
| 378 | * | ------ extent ------ |
|
| 379 |
|
| 380 | if (args->start <= key.offset && args->end >= extent_end) {
|
| 381 | delete_extent_item:
|
| 382 | if (del_nr == 0) {
|
| 383 | del_slot = path->slots[0];
|
| 384 | del_nr = 1;
|
| 385 | } else {
|
| 386 | if (WARN_ON(del_slot + del_nr != path->slots[0])) {
|
| 387 | btrfs_print_leaf(leaf);
|
| 388 | ret = -EINVAL;
|
| 389 | break;
|
| 390 | }
|
| 391 | del_nr++;
|
| 392 | }
|
| 393 |
|
| 394 | if (update_refs &&
|
| 395 | extent_type == BTRFS_FILE_EXTENT_INLINE) {
|
| 396 | args->bytes_found += extent_end - key.offset;
|
| 397 | extent_end = ALIGN(extent_end,
|
| 398 | fs_info->sectorsize);
|
| 399 | } else if (update_refs && disk_bytenr > 0) {
|
| 400 | struct btrfs_ref ref = {
|
| 401 | .action = BTRFS_DROP_DELAYED_REF,
|
| 402 | .bytenr = disk_bytenr,
|
| 403 | .num_bytes = num_bytes,
|
| 404 | .parent = 0,
|
| 405 | .owning_root = btrfs_root_id(root),
|
| 406 | .ref_root = btrfs_root_id(root),
|
| 407 | };
|
| 408 | btrfs_init_data_ref(&ref, key.objectid,
|
| 409 | key.offset - extent_offset,
|
| 410 | 0, false);
|
| 411 | ret = btrfs_free_extent(trans, &ref);
|
| 412 | if (unlikely(ret)) {
|
| 413 | btrfs_abort_transaction(trans, ret);
|
| 414 | break;
|
| 415 | }
|
| 416 | args->bytes_found += extent_end - key.offset;
|
| 417 | }
|
| 418 |
|
| 419 | if (args->end == extent_end)
|
| 420 | break;
|
| 421 |
|
| 422 | if (path->slots[0] + 1 < btrfs_header_nritems(leaf)) {
|
| 423 | path->slots[0]++;
|
| 424 | goto next_slot;
|
| 425 | }
|
| 426 |
|
| 427 | ret = btrfs_del_items(trans, root, path, del_slot,
|
| 428 | del_nr);
|
| 429 | if (unlikely(ret)) {
|
| 430 | btrfs_abort_transaction(trans, ret);
|
| 431 | break;
|
| 432 | }
|
| 433 |
|
| 434 | del_nr = 0;
|
| 435 | del_slot = 0;
|
| 436 |
|
| 437 | btrfs_release_path(path);
|
| 438 | continue;
|
| 439 | }
|
| 440 |
|
| 441 | BUG();
|
| 442 | }
|
| 443 |
|
| 444 | if (!ret && del_nr > 0) {
|
| 445 | |
| 446 | * Set path->slots[0] to first slot, so that after the delete
|
| 447 | * if items are move off from our leaf to its immediate left or
|
| 448 | * right neighbor leafs, we end up with a correct and adjusted
|
| 449 | * path->slots[0] for our insertion (if args->replace_extent).
|
| 450 |
|
| 451 | path->slots[0] = del_slot;
|
| 452 | ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
|
| 453 | if (ret)
|
| 454 | btrfs_abort_transaction(trans, ret);
|
| 455 | }
|
| 456 |
|
| 457 | leaf = path->nodes[0];
|
| 458 | |
| 459 | * If btrfs_del_items() was called, it might have deleted a leaf, in
|
| 460 | * which case it unlocked our path, so check path->locks[0] matches a
|
| 461 | * write lock.
|
| 462 |
|
| 463 | if (!ret && args->replace_extent &&
|
| 464 | path->locks[0] == BTRFS_WRITE_LOCK &&
|
| 465 | btrfs_leaf_free_space(leaf) >=
|
| 466 | sizeof(struct btrfs_item) + args->extent_item_size) {
|
| 467 |
|
| 468 | key.objectid = ino;
|
| 469 | key.type = BTRFS_EXTENT_DATA_KEY;
|
| 470 | key.offset = args->start;
|
| 471 | if (!del_nr && path->slots[0] < btrfs_header_nritems(leaf)) {
|
| 472 | struct btrfs_key slot_key;
|
| 473 |
|
| 474 | btrfs_item_key_to_cpu(leaf, &slot_key, path->slots[0]);
|
| 475 | if (btrfs_comp_cpu_keys(&key, &slot_key) > 0)
|
| 476 | path->slots[0]++;
|
| 477 | }
|
| 478 | btrfs_setup_item_for_insert(trans, root, path, &key,
|
| 479 | args->extent_item_size);
|
| 480 | args->extent_inserted = true;
|
| 481 | }
|
| 482 |
|
| 483 | if (!args->path)
|
| 484 | btrfs_free_path(path);
|
| 485 | else if (!args->extent_inserted)
|
| 486 | btrfs_release_path(path);
|
| 487 | out:
|
| 488 | args->drop_end = found ? min(args->end, last_end) : args->end;
|
| 489 |
|
| 490 | return ret;
|
| 491 | }
|
| 492 |
|
| 493 | static bool extent_mergeable(struct extent_buffer *leaf, int slot, u64 objectid,
|
| 494 | u64 bytenr, u64 orig_offset, u64 *start, u64 *end)
|
| 495 | {
|
| 496 | struct btrfs_file_extent_item *fi;
|
| 497 | struct btrfs_key key;
|
| 498 | u64 extent_end;
|
| 499 |
|
| 500 | if (slot < 0 || slot >= btrfs_header_nritems(leaf))
|
| 501 | return false;
|
| 502 |
|
| 503 | btrfs_item_key_to_cpu(leaf, &key, slot);
|
| 504 | if (key.objectid != objectid || key.type != BTRFS_EXTENT_DATA_KEY)
|
| 505 | return false;
|
| 506 |
|
| 507 | fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
|
| 508 | if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG ||
|
| 509 | btrfs_file_extent_disk_bytenr(leaf, fi) != bytenr ||
|
| 510 | btrfs_file_extent_offset(leaf, fi) != key.offset - orig_offset ||
|
| 511 | btrfs_file_extent_compression(leaf, fi) ||
|
| 512 | btrfs_file_extent_encryption(leaf, fi) ||
|
| 513 | btrfs_file_extent_other_encoding(leaf, fi))
|
| 514 | return false;
|
| 515 |
|
| 516 | extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
|
| 517 | if ((*start && *start != key.offset) || (*end && *end != extent_end))
|
| 518 | return false;
|
| 519 |
|
| 520 | *start = key.offset;
|
| 521 | *end = extent_end;
|
| 522 | return true;
|
| 523 | }
|
| 524 |
|
| 525 | |
| 526 | * Mark extent in the range start - end as written.
|
| 527 | *
|
| 528 | * This changes extent type from 'pre-allocated' to 'regular'. If only
|
| 529 | * part of extent is marked as written, the extent will be split into
|
| 530 | * two or three.
|
| 531 |
|
| 532 | int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
|
| 533 | struct btrfs_inode *inode, u64 start, u64 end)
|
| 534 | {
|
| 535 | struct btrfs_root *root = inode->root;
|
| 536 | struct extent_buffer *leaf;
|
| 537 | BTRFS_PATH_AUTO_FREE(path);
|
| 538 | struct btrfs_file_extent_item *fi;
|
| 539 | struct btrfs_ref ref = { 0 };
|
| 540 | struct btrfs_key key;
|
| 541 | struct btrfs_key new_key;
|
| 542 | u64 bytenr;
|
| 543 | u64 num_bytes;
|
| 544 | u64 extent_end;
|
| 545 | u64 orig_offset;
|
| 546 | u64 other_start;
|
| 547 | u64 other_end;
|
| 548 | u64 split;
|
| 549 | int del_nr = 0;
|
| 550 | int del_slot = 0;
|
| 551 | int recow;
|
| 552 | int ret;
|
| 553 | u64 ino = btrfs_ino(inode);
|
| 554 |
|
| 555 | path = btrfs_alloc_path();
|
| 556 | if (!path)
|
| 557 | return -ENOMEM;
|
| 558 | again:
|
| 559 | recow = 0;
|
| 560 | split = start;
|
| 561 | key.objectid = ino;
|
| 562 | key.type = BTRFS_EXTENT_DATA_KEY;
|
| 563 | key.offset = split;
|
| 564 |
|
| 565 | ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
|
| 566 | if (ret < 0)
|
| 567 | return ret;
|
| 568 | if (ret > 0 && path->slots[0] > 0)
|
| 569 | path->slots[0]--;
|
| 570 |
|
| 571 | leaf = path->nodes[0];
|
| 572 | btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
|
| 573 | if (unlikely(key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY)) {
|
| 574 | ret = -EINVAL;
|
| 575 | btrfs_abort_transaction(trans, ret);
|
| 576 | return ret;
|
| 577 | }
|
| 578 | fi = btrfs_item_ptr(leaf, path->slots[0],
|
| 579 | struct btrfs_file_extent_item);
|
| 580 | if (unlikely(btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_PREALLOC)) {
|
| 581 | ret = -EINVAL;
|
| 582 | btrfs_abort_transaction(trans, ret);
|
| 583 | return ret;
|
| 584 | }
|
| 585 | extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
|
| 586 | if (unlikely(key.offset > start || extent_end < end)) {
|
| 587 | ret = -EINVAL;
|
| 588 | btrfs_abort_transaction(trans, ret);
|
| 589 | return ret;
|
| 590 | }
|
| 591 |
|
| 592 | bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
|
| 593 | num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
|
| 594 | orig_offset = key.offset - btrfs_file_extent_offset(leaf, fi);
|
| 595 | memcpy(&new_key, &key, sizeof(new_key));
|
| 596 |
|
| 597 | if (start == key.offset && end < extent_end) {
|
| 598 | other_start = 0;
|
| 599 | other_end = start;
|
| 600 | if (extent_mergeable(leaf, path->slots[0] - 1,
|
| 601 | ino, bytenr, orig_offset,
|
| 602 | &other_start, &other_end)) {
|
| 603 | new_key.offset = end;
|
| 604 | btrfs_set_item_key_safe(trans, path, &new_key);
|
| 605 | fi = btrfs_item_ptr(leaf, path->slots[0],
|
| 606 | struct btrfs_file_extent_item);
|
| 607 | btrfs_set_file_extent_generation(leaf, fi,
|
| 608 | trans->transid);
|
| 609 | btrfs_set_file_extent_num_bytes(leaf, fi,
|
| 610 | extent_end - end);
|
| 611 | btrfs_set_file_extent_offset(leaf, fi,
|
| 612 | end - orig_offset);
|
| 613 | fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
|
| 614 | struct btrfs_file_extent_item);
|
| 615 | btrfs_set_file_extent_generation(leaf, fi,
|
| 616 | trans->transid);
|
| 617 | btrfs_set_file_extent_num_bytes(leaf, fi,
|
| 618 | end - other_start);
|
| 619 | goto mark_dirty;
|
| 620 | }
|
| 621 | }
|
| 622 |
|
| 623 | if (start > key.offset && end == extent_end) {
|
| 624 | other_start = end;
|
| 625 | other_end = 0;
|
| 626 | if (extent_mergeable(leaf, path->slots[0] + 1,
|
| 627 | ino, bytenr, orig_offset,
|
| 628 | &other_start, &other_end)) {
|
| 629 | fi = btrfs_item_ptr(leaf, path->slots[0],
|
| 630 | struct btrfs_file_extent_item);
|
| 631 | btrfs_set_file_extent_num_bytes(leaf, fi,
|
| 632 | start - key.offset);
|
| 633 | btrfs_set_file_extent_generation(leaf, fi,
|
| 634 | trans->transid);
|
| 635 | path->slots[0]++;
|
| 636 | new_key.offset = start;
|
| 637 | btrfs_set_item_key_safe(trans, path, &new_key);
|
| 638 |
|
| 639 | fi = btrfs_item_ptr(leaf, path->slots[0],
|
| 640 | struct btrfs_file_extent_item);
|
| 641 | btrfs_set_file_extent_generation(leaf, fi,
|
| 642 | trans->transid);
|
| 643 | btrfs_set_file_extent_num_bytes(leaf, fi,
|
| 644 | other_end - start);
|
| 645 | btrfs_set_file_extent_offset(leaf, fi,
|
| 646 | start - orig_offset);
|
| 647 | goto mark_dirty;
|
| 648 | }
|
| 649 | }
|
| 650 |
|
| 651 | while (start > key.offset || end < extent_end) {
|
| 652 | if (key.offset == start)
|
| 653 | split = end;
|
| 654 |
|
| 655 | new_key.offset = split;
|
| 656 | ret = btrfs_duplicate_item(trans, root, path, &new_key);
|
| 657 | if (ret == -EAGAIN) {
|
| 658 | btrfs_release_path(path);
|
| 659 | goto again;
|
| 660 | }
|
| 661 | if (unlikely(ret < 0)) {
|
| 662 | btrfs_abort_transaction(trans, ret);
|
| 663 | return ret;
|
| 664 | }
|
| 665 |
|
| 666 | leaf = path->nodes[0];
|
| 667 | fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
|
| 668 | struct btrfs_file_extent_item);
|
| 669 | btrfs_set_file_extent_generation(leaf, fi, trans->transid);
|
| 670 | btrfs_set_file_extent_num_bytes(leaf, fi,
|
| 671 | split - key.offset);
|
| 672 |
|
| 673 | fi = btrfs_item_ptr(leaf, path->slots[0],
|
| 674 | struct btrfs_file_extent_item);
|
| 675 |
|
| 676 | btrfs_set_file_extent_generation(leaf, fi, trans->transid);
|
| 677 | btrfs_set_file_extent_offset(leaf, fi, split - orig_offset);
|
| 678 | btrfs_set_file_extent_num_bytes(leaf, fi,
|
| 679 | extent_end - split);
|
| 680 |
|
| 681 | ref.action = BTRFS_ADD_DELAYED_REF;
|
| 682 | ref.bytenr = bytenr;
|
| 683 | ref.num_bytes = num_bytes;
|
| 684 | ref.parent = 0;
|
| 685 | ref.owning_root = btrfs_root_id(root);
|
| 686 | ref.ref_root = btrfs_root_id(root);
|
| 687 | btrfs_init_data_ref(&ref, ino, orig_offset, 0, false);
|
| 688 | ret = btrfs_inc_extent_ref(trans, &ref);
|
| 689 | if (unlikely(ret)) {
|
| 690 | btrfs_abort_transaction(trans, ret);
|
| 691 | return ret;
|
| 692 | }
|
| 693 |
|
| 694 | if (split == start) {
|
| 695 | key.offset = start;
|
| 696 | } else {
|
| 697 | if (unlikely(start != key.offset)) {
|
| 698 | ret = -EINVAL;
|
| 699 | btrfs_abort_transaction(trans, ret);
|
| 700 | return ret;
|
| 701 | }
|
| 702 | path->slots[0]--;
|
| 703 | extent_end = end;
|
| 704 | }
|
| 705 | recow = 1;
|
| 706 | }
|
| 707 |
|
| 708 | other_start = end;
|
| 709 | other_end = 0;
|
| 710 |
|
| 711 | ref.action = BTRFS_DROP_DELAYED_REF;
|
| 712 | ref.bytenr = bytenr;
|
| 713 | ref.num_bytes = num_bytes;
|
| 714 | ref.parent = 0;
|
| 715 | ref.owning_root = btrfs_root_id(root);
|
| 716 | ref.ref_root = btrfs_root_id(root);
|
| 717 | btrfs_init_data_ref(&ref, ino, orig_offset, 0, false);
|
| 718 | if (extent_mergeable(leaf, path->slots[0] + 1,
|
| 719 | ino, bytenr, orig_offset,
|
| 720 | &other_start, &other_end)) {
|
| 721 | if (recow) {
|
| 722 | btrfs_release_path(path);
|
| 723 | goto again;
|
| 724 | }
|
| 725 | extent_end = other_end;
|
| 726 | del_slot = path->slots[0] + 1;
|
| 727 | del_nr++;
|
| 728 | ret = btrfs_free_extent(trans, &ref);
|
| 729 | if (unlikely(ret)) {
|
| 730 | btrfs_abort_transaction(trans, ret);
|
| 731 | return ret;
|
| 732 | }
|
| 733 | }
|
| 734 | other_start = 0;
|
| 735 | other_end = start;
|
| 736 | if (extent_mergeable(leaf, path->slots[0] - 1,
|
| 737 | ino, bytenr, orig_offset,
|
| 738 | &other_start, &other_end)) {
|
| 739 | if (recow) {
|
| 740 | btrfs_release_path(path);
|
| 741 | goto again;
|
| 742 | }
|
| 743 | key.offset = other_start;
|
| 744 | del_slot = path->slots[0];
|
| 745 | del_nr++;
|
| 746 | ret = btrfs_free_extent(trans, &ref);
|
| 747 | if (unlikely(ret)) {
|
| 748 | btrfs_abort_transaction(trans, ret);
|
| 749 | return ret;
|
| 750 | }
|
| 751 | }
|
| 752 | if (del_nr == 0) {
|
| 753 | fi = btrfs_item_ptr(leaf, path->slots[0],
|
| 754 | struct btrfs_file_extent_item);
|
| 755 | btrfs_set_file_extent_type(leaf, fi,
|
| 756 | BTRFS_FILE_EXTENT_REG);
|
| 757 | btrfs_set_file_extent_generation(leaf, fi, trans->transid);
|
| 758 | } else {
|
| 759 | fi = btrfs_item_ptr(leaf, del_slot - 1,
|
| 760 | struct btrfs_file_extent_item);
|
| 761 | btrfs_set_file_extent_type(leaf, fi,
|
| 762 | BTRFS_FILE_EXTENT_REG);
|
| 763 | btrfs_set_file_extent_generation(leaf, fi, trans->transid);
|
| 764 | btrfs_set_file_extent_num_bytes(leaf, fi,
|
| 765 | extent_end - key.offset);
|
| 766 |
|
| 767 | ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
|
| 768 | if (unlikely(ret < 0)) {
|
| 769 | btrfs_abort_transaction(trans, ret);
|
| 770 | return ret;
|
| 771 | }
|
| 772 | }
|
| 773 |
|
| 774 | mark_dirty:
|
| 775 | ret = btrfs_inode_set_file_extent_range(inode, start, end - start);
|
| 776 | if (ret)
|
| 777 | btrfs_abort_transaction(trans, ret);
|
| 778 |
|
| 779 | return ret;
|
| 780 | }
|
| 781 |
|
| 782 | |
| 783 | * On error return an unlocked folio and the error value
|
| 784 | * On success return a locked folio and 0
|
| 785 |
|
| 786 | static int prepare_uptodate_folio(struct inode *inode, struct folio *folio, u64 pos,
|
| 787 | u64 len)
|
| 788 | {
|
| 789 | u64 clamp_start = max_t(u64, pos, folio_pos(folio));
|
| 790 | u64 clamp_end = min_t(u64, pos + len, folio_next_pos(folio));
|
| 791 | const u32 blocksize = inode_to_fs_info(inode)->sectorsize;
|
| 792 | int ret = 0;
|
| 793 |
|
| 794 | if (folio_test_uptodate(folio))
|
| 795 | return 0;
|
| 796 |
|
| 797 | if (IS_ALIGNED(clamp_start, blocksize) &&
|
| 798 | IS_ALIGNED(clamp_end, blocksize))
|
| 799 | return 0;
|
| 800 |
|
| 801 | ret = btrfs_read_folio(NULL, folio);
|
| 802 | if (ret)
|
| 803 | return ret;
|
| 804 | folio_lock(folio);
|
| 805 | if (unlikely(!folio_test_uptodate(folio))) {
|
| 806 | folio_unlock(folio);
|
| 807 | return -EIO;
|
| 808 | }
|
| 809 |
|
| 810 | |
| 811 | * Since btrfs_read_folio() will unlock the folio before it returns,
|
| 812 | * there is a window where btrfs_release_folio() can be called to
|
| 813 | * release the page. Here we check both inode mapping and page
|
| 814 | * private to make sure the page was not released.
|
| 815 | *
|
| 816 | * The private flag check is essential for subpage as we need to store
|
| 817 | * extra bitmap using folio private.
|
| 818 |
|
| 819 | if (folio->mapping != inode->i_mapping || !folio_test_private(folio)) {
|
| 820 | folio_unlock(folio);
|
| 821 | return -EAGAIN;
|
| 822 | }
|
| 823 | return 0;
|
| 824 | }
|
| 825 |
|
| 826 | static gfp_t get_prepare_gfp_flags(struct inode *inode, bool nowait)
|
| 827 | {
|
| 828 | gfp_t gfp;
|
| 829 |
|
| 830 | gfp = btrfs_alloc_write_mask(inode->i_mapping);
|
| 831 | if (nowait) {
|
| 832 | gfp &= ~__GFP_DIRECT_RECLAIM;
|
| 833 | gfp |= GFP_NOWAIT;
|
| 834 | }
|
| 835 |
|
| 836 | return gfp;
|
| 837 | }
|
| 838 |
|
| 839 | |
| 840 | * Get folio into the page cache and lock it.
|
| 841 |
|
| 842 | static noinline int prepare_one_folio(struct inode *inode, struct folio **folio_ret,
|
| 843 | loff_t pos, size_t write_bytes,
|
| 844 | bool nowait)
|
| 845 | {
|
| 846 | const pgoff_t index = pos >> PAGE_SHIFT;
|
| 847 | gfp_t mask = get_prepare_gfp_flags(inode, nowait);
|
| 848 | fgf_t fgp_flags = (nowait ? FGP_WRITEBEGIN | FGP_NOWAIT : FGP_WRITEBEGIN) |
|
| 849 | fgf_set_order(write_bytes);
|
| 850 | struct folio *folio;
|
| 851 | int ret;
|
| 852 |
|
| 853 | again:
|
| 854 | folio = __filemap_get_folio(inode->i_mapping, index, fgp_flags, mask);
|
| 855 | if (IS_ERR(folio))
|
| 856 | return PTR_ERR(folio);
|
| 857 |
|
| 858 | ret = set_folio_extent_mapped(folio);
|
| 859 | if (ret < 0) {
|
| 860 | folio_unlock(folio);
|
| 861 | folio_put(folio);
|
| 862 | return ret;
|
| 863 | }
|
| 864 | ret = prepare_uptodate_folio(inode, folio, pos, write_bytes);
|
| 865 | if (ret) {
|
| 866 |
|
| 867 | folio_put(folio);
|
| 868 | if (!nowait && ret == -EAGAIN)
|
| 869 | goto again;
|
| 870 | return ret;
|
| 871 | }
|
| 872 | *folio_ret = folio;
|
| 873 | return 0;
|
| 874 | }
|
| 875 |
|
| 876 | |
| 877 | * Locks the extent and properly waits for data=ordered extents to finish
|
| 878 | * before allowing the folios to be modified.
|
| 879 | *
|
| 880 | * Return:
|
| 881 | * 0 - the extent is locked
|
| 882 | * -EAGAIN - need to prepare the folios again
|
| 883 |
|
| 884 | static noinline int
|
| 885 | lock_and_cleanup_extent(struct btrfs_inode *inode, struct folio *folio,
|
| 886 | loff_t pos, size_t write_bytes,
|
| 887 | u64 *lockstart, u64 *lockend, bool nowait,
|
| 888 | struct extent_state **cached_state)
|
| 889 | {
|
| 890 | struct btrfs_fs_info *fs_info = inode->root->fs_info;
|
| 891 | struct btrfs_ordered_extent *ordered;
|
| 892 | u64 start_pos;
|
| 893 | u64 last_pos;
|
| 894 |
|
| 895 | start_pos = round_down(pos, fs_info->sectorsize);
|
| 896 | last_pos = round_up(pos + write_bytes, fs_info->sectorsize) - 1;
|
| 897 |
|
| 898 | if (nowait) {
|
| 899 | if (!btrfs_try_lock_extent(&inode->io_tree, start_pos,
|
| 900 | last_pos, cached_state)) {
|
| 901 | folio_unlock(folio);
|
| 902 | folio_put(folio);
|
| 903 | return -EAGAIN;
|
| 904 | }
|
| 905 | } else {
|
| 906 | btrfs_lock_extent(&inode->io_tree, start_pos, last_pos,
|
| 907 | cached_state);
|
| 908 | }
|
| 909 |
|
| 910 | ordered = btrfs_lookup_ordered_range(inode, start_pos,
|
| 911 | last_pos - start_pos + 1);
|
| 912 | if (ordered &&
|
| 913 | ordered->file_offset + ordered->num_bytes > start_pos &&
|
| 914 | ordered->file_offset <= last_pos) {
|
| 915 | btrfs_unlock_extent(&inode->io_tree, start_pos, last_pos,
|
| 916 | cached_state);
|
| 917 | folio_unlock(folio);
|
| 918 | folio_put(folio);
|
| 919 | btrfs_start_ordered_extent(ordered);
|
| 920 | btrfs_put_ordered_extent(ordered);
|
| 921 | return -EAGAIN;
|
| 922 | }
|
| 923 | if (ordered)
|
| 924 | btrfs_put_ordered_extent(ordered);
|
| 925 |
|
| 926 | *lockstart = start_pos;
|
| 927 | *lockend = last_pos;
|
| 928 |
|
| 929 | |
| 930 | * We should be called after prepare_one_folio() which should have locked
|
| 931 | * all pages in the range.
|
| 932 |
|
| 933 | WARN_ON(!folio_test_locked(folio));
|
| 934 |
|
| 935 | return 0;
|
| 936 | }
|
| 937 |
|
| 938 | |
| 939 | * Check if we can do nocow write into the range [@pos, @pos + @write_bytes)
|
| 940 | *
|
| 941 | * @pos: File offset.
|
| 942 | * @write_bytes: The length to write, will be updated to the nocow writeable
|
| 943 | * range.
|
| 944 | * @nowait: Indicate if we can block or not (non-blocking IO context).
|
| 945 | *
|
| 946 | * This function will flush ordered extents in the range to ensure proper
|
| 947 | * nocow checks.
|
| 948 | *
|
| 949 | * Return:
|
| 950 | * > 0 If we can nocow, and updates @write_bytes.
|
| 951 | * 0 If we can't do a nocow write.
|
| 952 | * -EAGAIN If we can't do a nocow write because snapshotting of the inode's
|
| 953 | * root is in progress or because we are in a non-blocking IO
|
| 954 | * context and need to block (@nowait is true).
|
| 955 | * < 0 If an error happened.
|
| 956 | *
|
| 957 | * NOTE: Callers need to call btrfs_check_nocow_unlock() if we return > 0.
|
| 958 |
|
| 959 | int btrfs_check_nocow_lock(struct btrfs_inode *inode, loff_t pos,
|
| 960 | size_t *write_bytes, bool nowait)
|
| 961 | {
|
| 962 | struct btrfs_fs_info *fs_info = inode->root->fs_info;
|
| 963 | struct btrfs_root *root = inode->root;
|
| 964 | struct extent_state *cached_state = NULL;
|
| 965 | u64 lockstart, lockend;
|
| 966 | u64 cur_offset;
|
| 967 | int ret = 0;
|
| 968 |
|
| 969 | if (!(inode->flags & (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC)))
|
| 970 | return 0;
|
| 971 |
|
| 972 | if (!btrfs_drew_try_write_lock(&root->snapshot_lock))
|
| 973 | return -EAGAIN;
|
| 974 |
|
| 975 | lockstart = round_down(pos, fs_info->sectorsize);
|
| 976 | lockend = round_up(pos + *write_bytes,
|
| 977 | fs_info->sectorsize) - 1;
|
| 978 |
|
| 979 | if (nowait) {
|
| 980 | if (!btrfs_try_lock_ordered_range(inode, lockstart, lockend,
|
| 981 | &cached_state)) {
|
| 982 | btrfs_drew_write_unlock(&root->snapshot_lock);
|
| 983 | return -EAGAIN;
|
| 984 | }
|
| 985 | } else {
|
| 986 | btrfs_lock_and_flush_ordered_range(inode, lockstart, lockend,
|
| 987 | &cached_state);
|
| 988 | }
|
| 989 |
|
| 990 | cur_offset = lockstart;
|
| 991 | while (cur_offset < lockend) {
|
| 992 | u64 num_bytes = lockend - cur_offset + 1;
|
| 993 |
|
| 994 | ret = can_nocow_extent(inode, cur_offset, &num_bytes, NULL, nowait);
|
| 995 | if (ret <= 0) {
|
| 996 | |
| 997 | * If cur_offset == lockstart it means we haven't found
|
| 998 | * any extent against which we can NOCOW, so unlock the
|
| 999 | * snapshot lock.
|
| 1000 |
|
| 1001 | if (cur_offset == lockstart)
|
| 1002 | btrfs_drew_write_unlock(&root->snapshot_lock);
|
| 1003 | break;
|
| 1004 | }
|
| 1005 | cur_offset += num_bytes;
|
| 1006 | }
|
| 1007 |
|
| 1008 | btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state);
|
| 1009 |
|
| 1010 | |
| 1011 | * cur_offset > lockstart means there's at least a partial range we can
|
| 1012 | * NOCOW, and that range can cover one or more extents.
|
| 1013 |
|
| 1014 | if (cur_offset > lockstart) {
|
| 1015 | *write_bytes = min_t(size_t, *write_bytes, cur_offset - pos);
|
| 1016 | return 1;
|
| 1017 | }
|
| 1018 |
|
| 1019 | return ret;
|
| 1020 | }
|
| 1021 |
|
| 1022 | void btrfs_check_nocow_unlock(struct btrfs_inode *inode)
|
| 1023 | {
|
| 1024 | btrfs_drew_write_unlock(&inode->root->snapshot_lock);
|
| 1025 | }
|
| 1026 |
|
| 1027 | int btrfs_write_check(struct kiocb *iocb, size_t count)
|
| 1028 | {
|
| 1029 | struct file *file = iocb->ki_filp;
|
| 1030 | struct inode *inode = file_inode(file);
|
| 1031 | struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
|
| 1032 | loff_t pos = iocb->ki_pos;
|
| 1033 | int ret;
|
| 1034 | loff_t oldsize;
|
| 1035 |
|
| 1036 | |
| 1037 | * Quickly bail out on NOWAIT writes if we don't have the nodatacow or
|
| 1038 | * prealloc flags, as without those flags we always have to COW. We will
|
| 1039 | * later check if we can really COW into the target range (using
|
| 1040 | * can_nocow_extent() at btrfs_get_blocks_direct_write()).
|
| 1041 |
|
| 1042 | if ((iocb->ki_flags & IOCB_NOWAIT) &&
|
| 1043 | !(BTRFS_I(inode)->flags & (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC)))
|
| 1044 | return -EAGAIN;
|
| 1045 |
|
| 1046 | ret = file_remove_privs(file);
|
| 1047 | if (ret)
|
| 1048 | return ret;
|
| 1049 |
|
| 1050 | |
| 1051 | * We reserve space for updating the inode when we reserve space for the
|
| 1052 | * extent we are going to write, so we will enospc out there. We don't
|
| 1053 | * need to start yet another transaction to update the inode as we will
|
| 1054 | * update the inode when we finish writing whatever data we write.
|
| 1055 |
|
| 1056 | if (!IS_NOCMTIME(inode)) {
|
| 1057 | inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode));
|
| 1058 | inode_inc_iversion(inode);
|
| 1059 | }
|
| 1060 |
|
| 1061 | oldsize = i_size_read(inode);
|
| 1062 | if (pos > oldsize) {
|
| 1063 |
|
| 1064 | loff_t end_pos = round_up(pos + count, fs_info->sectorsize);
|
| 1065 |
|
| 1066 | ret = btrfs_cont_expand(BTRFS_I(inode), oldsize, end_pos);
|
| 1067 | if (ret)
|
| 1068 | return ret;
|
| 1069 | }
|
| 1070 |
|
| 1071 | return 0;
|
| 1072 | }
|
| 1073 |
|
| 1074 | static void release_space(struct btrfs_inode *inode, struct extent_changeset *data_reserved,
|
| 1075 | u64 start, u64 len, bool only_release_metadata)
|
| 1076 | {
|
| 1077 | if (len == 0)
|
| 1078 | return;
|
| 1079 |
|
| 1080 | if (only_release_metadata) {
|
| 1081 | btrfs_check_nocow_unlock(inode);
|
| 1082 | btrfs_delalloc_release_metadata(inode, len, true);
|
| 1083 | } else {
|
| 1084 | const struct btrfs_fs_info *fs_info = inode->root->fs_info;
|
| 1085 |
|
| 1086 | btrfs_delalloc_release_space(inode, data_reserved,
|
| 1087 | round_down(start, fs_info->sectorsize),
|
| 1088 | len, true);
|
| 1089 | }
|
| 1090 | }
|
| 1091 |
|
| 1092 | |
| 1093 | * Reserve data and metadata space for this buffered write range.
|
| 1094 | *
|
| 1095 | * Return >0 for the number of bytes reserved, which is always block aligned.
|
| 1096 | * Return <0 for error.
|
| 1097 |
|
| 1098 | static ssize_t reserve_space(struct btrfs_inode *inode,
|
| 1099 | struct extent_changeset **data_reserved,
|
| 1100 | u64 start, size_t *len, bool nowait,
|
| 1101 | bool *only_release_metadata)
|
| 1102 | {
|
| 1103 | const struct btrfs_fs_info *fs_info = inode->root->fs_info;
|
| 1104 | const unsigned int block_offset = (start & (fs_info->sectorsize - 1));
|
| 1105 | size_t reserve_bytes;
|
| 1106 | int ret;
|
| 1107 |
|
| 1108 | ret = btrfs_check_data_free_space(inode, data_reserved, start, *len, nowait);
|
| 1109 | if (ret < 0) {
|
| 1110 | int can_nocow;
|
| 1111 |
|
| 1112 | if (nowait && (ret == -ENOSPC || ret == -EAGAIN))
|
| 1113 | return -EAGAIN;
|
| 1114 |
|
| 1115 | |
| 1116 | * If we don't have to COW at the offset, reserve metadata only.
|
| 1117 | * write_bytes may get smaller than requested here.
|
| 1118 |
|
| 1119 | can_nocow = btrfs_check_nocow_lock(inode, start, len, nowait);
|
| 1120 | if (can_nocow < 0)
|
| 1121 | ret = can_nocow;
|
| 1122 | if (can_nocow > 0)
|
| 1123 | ret = 0;
|
| 1124 | if (ret)
|
| 1125 | return ret;
|
| 1126 | *only_release_metadata = true;
|
| 1127 | }
|
| 1128 |
|
| 1129 | reserve_bytes = round_up(*len + block_offset, fs_info->sectorsize);
|
| 1130 | WARN_ON(reserve_bytes == 0);
|
| 1131 | ret = btrfs_delalloc_reserve_metadata(inode, reserve_bytes,
|
| 1132 | reserve_bytes, nowait);
|
| 1133 | if (ret) {
|
| 1134 | if (!*only_release_metadata)
|
| 1135 | btrfs_free_reserved_data_space(inode, *data_reserved,
|
| 1136 | start, *len);
|
| 1137 | else
|
| 1138 | btrfs_check_nocow_unlock(inode);
|
| 1139 |
|
| 1140 | if (nowait && ret == -ENOSPC)
|
| 1141 | ret = -EAGAIN;
|
| 1142 | return ret;
|
| 1143 | }
|
| 1144 | return reserve_bytes;
|
| 1145 | }
|
| 1146 |
|
| 1147 |
|
| 1148 | static void shrink_reserved_space(struct btrfs_inode *inode,
|
| 1149 | struct extent_changeset *data_reserved,
|
| 1150 | u64 reserved_start, u64 reserved_len,
|
| 1151 | u64 new_len, bool only_release_metadata)
|
| 1152 | {
|
| 1153 | const u64 diff = reserved_len - new_len;
|
| 1154 |
|
| 1155 | ASSERT(new_len <= reserved_len);
|
| 1156 | btrfs_delalloc_shrink_extents(inode, reserved_len, new_len);
|
| 1157 | if (only_release_metadata)
|
| 1158 | btrfs_delalloc_release_metadata(inode, diff, true);
|
| 1159 | else
|
| 1160 | btrfs_delalloc_release_space(inode, data_reserved,
|
| 1161 | reserved_start + new_len, diff, true);
|
| 1162 | }
|
| 1163 |
|
| 1164 |
|
| 1165 | static size_t calc_write_bytes(const struct btrfs_inode *inode,
|
| 1166 | const struct iov_iter *iter, u64 start)
|
| 1167 | {
|
| 1168 | const size_t max_folio_size = mapping_max_folio_size(inode->vfs_inode.i_mapping);
|
| 1169 |
|
| 1170 | return min(max_folio_size - (start & (max_folio_size - 1)),
|
| 1171 | iov_iter_count(iter));
|
| 1172 | }
|
| 1173 |
|
| 1174 | |
| 1175 | * Do the heavy-lifting work to copy one range into one folio of the page cache.
|
| 1176 | *
|
| 1177 | * Return > 0 in case we copied all bytes or just some of them.
|
| 1178 | * Return 0 if no bytes were copied, in which case the caller should retry.
|
| 1179 | * Return <0 on error.
|
| 1180 |
|
| 1181 | static int copy_one_range(struct btrfs_inode *inode, struct iov_iter *iter,
|
| 1182 | struct extent_changeset **data_reserved, u64 start,
|
| 1183 | bool nowait)
|
| 1184 | {
|
| 1185 | struct btrfs_fs_info *fs_info = inode->root->fs_info;
|
| 1186 | struct extent_state *cached_state = NULL;
|
| 1187 | size_t write_bytes = calc_write_bytes(inode, iter, start);
|
| 1188 | size_t copied;
|
| 1189 | const u64 reserved_start = round_down(start, fs_info->sectorsize);
|
| 1190 | u64 reserved_len;
|
| 1191 | struct folio *folio = NULL;
|
| 1192 | u64 lockstart;
|
| 1193 | u64 lockend;
|
| 1194 | bool only_release_metadata = false;
|
| 1195 | const unsigned int bdp_flags = (nowait ? BDP_ASYNC : 0);
|
| 1196 | int ret;
|
| 1197 |
|
| 1198 | |
| 1199 | * Fault all pages before locking them in prepare_one_folio() to avoid
|
| 1200 | * recursive lock.
|
| 1201 |
|
| 1202 | if (unlikely(fault_in_iov_iter_readable(iter, write_bytes)))
|
| 1203 | return -EFAULT;
|
| 1204 | extent_changeset_release(*data_reserved);
|
| 1205 | ret = reserve_space(inode, data_reserved, start, &write_bytes, nowait,
|
| 1206 | &only_release_metadata);
|
| 1207 | if (ret < 0)
|
| 1208 | return ret;
|
| 1209 | reserved_len = ret;
|
| 1210 |
|
| 1211 | ASSERT(reserved_start <= start, "reserved_start=%llu start=%llu",
|
| 1212 | reserved_start, start);
|
| 1213 | ASSERT(start + write_bytes <= reserved_start + reserved_len,
|
| 1214 | "start=%llu write_bytes=%zu reserved_start=%llu reserved_len=%llu",
|
| 1215 | start, write_bytes, reserved_start, reserved_len);
|
| 1216 |
|
| 1217 | again:
|
| 1218 | ret = balance_dirty_pages_ratelimited_flags(inode->vfs_inode.i_mapping,
|
| 1219 | bdp_flags);
|
| 1220 | if (ret) {
|
| 1221 | btrfs_delalloc_release_extents(inode, reserved_len);
|
| 1222 | release_space(inode, *data_reserved, reserved_start, reserved_len,
|
| 1223 | only_release_metadata);
|
| 1224 | return ret;
|
| 1225 | }
|
| 1226 |
|
| 1227 | ret = prepare_one_folio(&inode->vfs_inode, &folio, start, write_bytes, false);
|
| 1228 | if (ret) {
|
| 1229 | btrfs_delalloc_release_extents(inode, reserved_len);
|
| 1230 | release_space(inode, *data_reserved, reserved_start, reserved_len,
|
| 1231 | only_release_metadata);
|
| 1232 | return ret;
|
| 1233 | }
|
| 1234 |
|
| 1235 | |
| 1236 | * The reserved range goes beyond the current folio, shrink the reserved
|
| 1237 | * space to the folio boundary.
|
| 1238 |
|
| 1239 | if (reserved_start + reserved_len > folio_next_pos(folio)) {
|
| 1240 | const u64 last_block = folio_next_pos(folio);
|
| 1241 |
|
| 1242 | shrink_reserved_space(inode, *data_reserved, reserved_start,
|
| 1243 | reserved_len, last_block - reserved_start,
|
| 1244 | only_release_metadata);
|
| 1245 | write_bytes = last_block - start;
|
| 1246 | reserved_len = last_block - reserved_start;
|
| 1247 | }
|
| 1248 |
|
| 1249 | ret = lock_and_cleanup_extent(inode, folio, start, write_bytes,
|
| 1250 | &lockstart, &lockend, nowait, &cached_state);
|
| 1251 | if (ret < 0) {
|
| 1252 | if (!nowait)
|
| 1253 | goto again;
|
| 1254 |
|
| 1255 | btrfs_delalloc_release_extents(inode, reserved_len);
|
| 1256 | release_space(inode, *data_reserved, reserved_start, reserved_len,
|
| 1257 | only_release_metadata);
|
| 1258 | return ret;
|
| 1259 | }
|
| 1260 |
|
| 1261 | copied = copy_folio_from_iter_atomic(folio, offset_in_folio(folio, start),
|
| 1262 | write_bytes, iter);
|
| 1263 | flush_dcache_folio(folio);
|
| 1264 |
|
| 1265 | if (unlikely(copied < write_bytes)) {
|
| 1266 | u64 last_block;
|
| 1267 |
|
| 1268 | |
| 1269 | * The original write range doesn't need an uptodate folio as
|
| 1270 | * the range is block aligned. But now a short copy happened.
|
| 1271 | * We cannot handle it without an uptodate folio.
|
| 1272 | *
|
| 1273 | * So just revert the range and we will retry.
|
| 1274 |
|
| 1275 | if (!folio_test_uptodate(folio)) {
|
| 1276 | iov_iter_revert(iter, copied);
|
| 1277 | copied = 0;
|
| 1278 | }
|
| 1279 |
|
| 1280 |
|
| 1281 | if (copied == 0) {
|
| 1282 | btrfs_unlock_extent(&inode->io_tree, lockstart, lockend,
|
| 1283 | &cached_state);
|
| 1284 | btrfs_delalloc_release_extents(inode, reserved_len);
|
| 1285 | release_space(inode, *data_reserved, reserved_start, reserved_len,
|
| 1286 | only_release_metadata);
|
| 1287 | btrfs_drop_folio(fs_info, folio, start, copied);
|
| 1288 | return 0;
|
| 1289 | }
|
| 1290 |
|
| 1291 |
|
| 1292 | last_block = round_up(start + copied, fs_info->sectorsize);
|
| 1293 |
|
| 1294 | shrink_reserved_space(inode, *data_reserved, reserved_start,
|
| 1295 | reserved_len, last_block - reserved_start,
|
| 1296 | only_release_metadata);
|
| 1297 | reserved_len = last_block - reserved_start;
|
| 1298 | }
|
| 1299 |
|
| 1300 | ret = btrfs_dirty_folio(inode, folio, start, copied, &cached_state,
|
| 1301 | only_release_metadata);
|
| 1302 | btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state);
|
| 1303 |
|
| 1304 | btrfs_delalloc_release_extents(inode, reserved_len);
|
| 1305 | if (ret) {
|
| 1306 | btrfs_drop_folio(fs_info, folio, start, copied);
|
| 1307 | release_space(inode, *data_reserved, reserved_start, reserved_len,
|
| 1308 | only_release_metadata);
|
| 1309 | return ret;
|
| 1310 | }
|
| 1311 | if (only_release_metadata)
|
| 1312 | btrfs_check_nocow_unlock(inode);
|
| 1313 |
|
| 1314 | btrfs_drop_folio(fs_info, folio, start, copied);
|
| 1315 | return copied;
|
| 1316 | }
|
| 1317 |
|
| 1318 | ssize_t btrfs_buffered_write(struct kiocb *iocb, struct iov_iter *iter)
|
| 1319 | {
|
| 1320 | struct file *file = iocb->ki_filp;
|
| 1321 | loff_t pos;
|
| 1322 | struct inode *inode = file_inode(file);
|
| 1323 | struct extent_changeset *data_reserved = NULL;
|
| 1324 | size_t num_written = 0;
|
| 1325 | ssize_t ret;
|
| 1326 | loff_t old_isize;
|
| 1327 | unsigned int ilock_flags = 0;
|
| 1328 | const bool nowait = (iocb->ki_flags & IOCB_NOWAIT);
|
| 1329 |
|
| 1330 | if (nowait)
|
| 1331 | ilock_flags |= BTRFS_ILOCK_TRY;
|
| 1332 |
|
| 1333 | ret = btrfs_inode_lock(BTRFS_I(inode), ilock_flags);
|
| 1334 | if (ret < 0)
|
| 1335 | return ret;
|
| 1336 |
|
| 1337 | |
| 1338 | * We can only trust the isize with inode lock held, or it can race with
|
| 1339 | * other buffered writes and cause incorrect call of
|
| 1340 | * pagecache_isize_extended() to overwrite existing data.
|
| 1341 |
|
| 1342 | old_isize = i_size_read(inode);
|
| 1343 |
|
| 1344 | ret = generic_write_checks(iocb, iter);
|
| 1345 | if (ret <= 0)
|
| 1346 | goto out;
|
| 1347 |
|
| 1348 | ret = btrfs_write_check(iocb, ret);
|
| 1349 | if (ret < 0)
|
| 1350 | goto out;
|
| 1351 |
|
| 1352 | pos = iocb->ki_pos;
|
| 1353 | while (iov_iter_count(iter) > 0) {
|
| 1354 | ret = copy_one_range(BTRFS_I(inode), iter, &data_reserved, pos, nowait);
|
| 1355 | if (ret < 0)
|
| 1356 | break;
|
| 1357 | pos += ret;
|
| 1358 | num_written += ret;
|
| 1359 | cond_resched();
|
| 1360 | }
|
| 1361 |
|
| 1362 | extent_changeset_free(data_reserved);
|
| 1363 | if (num_written > 0) {
|
| 1364 | pagecache_isize_extended(inode, old_isize, iocb->ki_pos);
|
| 1365 | iocb->ki_pos += num_written;
|
| 1366 | }
|
| 1367 | out:
|
| 1368 | btrfs_inode_unlock(BTRFS_I(inode), ilock_flags);
|
| 1369 | return num_written ? num_written : ret;
|
| 1370 | }
|
| 1371 |
|
| 1372 | static ssize_t btrfs_encoded_write(struct kiocb *iocb, struct iov_iter *from,
|
| 1373 | const struct btrfs_ioctl_encoded_io_args *encoded)
|
| 1374 | {
|
| 1375 | struct file *file = iocb->ki_filp;
|
| 1376 | struct inode *inode = file_inode(file);
|
| 1377 | loff_t count;
|
| 1378 | ssize_t ret;
|
| 1379 |
|
| 1380 | btrfs_inode_lock(BTRFS_I(inode), 0);
|
| 1381 | count = encoded->len;
|
| 1382 | ret = generic_write_checks_count(iocb, &count);
|
| 1383 | if (ret == 0 && count != encoded->len) {
|
| 1384 | |
| 1385 | * The write got truncated by generic_write_checks_count(). We
|
| 1386 | * can't do a partial encoded write.
|
| 1387 |
|
| 1388 | ret = -EFBIG;
|
| 1389 | }
|
| 1390 | if (ret || encoded->len == 0)
|
| 1391 | goto out;
|
| 1392 |
|
| 1393 | ret = btrfs_write_check(iocb, encoded->len);
|
| 1394 | if (ret < 0)
|
| 1395 | goto out;
|
| 1396 |
|
| 1397 | ret = btrfs_do_encoded_write(iocb, from, encoded);
|
| 1398 | out:
|
| 1399 | btrfs_inode_unlock(BTRFS_I(inode), 0);
|
| 1400 | return ret;
|
| 1401 | }
|
| 1402 |
|
| 1403 | ssize_t btrfs_do_write_iter(struct kiocb *iocb, struct iov_iter *from,
|
| 1404 | const struct btrfs_ioctl_encoded_io_args *encoded)
|
| 1405 | {
|
| 1406 | struct file *file = iocb->ki_filp;
|
| 1407 | struct btrfs_inode *inode = BTRFS_I(file_inode(file));
|
| 1408 | ssize_t num_written, num_sync;
|
| 1409 |
|
| 1410 | if (btrfs_is_shutdown(inode->root->fs_info))
|
| 1411 | return -EIO;
|
| 1412 | |
| 1413 | * If the fs flips readonly due to some impossible error, although we
|
| 1414 | * have opened a file as writable, we have to stop this write operation
|
| 1415 | * to ensure consistency.
|
| 1416 |
|
| 1417 | if (unlikely(BTRFS_FS_ERROR(inode->root->fs_info)))
|
| 1418 | return -EROFS;
|
| 1419 |
|
| 1420 | if (encoded && (iocb->ki_flags & IOCB_NOWAIT))
|
| 1421 | return -EOPNOTSUPP;
|
| 1422 |
|
| 1423 | if (encoded) {
|
| 1424 | num_written = btrfs_encoded_write(iocb, from, encoded);
|
| 1425 | num_sync = encoded->len;
|
| 1426 | } else if (iocb->ki_flags & IOCB_DIRECT) {
|
| 1427 | num_written = btrfs_direct_write(iocb, from);
|
| 1428 | num_sync = num_written;
|
| 1429 | } else {
|
| 1430 | num_written = btrfs_buffered_write(iocb, from);
|
| 1431 | num_sync = num_written;
|
| 1432 | }
|
| 1433 |
|
| 1434 | btrfs_set_inode_last_sub_trans(inode);
|
| 1435 |
|
| 1436 | if (num_sync > 0) {
|
| 1437 | num_sync = generic_write_sync(iocb, num_sync);
|
| 1438 | if (num_sync < 0)
|
| 1439 | num_written = num_sync;
|
| 1440 | }
|
| 1441 |
|
| 1442 | return num_written;
|
| 1443 | }
|
| 1444 |
|
| 1445 | static ssize_t btrfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
|
| 1446 | {
|
| 1447 | return btrfs_do_write_iter(iocb, from, NULL);
|
| 1448 | }
|
| 1449 |
|
| 1450 | int btrfs_release_file(struct inode *inode, struct file *filp)
|
| 1451 | {
|
| 1452 | struct btrfs_file_private *private = filp->private_data;
|
| 1453 |
|
| 1454 | if (private) {
|
| 1455 | kfree(private->filldir_buf);
|
| 1456 | btrfs_free_extent_state(private->llseek_cached_state);
|
| 1457 | kfree(private);
|
| 1458 | filp->private_data = NULL;
|
| 1459 | }
|
| 1460 |
|
| 1461 | |
| 1462 | * Set by setattr when we are about to truncate a file from a non-zero
|
| 1463 | * size to a zero size. This tries to flush down new bytes that may
|
| 1464 | * have been written if the application were using truncate to replace
|
| 1465 | * a file in place.
|
| 1466 |
|
| 1467 | if (test_and_clear_bit(BTRFS_INODE_FLUSH_ON_CLOSE,
|
| 1468 | &BTRFS_I(inode)->runtime_flags))
|
| 1469 | filemap_flush(inode->i_mapping);
|
| 1470 | return 0;
|
| 1471 | }
|
| 1472 |
|
| 1473 | static int start_ordered_ops(struct btrfs_inode *inode, loff_t start, loff_t end)
|
| 1474 | {
|
| 1475 | int ret;
|
| 1476 | struct blk_plug plug;
|
| 1477 |
|
| 1478 | |
| 1479 | * This is only called in fsync, which would do synchronous writes, so
|
| 1480 | * a plug can merge adjacent IOs as much as possible. Esp. in case of
|
| 1481 | * multiple disks using raid profile, a large IO can be split to
|
| 1482 | * several segments of stripe length (currently 64K).
|
| 1483 |
|
| 1484 | blk_start_plug(&plug);
|
| 1485 | ret = btrfs_fdatawrite_range(inode, start, end);
|
| 1486 | blk_finish_plug(&plug);
|
| 1487 |
|
| 1488 | return ret;
|
| 1489 | }
|
| 1490 |
|
| 1491 | static inline bool skip_inode_logging(const struct btrfs_log_ctx *ctx)
|
| 1492 | {
|
| 1493 | struct btrfs_inode *inode = ctx->inode;
|
| 1494 | struct btrfs_fs_info *fs_info = inode->root->fs_info;
|
| 1495 |
|
| 1496 | if (btrfs_inode_in_log(inode, btrfs_get_fs_generation(fs_info)) &&
|
| 1497 | list_empty(&ctx->ordered_extents))
|
| 1498 | return true;
|
| 1499 |
|
| 1500 | |
| 1501 | * If we are doing a fast fsync we can not bail out if the inode's
|
| 1502 | * last_trans is <= then the last committed transaction, because we only
|
| 1503 | * update the last_trans of the inode during ordered extent completion,
|
| 1504 | * and for a fast fsync we don't wait for that, we only wait for the
|
| 1505 | * writeback to complete.
|
| 1506 |
|
| 1507 | if (inode->last_trans <= btrfs_get_last_trans_committed(fs_info) &&
|
| 1508 | (test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags) ||
|
| 1509 | list_empty(&ctx->ordered_extents)))
|
| 1510 | return true;
|
| 1511 |
|
| 1512 | return false;
|
| 1513 | }
|
| 1514 |
|
| 1515 | |
| 1516 | * fsync call for both files and directories. This logs the inode into
|
| 1517 | * the tree log instead of forcing full commits whenever possible.
|
| 1518 | *
|
| 1519 | * It needs to call filemap_fdatawait so that all ordered extent updates are
|
| 1520 | * in the metadata btree are up to date for copying to the log.
|
| 1521 | *
|
| 1522 | * It drops the inode mutex before doing the tree log commit. This is an
|
| 1523 | * important optimization for directories because holding the mutex prevents
|
| 1524 | * new operations on the dir while we write to disk.
|
| 1525 |
|
| 1526 | int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync)
|
| 1527 | {
|
| 1528 | struct dentry *dentry = file_dentry(file);
|
| 1529 | struct btrfs_inode *inode = BTRFS_I(d_inode(dentry));
|
| 1530 | struct btrfs_root *root = inode->root;
|
| 1531 | struct btrfs_fs_info *fs_info = root->fs_info;
|
| 1532 | struct btrfs_trans_handle *trans;
|
| 1533 | struct btrfs_log_ctx ctx;
|
| 1534 | int ret = 0, err;
|
| 1535 | u64 len;
|
| 1536 | bool full_sync;
|
| 1537 | bool skip_ilock = false;
|
| 1538 |
|
| 1539 | if (current->journal_info == BTRFS_TRANS_DIO_WRITE_STUB) {
|
| 1540 | skip_ilock = true;
|
| 1541 | current->journal_info = NULL;
|
| 1542 | btrfs_assert_inode_locked(inode);
|
| 1543 | }
|
| 1544 |
|
| 1545 | trace_btrfs_sync_file_enter(file, datasync);
|
| 1546 |
|
| 1547 | btrfs_init_log_ctx(&ctx, inode);
|
| 1548 |
|
| 1549 | |
| 1550 | * Always set the range to a full range, otherwise we can get into
|
| 1551 | * several problems, from missing file extent items to represent holes
|
| 1552 | * when not using the NO_HOLES feature, to log tree corruption due to
|
| 1553 | * races between hole detection during logging and completion of ordered
|
| 1554 | * extents outside the range, to missing checksums due to ordered extents
|
| 1555 | * for which we flushed only a subset of their pages.
|
| 1556 |
|
| 1557 | start = 0;
|
| 1558 | end = LLONG_MAX;
|
| 1559 | len = (u64)LLONG_MAX + 1;
|
| 1560 |
|
| 1561 | |
| 1562 | * We write the dirty pages in the range and wait until they complete
|
| 1563 | * out of the ->i_mutex. If so, we can flush the dirty pages by
|
| 1564 | * multi-task, and make the performance up. See
|
| 1565 | * btrfs_wait_ordered_range for an explanation of the ASYNC check.
|
| 1566 |
|
| 1567 | ret = start_ordered_ops(inode, start, end);
|
| 1568 | if (ret)
|
| 1569 | goto out;
|
| 1570 |
|
| 1571 | if (skip_ilock)
|
| 1572 | down_write(&inode->i_mmap_lock);
|
| 1573 | else
|
| 1574 | btrfs_inode_lock(inode, BTRFS_ILOCK_MMAP);
|
| 1575 |
|
| 1576 | |
| 1577 | * Before we acquired the inode's lock and the mmap lock, someone may
|
| 1578 | * have dirtied more pages in the target range. We need to make sure
|
| 1579 | * that writeback for any such pages does not start while we are logging
|
| 1580 | * the inode, because if it does, any of the following might happen when
|
| 1581 | * we are not doing a full inode sync:
|
| 1582 | *
|
| 1583 | * 1) We log an extent after its writeback finishes but before its
|
| 1584 | * checksums are added to the csum tree, leading to -EIO errors
|
| 1585 | * when attempting to read the extent after a log replay.
|
| 1586 | *
|
| 1587 | * 2) We can end up logging an extent before its writeback finishes.
|
| 1588 | * Therefore after the log replay we will have a file extent item
|
| 1589 | * pointing to an unwritten extent (and no data checksums as well).
|
| 1590 | *
|
| 1591 | * So trigger writeback for any eventual new dirty pages and then we
|
| 1592 | * wait for all ordered extents to complete below.
|
| 1593 |
|
| 1594 | ret = start_ordered_ops(inode, start, end);
|
| 1595 | if (ret) {
|
| 1596 | if (skip_ilock)
|
| 1597 | up_write(&inode->i_mmap_lock);
|
| 1598 | else
|
| 1599 | btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
|
| 1600 | goto out;
|
| 1601 | }
|
| 1602 |
|
| 1603 | |
| 1604 | * Always check for the full sync flag while holding the inode's lock,
|
| 1605 | * to avoid races with other tasks. The flag must be either set all the
|
| 1606 | * time during logging or always off all the time while logging.
|
| 1607 | * We check the flag here after starting delalloc above, because when
|
| 1608 | * running delalloc the full sync flag may be set if we need to drop
|
| 1609 | * extra extent map ranges due to temporary memory allocation failures.
|
| 1610 |
|
| 1611 | full_sync = test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags);
|
| 1612 |
|
| 1613 | |
| 1614 | * We have to do this here to avoid the priority inversion of waiting on
|
| 1615 | * IO of a lower priority task while holding a transaction open.
|
| 1616 | *
|
| 1617 | * For a full fsync we wait for the ordered extents to complete while
|
| 1618 | * for a fast fsync we wait just for writeback to complete, and then
|
| 1619 | * attach the ordered extents to the transaction so that a transaction
|
| 1620 | * commit waits for their completion, to avoid data loss if we fsync,
|
| 1621 | * the current transaction commits before the ordered extents complete
|
| 1622 | * and a power failure happens right after that.
|
| 1623 | *
|
| 1624 | * For zoned filesystem, if a write IO uses a ZONE_APPEND command, the
|
| 1625 | * logical address recorded in the ordered extent may change. We need
|
| 1626 | * to wait for the IO to stabilize the logical address.
|
| 1627 |
|
| 1628 | if (full_sync || btrfs_is_zoned(fs_info)) {
|
| 1629 | ret = btrfs_wait_ordered_range(inode, start, len);
|
| 1630 | clear_bit(BTRFS_INODE_COW_WRITE_ERROR, &inode->runtime_flags);
|
| 1631 | } else {
|
| 1632 | |
| 1633 | * Get our ordered extents as soon as possible to avoid doing
|
| 1634 | * checksum lookups in the csum tree, and use instead the
|
| 1635 | * checksums attached to the ordered extents.
|
| 1636 |
|
| 1637 | btrfs_get_ordered_extents_for_logging(inode, &ctx.ordered_extents);
|
| 1638 | ret = filemap_fdatawait_range(inode->vfs_inode.i_mapping, start, end);
|
| 1639 | if (ret)
|
| 1640 | goto out_release_extents;
|
| 1641 |
|
| 1642 | |
| 1643 | * Check and clear the BTRFS_INODE_COW_WRITE_ERROR now after
|
| 1644 | * starting and waiting for writeback, because for buffered IO
|
| 1645 | * it may have been set during the end IO callback
|
| 1646 | * (end_bbio_data_write() -> btrfs_finish_ordered_extent()) in
|
| 1647 | * case an error happened and we need to wait for ordered
|
| 1648 | * extents to complete so that any extent maps that point to
|
| 1649 | * unwritten locations are dropped and we don't log them.
|
| 1650 |
|
| 1651 | if (test_and_clear_bit(BTRFS_INODE_COW_WRITE_ERROR, &inode->runtime_flags))
|
| 1652 | ret = btrfs_wait_ordered_range(inode, start, len);
|
| 1653 | }
|
| 1654 |
|
| 1655 | if (ret)
|
| 1656 | goto out_release_extents;
|
| 1657 |
|
| 1658 | if (skip_inode_logging(&ctx)) {
|
| 1659 | |
| 1660 | * We've had everything committed since the last time we were
|
| 1661 | * modified so clear this flag in case it was set for whatever
|
| 1662 | * reason, it's no longer relevant.
|
| 1663 |
|
| 1664 | clear_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags);
|
| 1665 | goto out_release_extents;
|
| 1666 | }
|
| 1667 |
|
| 1668 | btrfs_init_log_ctx_scratch_eb(&ctx);
|
| 1669 |
|
| 1670 | |
| 1671 | * We use start here because we will need to wait on the IO to complete
|
| 1672 | * in btrfs_sync_log, which could require joining a transaction (for
|
| 1673 | * example checking cross references in the nocow path). If we use join
|
| 1674 | * here we could get into a situation where we're waiting on IO to
|
| 1675 | * happen that is blocked on a transaction trying to commit. With start
|
| 1676 | * we inc the extwriter counter, so we wait for all extwriters to exit
|
| 1677 | * before we start blocking joiners. This comment is to keep somebody
|
| 1678 | * from thinking they are super smart and changing this to
|
| 1679 | * btrfs_join_transaction *cough*Josef*cough*.
|
| 1680 |
|
| 1681 | trans = btrfs_start_transaction(root, 0);
|
| 1682 | if (IS_ERR(trans)) {
|
| 1683 | ret = PTR_ERR(trans);
|
| 1684 | goto out_release_extents;
|
| 1685 | }
|
| 1686 | trans->in_fsync = true;
|
| 1687 |
|
| 1688 | ret = btrfs_log_dentry_safe(trans, dentry, &ctx);
|
| 1689 | |
| 1690 | * Scratch eb no longer needed, release before syncing log or commit
|
| 1691 | * transaction, to avoid holding unnecessary memory during such long
|
| 1692 | * operations.
|
| 1693 |
|
| 1694 | if (ctx.scratch_eb) {
|
| 1695 | free_extent_buffer(ctx.scratch_eb);
|
| 1696 | ctx.scratch_eb = NULL;
|
| 1697 | }
|
| 1698 | btrfs_release_log_ctx_extents(&ctx);
|
| 1699 | if (ret < 0) {
|
| 1700 |
|
| 1701 | ret = BTRFS_LOG_FORCE_COMMIT;
|
| 1702 | }
|
| 1703 |
|
| 1704 | |
| 1705 | * version of the file in the log. It is possible that
|
| 1706 | * someone will come in and modify the file, but that's
|
| 1707 | * fine because the log is consistent on disk, and we
|
| 1708 | * have references to all of the file's extents
|
| 1709 | *
|
| 1710 | * It is possible that someone will come in and log the
|
| 1711 | * file again, but that will end up using the synchronization
|
| 1712 | * inside btrfs_sync_log to keep things safe.
|
| 1713 |
|
| 1714 | if (skip_ilock)
|
| 1715 | up_write(&inode->i_mmap_lock);
|
| 1716 | else
|
| 1717 | btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
|
| 1718 |
|
| 1719 | if (ret == BTRFS_NO_LOG_SYNC) {
|
| 1720 | ret = btrfs_end_transaction(trans);
|
| 1721 | goto out;
|
| 1722 | }
|
| 1723 |
|
| 1724 |
|
| 1725 | if (!ret) {
|
| 1726 | ret = btrfs_sync_log(trans, root, &ctx);
|
| 1727 | if (!ret) {
|
| 1728 | ret = btrfs_end_transaction(trans);
|
| 1729 | goto out;
|
| 1730 | }
|
| 1731 | }
|
| 1732 |
|
| 1733 | |
| 1734 | * At this point we need to commit the transaction because we had
|
| 1735 | * btrfs_need_log_full_commit() or some other error.
|
| 1736 | *
|
| 1737 | * If we didn't do a full sync we have to stop the trans handle, wait on
|
| 1738 | * the ordered extents, start it again and commit the transaction. If
|
| 1739 | * we attempt to wait on the ordered extents here we could deadlock with
|
| 1740 | * something like fallocate() that is holding the extent lock trying to
|
| 1741 | * start a transaction while some other thread is trying to commit the
|
| 1742 | * transaction while we (fsync) are currently holding the transaction
|
| 1743 | * open.
|
| 1744 |
|
| 1745 | if (!full_sync) {
|
| 1746 | ret = btrfs_end_transaction(trans);
|
| 1747 | if (ret)
|
| 1748 | goto out;
|
| 1749 | ret = btrfs_wait_ordered_range(inode, start, len);
|
| 1750 | if (ret)
|
| 1751 | goto out;
|
| 1752 |
|
| 1753 | |
| 1754 | * This is safe to use here because we're only interested in
|
| 1755 | * making sure the transaction that had the ordered extents is
|
| 1756 | * committed. We aren't waiting on anything past this point,
|
| 1757 | * we're purely getting the transaction and committing it.
|
| 1758 |
|
| 1759 | trans = btrfs_attach_transaction_barrier(root);
|
| 1760 | if (IS_ERR(trans)) {
|
| 1761 | ret = PTR_ERR(trans);
|
| 1762 |
|
| 1763 | |
| 1764 | * We committed the transaction and there's no currently
|
| 1765 | * running transaction, this means everything we care
|
| 1766 | * about made it to disk and we are done.
|
| 1767 |
|
| 1768 | if (ret == -ENOENT)
|
| 1769 | ret = 0;
|
| 1770 | goto out;
|
| 1771 | }
|
| 1772 | }
|
| 1773 |
|
| 1774 | ret = btrfs_commit_transaction(trans);
|
| 1775 | out:
|
| 1776 | free_extent_buffer(ctx.scratch_eb);
|
| 1777 | ASSERT(list_empty(&ctx.list));
|
| 1778 | ASSERT(list_empty(&ctx.conflict_inodes));
|
| 1779 | ASSERT(ret <= 0, "ret=%d", ret);
|
| 1780 | |
| 1781 | * Ordered extents might have started and completed before this fsync,
|
| 1782 | * so check for any io errors and advance the writeback error sequence.
|
| 1783 |
|
| 1784 | err = file_check_and_advance_wb_err(file);
|
| 1785 | if (!ret)
|
| 1786 | ret = err;
|
| 1787 | trace_btrfs_sync_file_exit(file, ret);
|
| 1788 |
|
| 1789 | return ret;
|
| 1790 |
|
| 1791 | out_release_extents:
|
| 1792 | btrfs_release_log_ctx_extents(&ctx);
|
| 1793 | if (skip_ilock)
|
| 1794 | up_write(&inode->i_mmap_lock);
|
| 1795 | else
|
| 1796 | btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
|
| 1797 | goto out;
|
| 1798 | }
|
| 1799 |
|
| 1800 | |
| 1801 | * btrfs_page_mkwrite() is not allowed to change the file size as it gets
|
| 1802 | * called from a page fault handler when a page is first dirtied. Hence we must
|
| 1803 | * be careful to check for EOF conditions here. We set the page up correctly
|
| 1804 | * for a written page which means we get ENOSPC checking when writing into
|
| 1805 | * holes and correct delalloc and unwritten extent mapping on filesystems that
|
| 1806 | * support these features.
|
| 1807 | *
|
| 1808 | * We are not allowed to take the i_mutex here so we have to play games to
|
| 1809 | * protect against truncate races as the page could now be beyond EOF. Because
|
| 1810 | * truncate_setsize() writes the inode size before removing pages, once we have
|
| 1811 | * the page lock we can determine safely if the page is beyond EOF. If it is not
|
| 1812 | * beyond EOF, then the page is guaranteed safe against truncation until we
|
| 1813 | * unlock the page.
|
| 1814 |
|
| 1815 | static vm_fault_t btrfs_page_mkwrite(struct vm_fault *vmf)
|
| 1816 | {
|
| 1817 | struct page *page = vmf->page;
|
| 1818 | struct folio *folio = page_folio(page);
|
| 1819 | struct btrfs_inode *inode = BTRFS_I(file_inode(vmf->vma->vm_file));
|
| 1820 | struct btrfs_fs_info *fs_info = inode->root->fs_info;
|
| 1821 | struct extent_io_tree *io_tree = &inode->io_tree;
|
| 1822 | struct btrfs_ordered_extent *ordered;
|
| 1823 | struct extent_state *cached_state = NULL;
|
| 1824 | struct extent_changeset *data_reserved = NULL;
|
| 1825 | unsigned long zero_start;
|
| 1826 | loff_t size;
|
| 1827 | size_t fsize = folio_size(folio);
|
| 1828 | int ret;
|
| 1829 | bool only_release_metadata = false;
|
| 1830 | u64 reserved_space;
|
| 1831 | u64 page_start;
|
| 1832 | u64 page_end;
|
| 1833 | u64 end;
|
| 1834 |
|
| 1835 | reserved_space = fsize;
|
| 1836 |
|
| 1837 | sb_start_pagefault(inode->vfs_inode.i_sb);
|
| 1838 | page_start = folio_pos(folio);
|
| 1839 | page_end = page_start + folio_size(folio) - 1;
|
| 1840 | end = page_end;
|
| 1841 |
|
| 1842 | |
| 1843 | * Reserving delalloc space after obtaining the page lock can lead to
|
| 1844 | * deadlock. For example, if a dirty page is locked by this function
|
| 1845 | * and the call to btrfs_delalloc_reserve_space() ends up triggering
|
| 1846 | * dirty page write out, then the btrfs_writepages() function could
|
| 1847 | * end up waiting indefinitely to get a lock on the page currently
|
| 1848 | * being processed by btrfs_page_mkwrite() function.
|
| 1849 |
|
| 1850 | ret = btrfs_check_data_free_space(inode, &data_reserved, page_start,
|
| 1851 | reserved_space, false);
|
| 1852 | if (ret < 0) {
|
| 1853 | size_t write_bytes = reserved_space;
|
| 1854 |
|
| 1855 | if (btrfs_check_nocow_lock(inode, page_start, &write_bytes, false) <= 0)
|
| 1856 | goto out_noreserve;
|
| 1857 |
|
| 1858 | only_release_metadata = true;
|
| 1859 |
|
| 1860 | |
| 1861 | * Can't write the whole range, there may be shared extents or
|
| 1862 | * holes in the range, bail out with @only_release_metadata set
|
| 1863 | * to true so that we unlock the nocow lock before returning the
|
| 1864 | * error.
|
| 1865 |
|
| 1866 | if (write_bytes < reserved_space)
|
| 1867 | goto out_noreserve;
|
| 1868 | }
|
| 1869 | ret = btrfs_delalloc_reserve_metadata(inode, reserved_space,
|
| 1870 | reserved_space, false);
|
| 1871 | if (ret < 0) {
|
| 1872 | if (!only_release_metadata)
|
| 1873 | btrfs_free_reserved_data_space(inode, data_reserved,
|
| 1874 | page_start, reserved_space);
|
| 1875 | goto out_noreserve;
|
| 1876 | }
|
| 1877 |
|
| 1878 | ret = file_update_time(vmf->vma->vm_file);
|
| 1879 | if (ret < 0)
|
| 1880 | goto out;
|
| 1881 | again:
|
| 1882 | down_read(&inode->i_mmap_lock);
|
| 1883 | folio_lock(folio);
|
| 1884 | size = i_size_read(&inode->vfs_inode);
|
| 1885 |
|
| 1886 | if ((folio->mapping != inode->vfs_inode.i_mapping) ||
|
| 1887 | (page_start >= size)) {
|
| 1888 |
|
| 1889 | goto out_unlock;
|
| 1890 | }
|
| 1891 | folio_wait_writeback(folio);
|
| 1892 |
|
| 1893 | btrfs_lock_extent(io_tree, page_start, page_end, &cached_state);
|
| 1894 | ret = set_folio_extent_mapped(folio);
|
| 1895 | if (ret < 0) {
|
| 1896 | btrfs_unlock_extent(io_tree, page_start, page_end, &cached_state);
|
| 1897 | goto out_unlock;
|
| 1898 | }
|
| 1899 |
|
| 1900 | |
| 1901 | * We can't set the delalloc bits if there are pending ordered
|
| 1902 | * extents. Drop our locks and wait for them to finish.
|
| 1903 |
|
| 1904 | ordered = btrfs_lookup_ordered_range(inode, page_start, fsize);
|
| 1905 | if (ordered) {
|
| 1906 | btrfs_unlock_extent(io_tree, page_start, page_end, &cached_state);
|
| 1907 | folio_unlock(folio);
|
| 1908 | up_read(&inode->i_mmap_lock);
|
| 1909 | btrfs_start_ordered_extent(ordered);
|
| 1910 | btrfs_put_ordered_extent(ordered);
|
| 1911 | goto again;
|
| 1912 | }
|
| 1913 |
|
| 1914 | if (folio_contains(folio, (size - 1) >> PAGE_SHIFT)) {
|
| 1915 | reserved_space = round_up(size - page_start, fs_info->sectorsize);
|
| 1916 | if (reserved_space < fsize) {
|
| 1917 | const u64 to_free = fsize - reserved_space;
|
| 1918 |
|
| 1919 | end = page_start + reserved_space - 1;
|
| 1920 | if (only_release_metadata)
|
| 1921 | btrfs_delalloc_release_metadata(inode, to_free, true);
|
| 1922 | else
|
| 1923 | btrfs_delalloc_release_space(inode, data_reserved,
|
| 1924 | end + 1, to_free, true);
|
| 1925 | }
|
| 1926 | }
|
| 1927 |
|
| 1928 | ret = btrfs_reset_extent_delalloc(inode, page_start, end, 0, &cached_state);
|
| 1929 | if (ret < 0) {
|
| 1930 | btrfs_unlock_extent(io_tree, page_start, page_end, &cached_state);
|
| 1931 | goto out_unlock;
|
| 1932 | }
|
| 1933 |
|
| 1934 |
|
| 1935 | if (page_start + folio_size(folio) > size)
|
| 1936 | zero_start = offset_in_folio(folio, size);
|
| 1937 | else
|
| 1938 | zero_start = fsize;
|
| 1939 |
|
| 1940 | if (zero_start != fsize)
|
| 1941 | folio_zero_range(folio, zero_start, folio_size(folio) - zero_start);
|
| 1942 |
|
| 1943 | btrfs_folio_set_dirty(fs_info, folio, page_start, end + 1 - page_start);
|
| 1944 | btrfs_folio_set_uptodate(fs_info, folio, page_start, end + 1 - page_start);
|
| 1945 |
|
| 1946 | btrfs_set_inode_last_sub_trans(inode);
|
| 1947 |
|
| 1948 | if (only_release_metadata)
|
| 1949 | btrfs_set_extent_bit(io_tree, page_start, end, EXTENT_NORESERVE,
|
| 1950 | &cached_state);
|
| 1951 |
|
| 1952 | btrfs_unlock_extent(io_tree, page_start, page_end, &cached_state);
|
| 1953 | up_read(&inode->i_mmap_lock);
|
| 1954 |
|
| 1955 | btrfs_delalloc_release_extents(inode, fsize);
|
| 1956 | if (only_release_metadata)
|
| 1957 | btrfs_check_nocow_unlock(inode);
|
| 1958 | sb_end_pagefault(inode->vfs_inode.i_sb);
|
| 1959 | extent_changeset_free(data_reserved);
|
| 1960 | return VM_FAULT_LOCKED;
|
| 1961 |
|
| 1962 | out_unlock:
|
| 1963 | folio_unlock(folio);
|
| 1964 | up_read(&inode->i_mmap_lock);
|
| 1965 | out:
|
| 1966 | btrfs_delalloc_release_extents(inode, fsize);
|
| 1967 | if (only_release_metadata)
|
| 1968 | btrfs_delalloc_release_metadata(inode, reserved_space, true);
|
| 1969 | else
|
| 1970 | btrfs_delalloc_release_space(inode, data_reserved, page_start,
|
| 1971 | reserved_space, true);
|
| 1972 | out_noreserve:
|
| 1973 | if (only_release_metadata)
|
| 1974 | btrfs_check_nocow_unlock(inode);
|
| 1975 |
|
| 1976 | sb_end_pagefault(inode->vfs_inode.i_sb);
|
| 1977 |
|
| 1978 | extent_changeset_free(data_reserved);
|
| 1979 |
|
| 1980 | if (ret < 0)
|
| 1981 | return vmf_error(ret);
|
| 1982 |
|
| 1983 |
|
| 1984 | return VM_FAULT_NOPAGE;
|
| 1985 | }
|
| 1986 |
|
| 1987 | static const struct vm_operations_struct btrfs_file_vm_ops = {
|
| 1988 | .fault = filemap_fault,
|
| 1989 | .map_pages = filemap_map_pages,
|
| 1990 | .page_mkwrite = btrfs_page_mkwrite,
|
| 1991 | };
|
| 1992 |
|
| 1993 | static int btrfs_file_mmap_prepare(struct vm_area_desc *desc)
|
| 1994 | {
|
| 1995 | struct file *filp = desc->file;
|
| 1996 | struct address_space *mapping = filp->f_mapping;
|
| 1997 |
|
| 1998 | if (btrfs_is_shutdown(inode_to_fs_info(file_inode(filp))))
|
| 1999 | return -EIO;
|
| 2000 | if (!mapping->a_ops->read_folio)
|
| 2001 | return -ENOEXEC;
|
| 2002 |
|
| 2003 | file_accessed(filp);
|
| 2004 | desc->vm_ops = &btrfs_file_vm_ops;
|
| 2005 |
|
| 2006 | return 0;
|
| 2007 | }
|
| 2008 |
|
| 2009 | static bool hole_mergeable(struct btrfs_inode *inode, struct extent_buffer *leaf,
|
| 2010 | int slot, u64 start, u64 end)
|
| 2011 | {
|
| 2012 | struct btrfs_file_extent_item *fi;
|
| 2013 | struct btrfs_key key;
|
| 2014 |
|
| 2015 | if (slot < 0 || slot >= btrfs_header_nritems(leaf))
|
| 2016 | return false;
|
| 2017 |
|
| 2018 | btrfs_item_key_to_cpu(leaf, &key, slot);
|
| 2019 | if (key.objectid != btrfs_ino(inode) ||
|
| 2020 | key.type != BTRFS_EXTENT_DATA_KEY)
|
| 2021 | return false;
|
| 2022 |
|
| 2023 | fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
|
| 2024 |
|
| 2025 | if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG)
|
| 2026 | return false;
|
| 2027 |
|
| 2028 | if (btrfs_file_extent_disk_bytenr(leaf, fi))
|
| 2029 | return false;
|
| 2030 |
|
| 2031 | if (key.offset == end)
|
| 2032 | return true;
|
| 2033 | if (key.offset + btrfs_file_extent_num_bytes(leaf, fi) == start)
|
| 2034 | return true;
|
| 2035 | return false;
|
| 2036 | }
|
| 2037 |
|
| 2038 | static int fill_holes(struct btrfs_trans_handle *trans,
|
| 2039 | struct btrfs_inode *inode,
|
| 2040 | struct btrfs_path *path, u64 offset, u64 end)
|
| 2041 | {
|
| 2042 | struct btrfs_fs_info *fs_info = trans->fs_info;
|
| 2043 | struct btrfs_root *root = inode->root;
|
| 2044 | struct extent_buffer *leaf;
|
| 2045 | struct btrfs_file_extent_item *fi;
|
| 2046 | struct extent_map *hole_em;
|
| 2047 | struct btrfs_key key;
|
| 2048 | int modify_slot = -1;
|
| 2049 | int del_slot = -1;
|
| 2050 | bool update_offset = false;
|
| 2051 | u64 num_bytes = 0;
|
| 2052 | int ret;
|
| 2053 |
|
| 2054 | if (btrfs_fs_incompat(fs_info, NO_HOLES))
|
| 2055 | goto out;
|
| 2056 |
|
| 2057 | key.objectid = btrfs_ino(inode);
|
| 2058 | key.type = BTRFS_EXTENT_DATA_KEY;
|
| 2059 | key.offset = offset;
|
| 2060 |
|
| 2061 | ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
|
| 2062 | if (ret <= 0) {
|
| 2063 | |
| 2064 | * We should have dropped this offset, so if we find it then
|
| 2065 | * something has gone horribly wrong.
|
| 2066 |
|
| 2067 | if (ret == 0)
|
| 2068 | ret = -EINVAL;
|
| 2069 | return ret;
|
| 2070 | }
|
| 2071 |
|
| 2072 | leaf = path->nodes[0];
|
| 2073 | if (hole_mergeable(inode, leaf, path->slots[0] - 1, offset, end)) {
|
| 2074 | fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
|
| 2075 | struct btrfs_file_extent_item);
|
| 2076 | num_bytes = btrfs_file_extent_num_bytes(leaf, fi) +
|
| 2077 | end - offset;
|
| 2078 | modify_slot = path->slots[0] - 1;
|
| 2079 | }
|
| 2080 | if (hole_mergeable(inode, leaf, path->slots[0], offset, end)) {
|
| 2081 | fi = btrfs_item_ptr(leaf, path->slots[0],
|
| 2082 | struct btrfs_file_extent_item);
|
| 2083 | if (modify_slot != -1) {
|
| 2084 | num_bytes += btrfs_file_extent_num_bytes(leaf, fi);
|
| 2085 | del_slot = path->slots[0];
|
| 2086 | } else {
|
| 2087 | num_bytes = btrfs_file_extent_num_bytes(leaf, fi) +
|
| 2088 | end - offset;
|
| 2089 | modify_slot = path->slots[0];
|
| 2090 | update_offset = true;
|
| 2091 | }
|
| 2092 | }
|
| 2093 | if (modify_slot >= 0) {
|
| 2094 | fi = btrfs_item_ptr(leaf, modify_slot,
|
| 2095 | struct btrfs_file_extent_item);
|
| 2096 | btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
|
| 2097 | btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes);
|
| 2098 | if (update_offset) {
|
| 2099 | key.offset = offset;
|
| 2100 | btrfs_set_item_key_safe(trans, path, &key);
|
| 2101 | }
|
| 2102 | btrfs_set_file_extent_offset(leaf, fi, 0);
|
| 2103 | btrfs_set_file_extent_generation(leaf, fi, trans->transid);
|
| 2104 | if (del_slot >= 0) {
|
| 2105 | ret = btrfs_del_items(trans, root, path, del_slot, 1);
|
| 2106 | if (ret) {
|
| 2107 | btrfs_abort_transaction(trans, ret);
|
| 2108 | btrfs_release_path(path);
|
| 2109 | return ret;
|
| 2110 | }
|
| 2111 | }
|
| 2112 | goto out;
|
| 2113 | }
|
| 2114 | btrfs_release_path(path);
|
| 2115 |
|
| 2116 | ret = btrfs_insert_hole_extent(trans, root, btrfs_ino(inode), offset,
|
| 2117 | end - offset);
|
| 2118 | if (ret)
|
| 2119 | return ret;
|
| 2120 |
|
| 2121 | out:
|
| 2122 | btrfs_release_path(path);
|
| 2123 |
|
| 2124 | hole_em = btrfs_alloc_extent_map();
|
| 2125 | if (!hole_em) {
|
| 2126 | btrfs_drop_extent_map_range(inode, offset, end - 1, false);
|
| 2127 | btrfs_set_inode_full_sync(inode);
|
| 2128 | } else {
|
| 2129 | hole_em->start = offset;
|
| 2130 | hole_em->len = end - offset;
|
| 2131 | hole_em->ram_bytes = hole_em->len;
|
| 2132 |
|
| 2133 | hole_em->disk_bytenr = EXTENT_MAP_HOLE;
|
| 2134 | hole_em->disk_num_bytes = 0;
|
| 2135 | hole_em->generation = trans->transid;
|
| 2136 |
|
| 2137 | ret = btrfs_replace_extent_map_range(inode, hole_em, true);
|
| 2138 | btrfs_free_extent_map(hole_em);
|
| 2139 | if (ret)
|
| 2140 | btrfs_set_inode_full_sync(inode);
|
| 2141 | }
|
| 2142 |
|
| 2143 | return 0;
|
| 2144 | }
|
| 2145 |
|
| 2146 | |
| 2147 | * Find a hole extent on given inode and change start/len to the end of hole
|
| 2148 | * extent.(hole/vacuum extent whose em->start <= start &&
|
| 2149 | * em->start + em->len > start)
|
| 2150 | * When a hole extent is found, return 1 and modify start/len.
|
| 2151 |
|
| 2152 | static int find_first_non_hole(struct btrfs_inode *inode, u64 *start, u64 *len)
|
| 2153 | {
|
| 2154 | struct btrfs_fs_info *fs_info = inode->root->fs_info;
|
| 2155 | struct extent_map *em;
|
| 2156 | int ret = 0;
|
| 2157 |
|
| 2158 | em = btrfs_get_extent(inode, NULL,
|
| 2159 | round_down(*start, fs_info->sectorsize),
|
| 2160 | round_up(*len, fs_info->sectorsize));
|
| 2161 | if (IS_ERR(em))
|
| 2162 | return PTR_ERR(em);
|
| 2163 |
|
| 2164 |
|
| 2165 | if (em->disk_bytenr == EXTENT_MAP_HOLE) {
|
| 2166 | const u64 em_end = btrfs_extent_map_end(em);
|
| 2167 |
|
| 2168 | ret = 1;
|
| 2169 | *len = (em_end > *start + *len) ? 0 : (*start + *len - em_end);
|
| 2170 | *start = em_end;
|
| 2171 | }
|
| 2172 | btrfs_free_extent_map(em);
|
| 2173 | return ret;
|
| 2174 | }
|
| 2175 |
|
| 2176 | |
| 2177 | * Check if there is no folio in the range.
|
| 2178 | *
|
| 2179 | * We cannot utilize filemap_range_has_page() in a filemap with large folios
|
| 2180 | * as we can hit the following false positive:
|
| 2181 | *
|
| 2182 | * start end
|
| 2183 | * | |
|
| 2184 | * |//|//|//|//| | | | | | | | |//|//|
|
| 2185 | * \ / \ /
|
| 2186 | * Folio A Folio B
|
| 2187 | *
|
| 2188 | * That large folio A and B cover the start and end indexes.
|
| 2189 | * In that case filemap_range_has_page() will always return true, but the above
|
| 2190 | * case is fine for btrfs_punch_hole_lock_range() usage.
|
| 2191 | *
|
| 2192 | * So here we only ensure that no other folios is in the range, excluding the
|
| 2193 | * head/tail large folio.
|
| 2194 |
|
| 2195 | static bool check_range_has_page(struct inode *inode, u64 start, u64 end)
|
| 2196 | {
|
| 2197 | struct folio_batch fbatch;
|
| 2198 | bool ret = false;
|
| 2199 | |
| 2200 | * For subpage case, if the range is not at page boundary, we could
|
| 2201 | * have pages at the leading/tailing part of the range.
|
| 2202 | * This could lead to dead loop since filemap_range_has_page()
|
| 2203 | * will always return true.
|
| 2204 | * So here we need to do extra page alignment for
|
| 2205 | * filemap_range_has_page().
|
| 2206 | *
|
| 2207 | * And do not decrease page_lockend right now, as it can be 0.
|
| 2208 |
|
| 2209 | const u64 page_lockstart = round_up(start, PAGE_SIZE);
|
| 2210 | const u64 page_lockend = round_down(end + 1, PAGE_SIZE);
|
| 2211 | const pgoff_t start_index = page_lockstart >> PAGE_SHIFT;
|
| 2212 | const pgoff_t end_index = (page_lockend - 1) >> PAGE_SHIFT;
|
| 2213 | pgoff_t tmp = start_index;
|
| 2214 | int found_folios;
|
| 2215 |
|
| 2216 |
|
| 2217 | if (page_lockend <= page_lockstart)
|
| 2218 | return false;
|
| 2219 |
|
| 2220 | folio_batch_init(&fbatch);
|
| 2221 | found_folios = filemap_get_folios(inode->i_mapping, &tmp, end_index, &fbatch);
|
| 2222 | for (int i = 0; i < found_folios; i++) {
|
| 2223 | struct folio *folio = fbatch.folios[i];
|
| 2224 |
|
| 2225 |
|
| 2226 | if (folio->index < start_index)
|
| 2227 | continue;
|
| 2228 |
|
| 2229 | if (folio_next_index(folio) > end_index)
|
| 2230 | continue;
|
| 2231 |
|
| 2232 | ret = true;
|
| 2233 | break;
|
| 2234 | }
|
| 2235 | folio_batch_release(&fbatch);
|
| 2236 | return ret;
|
| 2237 | }
|
| 2238 |
|
| 2239 | static void btrfs_punch_hole_lock_range(struct inode *inode,
|
| 2240 | const u64 lockstart, const u64 lockend,
|
| 2241 | struct extent_state **cached_state)
|
| 2242 | {
|
| 2243 | while (1) {
|
| 2244 | truncate_pagecache_range(inode, lockstart, lockend);
|
| 2245 |
|
| 2246 | btrfs_lock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend,
|
| 2247 | cached_state);
|
| 2248 | |
| 2249 | * We can't have ordered extents in the range, nor dirty/writeback
|
| 2250 | * pages, because we have locked the inode's VFS lock in exclusive
|
| 2251 | * mode, we have locked the inode's i_mmap_lock in exclusive mode,
|
| 2252 | * we have flushed all delalloc in the range and we have waited
|
| 2253 | * for any ordered extents in the range to complete.
|
| 2254 | * We can race with anyone reading pages from this range, so after
|
| 2255 | * locking the range check if we have pages in the range, and if
|
| 2256 | * we do, unlock the range and retry.
|
| 2257 |
|
| 2258 | if (!check_range_has_page(inode, lockstart, lockend))
|
| 2259 | break;
|
| 2260 |
|
| 2261 | btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend,
|
| 2262 | cached_state);
|
| 2263 | }
|
| 2264 |
|
| 2265 | btrfs_assert_inode_range_clean(BTRFS_I(inode), lockstart, lockend);
|
| 2266 | }
|
| 2267 |
|
| 2268 | static int btrfs_insert_replace_extent(struct btrfs_trans_handle *trans,
|
| 2269 | struct btrfs_inode *inode,
|
| 2270 | struct btrfs_path *path,
|
| 2271 | struct btrfs_replace_extent_info *extent_info,
|
| 2272 | const u64 replace_len,
|
| 2273 | const u64 bytes_to_drop)
|
| 2274 | {
|
| 2275 | struct btrfs_fs_info *fs_info = trans->fs_info;
|
| 2276 | struct btrfs_root *root = inode->root;
|
| 2277 | struct btrfs_file_extent_item *extent;
|
| 2278 | struct extent_buffer *leaf;
|
| 2279 | struct btrfs_key key;
|
| 2280 | int slot;
|
| 2281 | int ret;
|
| 2282 |
|
| 2283 | if (replace_len == 0)
|
| 2284 | return 0;
|
| 2285 |
|
| 2286 | if (extent_info->disk_offset == 0 &&
|
| 2287 | btrfs_fs_incompat(fs_info, NO_HOLES)) {
|
| 2288 | btrfs_update_inode_bytes(inode, 0, bytes_to_drop);
|
| 2289 | return 0;
|
| 2290 | }
|
| 2291 |
|
| 2292 | key.objectid = btrfs_ino(inode);
|
| 2293 | key.type = BTRFS_EXTENT_DATA_KEY;
|
| 2294 | key.offset = extent_info->file_offset;
|
| 2295 | ret = btrfs_insert_empty_item(trans, root, path, &key,
|
| 2296 | sizeof(struct btrfs_file_extent_item));
|
| 2297 | if (ret)
|
| 2298 | return ret;
|
| 2299 | leaf = path->nodes[0];
|
| 2300 | slot = path->slots[0];
|
| 2301 | write_extent_buffer(leaf, extent_info->extent_buf,
|
| 2302 | btrfs_item_ptr_offset(leaf, slot),
|
| 2303 | sizeof(struct btrfs_file_extent_item));
|
| 2304 | extent = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
|
| 2305 | ASSERT(btrfs_file_extent_type(leaf, extent) != BTRFS_FILE_EXTENT_INLINE);
|
| 2306 | btrfs_set_file_extent_offset(leaf, extent, extent_info->data_offset);
|
| 2307 | btrfs_set_file_extent_num_bytes(leaf, extent, replace_len);
|
| 2308 | if (extent_info->is_new_extent)
|
| 2309 | btrfs_set_file_extent_generation(leaf, extent, trans->transid);
|
| 2310 | btrfs_release_path(path);
|
| 2311 |
|
| 2312 | ret = btrfs_inode_set_file_extent_range(inode, extent_info->file_offset,
|
| 2313 | replace_len);
|
| 2314 | if (ret)
|
| 2315 | return ret;
|
| 2316 |
|
| 2317 |
|
| 2318 | if (extent_info->disk_offset == 0) {
|
| 2319 | btrfs_update_inode_bytes(inode, 0, bytes_to_drop);
|
| 2320 | return 0;
|
| 2321 | }
|
| 2322 |
|
| 2323 | btrfs_update_inode_bytes(inode, replace_len, bytes_to_drop);
|
| 2324 |
|
| 2325 | if (extent_info->is_new_extent && extent_info->insertions == 0) {
|
| 2326 | key.objectid = extent_info->disk_offset;
|
| 2327 | key.type = BTRFS_EXTENT_ITEM_KEY;
|
| 2328 | key.offset = extent_info->disk_len;
|
| 2329 | ret = btrfs_alloc_reserved_file_extent(trans, root,
|
| 2330 | btrfs_ino(inode),
|
| 2331 | extent_info->file_offset,
|
| 2332 | extent_info->qgroup_reserved,
|
| 2333 | &key);
|
| 2334 | } else {
|
| 2335 | struct btrfs_ref ref = {
|
| 2336 | .action = BTRFS_ADD_DELAYED_REF,
|
| 2337 | .bytenr = extent_info->disk_offset,
|
| 2338 | .num_bytes = extent_info->disk_len,
|
| 2339 | .owning_root = btrfs_root_id(root),
|
| 2340 | .ref_root = btrfs_root_id(root),
|
| 2341 | };
|
| 2342 | u64 ref_offset;
|
| 2343 |
|
| 2344 | ref_offset = extent_info->file_offset - extent_info->data_offset;
|
| 2345 | btrfs_init_data_ref(&ref, btrfs_ino(inode), ref_offset, 0, false);
|
| 2346 | ret = btrfs_inc_extent_ref(trans, &ref);
|
| 2347 | }
|
| 2348 |
|
| 2349 | extent_info->insertions++;
|
| 2350 |
|
| 2351 | return ret;
|
| 2352 | }
|
| 2353 |
|
| 2354 | |
| 2355 | * The respective range must have been previously locked, as well as the inode.
|
| 2356 | * The end offset is inclusive (last byte of the range).
|
| 2357 | * @extent_info is NULL for fallocate's hole punching and non-NULL when replacing
|
| 2358 | * the file range with an extent.
|
| 2359 | * When not punching a hole, we don't want to end up in a state where we dropped
|
| 2360 | * extents without inserting a new one, so we must abort the transaction to avoid
|
| 2361 | * a corruption.
|
| 2362 |
|
| 2363 | int btrfs_replace_file_extents(struct btrfs_inode *inode,
|
| 2364 | struct btrfs_path *path, const u64 start,
|
| 2365 | const u64 end,
|
| 2366 | struct btrfs_replace_extent_info *extent_info,
|
| 2367 | struct btrfs_trans_handle **trans_out)
|
| 2368 | {
|
| 2369 | struct btrfs_drop_extents_args drop_args = { 0 };
|
| 2370 | struct btrfs_root *root = inode->root;
|
| 2371 | struct btrfs_fs_info *fs_info = root->fs_info;
|
| 2372 | const u64 min_size = btrfs_calc_insert_metadata_size(fs_info, 1);
|
| 2373 | u64 ino_size = round_up(inode->vfs_inode.i_size, fs_info->sectorsize);
|
| 2374 | struct btrfs_trans_handle *trans = NULL;
|
| 2375 | struct btrfs_block_rsv rsv;
|
| 2376 | unsigned int rsv_count;
|
| 2377 | u64 cur_offset;
|
| 2378 | u64 len = end - start;
|
| 2379 | int ret = 0;
|
| 2380 |
|
| 2381 | if (end <= start)
|
| 2382 | return -EINVAL;
|
| 2383 |
|
| 2384 | btrfs_init_metadata_block_rsv(fs_info, &rsv, BTRFS_BLOCK_RSV_TEMP);
|
| 2385 | rsv.size = min_size;
|
| 2386 | rsv.failfast = true;
|
| 2387 |
|
| 2388 | |
| 2389 | * 1 - update the inode
|
| 2390 | * 1 - removing the extents in the range
|
| 2391 | * 1 - adding the hole extent if no_holes isn't set or if we are
|
| 2392 | * replacing the range with a new extent
|
| 2393 |
|
| 2394 | if (!btrfs_fs_incompat(fs_info, NO_HOLES) || extent_info)
|
| 2395 | rsv_count = 3;
|
| 2396 | else
|
| 2397 | rsv_count = 2;
|
| 2398 |
|
| 2399 | trans = btrfs_start_transaction(root, rsv_count);
|
| 2400 | if (IS_ERR(trans)) {
|
| 2401 | ret = PTR_ERR(trans);
|
| 2402 | trans = NULL;
|
| 2403 | goto out_release;
|
| 2404 | }
|
| 2405 |
|
| 2406 | ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, &rsv,
|
| 2407 | min_size, false);
|
| 2408 | if (WARN_ON(ret))
|
| 2409 | goto out_trans;
|
| 2410 | trans->block_rsv = &rsv;
|
| 2411 |
|
| 2412 | cur_offset = start;
|
| 2413 | drop_args.path = path;
|
| 2414 | drop_args.end = end + 1;
|
| 2415 | drop_args.drop_cache = true;
|
| 2416 | while (cur_offset < end) {
|
| 2417 | drop_args.start = cur_offset;
|
| 2418 | ret = btrfs_drop_extents(trans, root, inode, &drop_args);
|
| 2419 |
|
| 2420 | if (!extent_info)
|
| 2421 | btrfs_update_inode_bytes(inode, 0,
|
| 2422 | drop_args.bytes_found);
|
| 2423 | if (ret != -ENOSPC) {
|
| 2424 | |
| 2425 | * The only time we don't want to abort is if we are
|
| 2426 | * attempting to clone a partial inline extent, in which
|
| 2427 | * case we'll get EOPNOTSUPP. However if we aren't
|
| 2428 | * clone we need to abort no matter what, because if we
|
| 2429 | * got EOPNOTSUPP via prealloc then we messed up and
|
| 2430 | * need to abort.
|
| 2431 |
|
| 2432 | if (unlikely(ret &&
|
| 2433 | (ret != -EOPNOTSUPP ||
|
| 2434 | (extent_info && extent_info->is_new_extent))))
|
| 2435 | btrfs_abort_transaction(trans, ret);
|
| 2436 | break;
|
| 2437 | }
|
| 2438 |
|
| 2439 | trans->block_rsv = &fs_info->trans_block_rsv;
|
| 2440 |
|
| 2441 | if (!extent_info && cur_offset < drop_args.drop_end &&
|
| 2442 | cur_offset < ino_size) {
|
| 2443 | ret = fill_holes(trans, inode, path, cur_offset,
|
| 2444 | drop_args.drop_end);
|
| 2445 | if (unlikely(ret)) {
|
| 2446 | |
| 2447 | * If we failed then we didn't insert our hole
|
| 2448 | * entries for the area we dropped, so now the
|
| 2449 | * fs is corrupted, so we must abort the
|
| 2450 | * transaction.
|
| 2451 |
|
| 2452 | btrfs_abort_transaction(trans, ret);
|
| 2453 | break;
|
| 2454 | }
|
| 2455 | } else if (!extent_info && cur_offset < drop_args.drop_end) {
|
| 2456 | |
| 2457 | * We are past the i_size here, but since we didn't
|
| 2458 | * insert holes we need to clear the mapped area so we
|
| 2459 | * know to not set disk_i_size in this area until a new
|
| 2460 | * file extent is inserted here.
|
| 2461 |
|
| 2462 | ret = btrfs_inode_clear_file_extent_range(inode,
|
| 2463 | cur_offset,
|
| 2464 | drop_args.drop_end - cur_offset);
|
| 2465 | if (unlikely(ret)) {
|
| 2466 | |
| 2467 | * We couldn't clear our area, so we could
|
| 2468 | * presumably adjust up and corrupt the fs, so
|
| 2469 | * we need to abort.
|
| 2470 |
|
| 2471 | btrfs_abort_transaction(trans, ret);
|
| 2472 | break;
|
| 2473 | }
|
| 2474 | }
|
| 2475 |
|
| 2476 | if (extent_info &&
|
| 2477 | drop_args.drop_end > extent_info->file_offset) {
|
| 2478 | u64 replace_len = drop_args.drop_end -
|
| 2479 | extent_info->file_offset;
|
| 2480 |
|
| 2481 | ret = btrfs_insert_replace_extent(trans, inode, path,
|
| 2482 | extent_info, replace_len,
|
| 2483 | drop_args.bytes_found);
|
| 2484 | if (unlikely(ret)) {
|
| 2485 | btrfs_abort_transaction(trans, ret);
|
| 2486 | break;
|
| 2487 | }
|
| 2488 | extent_info->data_len -= replace_len;
|
| 2489 | extent_info->data_offset += replace_len;
|
| 2490 | extent_info->file_offset += replace_len;
|
| 2491 | }
|
| 2492 |
|
| 2493 | |
| 2494 | * We are releasing our handle on the transaction, balance the
|
| 2495 | * dirty pages of the btree inode and flush delayed items, and
|
| 2496 | * then get a new transaction handle, which may now point to a
|
| 2497 | * new transaction in case someone else may have committed the
|
| 2498 | * transaction we used to replace/drop file extent items. So
|
| 2499 | * bump the inode's iversion and update mtime and ctime except
|
| 2500 | * if we are called from a dedupe context. This is because a
|
| 2501 | * power failure/crash may happen after the transaction is
|
| 2502 | * committed and before we finish replacing/dropping all the
|
| 2503 | * file extent items we need.
|
| 2504 |
|
| 2505 | inode_inc_iversion(&inode->vfs_inode);
|
| 2506 |
|
| 2507 | if (!extent_info || extent_info->update_times)
|
| 2508 | inode_set_mtime_to_ts(&inode->vfs_inode,
|
| 2509 | inode_set_ctime_current(&inode->vfs_inode));
|
| 2510 |
|
| 2511 | ret = btrfs_update_inode(trans, inode);
|
| 2512 | if (unlikely(ret)) {
|
| 2513 | btrfs_abort_transaction(trans, ret);
|
| 2514 | break;
|
| 2515 | }
|
| 2516 |
|
| 2517 | btrfs_end_transaction(trans);
|
| 2518 | btrfs_btree_balance_dirty(fs_info);
|
| 2519 |
|
| 2520 | trans = btrfs_start_transaction(root, rsv_count);
|
| 2521 | if (IS_ERR(trans)) {
|
| 2522 | ret = PTR_ERR(trans);
|
| 2523 | trans = NULL;
|
| 2524 | break;
|
| 2525 | }
|
| 2526 |
|
| 2527 | ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv,
|
| 2528 | &rsv, min_size, false);
|
| 2529 | if (WARN_ON(ret))
|
| 2530 | break;
|
| 2531 | trans->block_rsv = &rsv;
|
| 2532 |
|
| 2533 | cur_offset = drop_args.drop_end;
|
| 2534 | len = end - cur_offset;
|
| 2535 | if (!extent_info && len) {
|
| 2536 | ret = find_first_non_hole(inode, &cur_offset, &len);
|
| 2537 | if (unlikely(ret < 0))
|
| 2538 | break;
|
| 2539 | if (ret && !len) {
|
| 2540 | ret = 0;
|
| 2541 | break;
|
| 2542 | }
|
| 2543 | }
|
| 2544 | }
|
| 2545 |
|
| 2546 | |
| 2547 | * If we were cloning, force the next fsync to be a full one since we
|
| 2548 | * we replaced (or just dropped in the case of cloning holes when
|
| 2549 | * NO_HOLES is enabled) file extent items and did not setup new extent
|
| 2550 | * maps for the replacement extents (or holes).
|
| 2551 |
|
| 2552 | if (extent_info && !extent_info->is_new_extent)
|
| 2553 | btrfs_set_inode_full_sync(inode);
|
| 2554 |
|
| 2555 | if (ret)
|
| 2556 | goto out_trans;
|
| 2557 |
|
| 2558 | trans->block_rsv = &fs_info->trans_block_rsv;
|
| 2559 | |
| 2560 | * If we are using the NO_HOLES feature we might have had already an
|
| 2561 | * hole that overlaps a part of the region [lockstart, lockend] and
|
| 2562 | * ends at (or beyond) lockend. Since we have no file extent items to
|
| 2563 | * represent holes, drop_end can be less than lockend and so we must
|
| 2564 | * make sure we have an extent map representing the existing hole (the
|
| 2565 | * call to __btrfs_drop_extents() might have dropped the existing extent
|
| 2566 | * map representing the existing hole), otherwise the fast fsync path
|
| 2567 | * will not record the existence of the hole region
|
| 2568 | * [existing_hole_start, lockend].
|
| 2569 |
|
| 2570 | if (drop_args.drop_end <= end)
|
| 2571 | drop_args.drop_end = end + 1;
|
| 2572 | |
| 2573 | * Don't insert file hole extent item if it's for a range beyond eof
|
| 2574 | * (because it's useless) or if it represents a 0 bytes range (when
|
| 2575 | * cur_offset == drop_end).
|
| 2576 |
|
| 2577 | if (!extent_info && cur_offset < ino_size &&
|
| 2578 | cur_offset < drop_args.drop_end) {
|
| 2579 | ret = fill_holes(trans, inode, path, cur_offset,
|
| 2580 | drop_args.drop_end);
|
| 2581 | if (unlikely(ret)) {
|
| 2582 |
|
| 2583 | btrfs_abort_transaction(trans, ret);
|
| 2584 | goto out_trans;
|
| 2585 | }
|
| 2586 | } else if (!extent_info && cur_offset < drop_args.drop_end) {
|
| 2587 |
|
| 2588 | ret = btrfs_inode_clear_file_extent_range(inode, cur_offset,
|
| 2589 | drop_args.drop_end - cur_offset);
|
| 2590 | if (unlikely(ret)) {
|
| 2591 | btrfs_abort_transaction(trans, ret);
|
| 2592 | goto out_trans;
|
| 2593 | }
|
| 2594 |
|
| 2595 | }
|
| 2596 | if (extent_info) {
|
| 2597 | ret = btrfs_insert_replace_extent(trans, inode, path,
|
| 2598 | extent_info, extent_info->data_len,
|
| 2599 | drop_args.bytes_found);
|
| 2600 | if (unlikely(ret)) {
|
| 2601 | btrfs_abort_transaction(trans, ret);
|
| 2602 | goto out_trans;
|
| 2603 | }
|
| 2604 | }
|
| 2605 |
|
| 2606 | out_trans:
|
| 2607 | if (!trans)
|
| 2608 | goto out_release;
|
| 2609 |
|
| 2610 | trans->block_rsv = &fs_info->trans_block_rsv;
|
| 2611 | if (ret)
|
| 2612 | btrfs_end_transaction(trans);
|
| 2613 | else
|
| 2614 | *trans_out = trans;
|
| 2615 | out_release:
|
| 2616 | btrfs_block_rsv_release(fs_info, &rsv, (u64)-1, NULL);
|
| 2617 | return ret;
|
| 2618 | }
|
| 2619 |
|
| 2620 | static int btrfs_punch_hole(struct file *file, loff_t offset, loff_t len)
|
| 2621 | {
|
| 2622 | struct inode *inode = file_inode(file);
|
| 2623 | struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
|
| 2624 | struct btrfs_root *root = BTRFS_I(inode)->root;
|
| 2625 | struct extent_state *cached_state = NULL;
|
| 2626 | struct btrfs_path *path;
|
| 2627 | struct btrfs_trans_handle *trans = NULL;
|
| 2628 | u64 lockstart;
|
| 2629 | u64 lockend;
|
| 2630 | u64 tail_start;
|
| 2631 | u64 tail_len;
|
| 2632 | const u64 orig_start = offset;
|
| 2633 | const u64 orig_end = offset + len - 1;
|
| 2634 | int ret = 0;
|
| 2635 | bool same_block;
|
| 2636 | u64 ino_size;
|
| 2637 | bool truncated_block = false;
|
| 2638 | bool updated_inode = false;
|
| 2639 |
|
| 2640 | btrfs_inode_lock(BTRFS_I(inode), BTRFS_ILOCK_MMAP);
|
| 2641 |
|
| 2642 | ret = btrfs_wait_ordered_range(BTRFS_I(inode), offset, len);
|
| 2643 | if (ret)
|
| 2644 | goto out_only_mutex;
|
| 2645 |
|
| 2646 | ino_size = round_up(inode->i_size, fs_info->sectorsize);
|
| 2647 | ret = find_first_non_hole(BTRFS_I(inode), &offset, &len);
|
| 2648 | if (ret < 0)
|
| 2649 | goto out_only_mutex;
|
| 2650 | if (ret && !len) {
|
| 2651 |
|
| 2652 | ret = 0;
|
| 2653 | goto out_only_mutex;
|
| 2654 | }
|
| 2655 |
|
| 2656 | ret = file_modified(file);
|
| 2657 | if (ret)
|
| 2658 | goto out_only_mutex;
|
| 2659 |
|
| 2660 | lockstart = round_up(offset, fs_info->sectorsize);
|
| 2661 | lockend = round_down(offset + len, fs_info->sectorsize) - 1;
|
| 2662 | same_block = (offset >> fs_info->sectorsize_bits) ==
|
| 2663 | ((offset + len - 1) >> fs_info->sectorsize_bits);
|
| 2664 | |
| 2665 | * Only do this if we are in the same block and we aren't doing the
|
| 2666 | * entire block.
|
| 2667 |
|
| 2668 | if (same_block && len < fs_info->sectorsize) {
|
| 2669 | if (offset < ino_size) {
|
| 2670 | truncated_block = true;
|
| 2671 | ret = btrfs_truncate_block(BTRFS_I(inode), offset + len - 1,
|
| 2672 | orig_start, orig_end);
|
| 2673 | } else {
|
| 2674 | ret = 0;
|
| 2675 | }
|
| 2676 | goto out_only_mutex;
|
| 2677 | }
|
| 2678 |
|
| 2679 |
|
| 2680 | if (offset < ino_size) {
|
| 2681 | truncated_block = true;
|
| 2682 | ret = btrfs_truncate_block(BTRFS_I(inode), offset, orig_start, orig_end);
|
| 2683 | if (ret) {
|
| 2684 | btrfs_inode_unlock(BTRFS_I(inode), BTRFS_ILOCK_MMAP);
|
| 2685 | return ret;
|
| 2686 | }
|
| 2687 | }
|
| 2688 |
|
| 2689 | |
| 2690 | * if offset != orig_start, which means the first unaligned page
|
| 2691 | * including several following pages are already in holes,
|
| 2692 | * the extra check can be skipped
|
| 2693 | if (offset == orig_start) {
|
| 2694 |
|
| 2695 | len = offset + len - lockstart;
|
| 2696 | offset = lockstart;
|
| 2697 | ret = find_first_non_hole(BTRFS_I(inode), &offset, &len);
|
| 2698 | if (ret < 0)
|
| 2699 | goto out_only_mutex;
|
| 2700 | if (ret && !len) {
|
| 2701 | ret = 0;
|
| 2702 | goto out_only_mutex;
|
| 2703 | }
|
| 2704 | lockstart = offset;
|
| 2705 | }
|
| 2706 |
|
| 2707 |
|
| 2708 | tail_start = lockend + 1;
|
| 2709 | tail_len = offset + len - tail_start;
|
| 2710 | if (tail_len) {
|
| 2711 | ret = find_first_non_hole(BTRFS_I(inode), &tail_start, &tail_len);
|
| 2712 | if (unlikely(ret < 0))
|
| 2713 | goto out_only_mutex;
|
| 2714 | if (!ret) {
|
| 2715 |
|
| 2716 | if (tail_start + tail_len < ino_size) {
|
| 2717 | truncated_block = true;
|
| 2718 | ret = btrfs_truncate_block(BTRFS_I(inode),
|
| 2719 | tail_start + tail_len - 1,
|
| 2720 | orig_start, orig_end);
|
| 2721 | if (ret)
|
| 2722 | goto out_only_mutex;
|
| 2723 | }
|
| 2724 | }
|
| 2725 | }
|
| 2726 |
|
| 2727 | if (lockend < lockstart) {
|
| 2728 | ret = 0;
|
| 2729 | goto out_only_mutex;
|
| 2730 | }
|
| 2731 |
|
| 2732 | btrfs_punch_hole_lock_range(inode, lockstart, lockend, &cached_state);
|
| 2733 |
|
| 2734 | path = btrfs_alloc_path();
|
| 2735 | if (!path) {
|
| 2736 | ret = -ENOMEM;
|
| 2737 | goto out;
|
| 2738 | }
|
| 2739 |
|
| 2740 | ret = btrfs_replace_file_extents(BTRFS_I(inode), path, lockstart,
|
| 2741 | lockend, NULL, &trans);
|
| 2742 | btrfs_free_path(path);
|
| 2743 | if (ret)
|
| 2744 | goto out;
|
| 2745 |
|
| 2746 | ASSERT(trans != NULL);
|
| 2747 | inode_inc_iversion(inode);
|
| 2748 | inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode));
|
| 2749 | ret = btrfs_update_inode(trans, BTRFS_I(inode));
|
| 2750 | updated_inode = true;
|
| 2751 | btrfs_end_transaction(trans);
|
| 2752 | btrfs_btree_balance_dirty(fs_info);
|
| 2753 | out:
|
| 2754 | btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend,
|
| 2755 | &cached_state);
|
| 2756 | out_only_mutex:
|
| 2757 | if (!updated_inode && truncated_block && !ret) {
|
| 2758 | |
| 2759 | * If we only end up zeroing part of a page, we still need to
|
| 2760 | * update the inode item, so that all the time fields are
|
| 2761 | * updated as well as the necessary btrfs inode in memory fields
|
| 2762 | * for detecting, at fsync time, if the inode isn't yet in the
|
| 2763 | * log tree or it's there but not up to date.
|
| 2764 |
|
| 2765 | struct timespec64 now = inode_set_ctime_current(inode);
|
| 2766 |
|
| 2767 | inode_inc_iversion(inode);
|
| 2768 | inode_set_mtime_to_ts(inode, now);
|
| 2769 | trans = btrfs_start_transaction(root, 1);
|
| 2770 | if (IS_ERR(trans)) {
|
| 2771 | ret = PTR_ERR(trans);
|
| 2772 | } else {
|
| 2773 | int ret2;
|
| 2774 |
|
| 2775 | ret = btrfs_update_inode(trans, BTRFS_I(inode));
|
| 2776 | ret2 = btrfs_end_transaction(trans);
|
| 2777 | if (!ret)
|
| 2778 | ret = ret2;
|
| 2779 | }
|
| 2780 | }
|
| 2781 | btrfs_inode_unlock(BTRFS_I(inode), BTRFS_ILOCK_MMAP);
|
| 2782 | return ret;
|
| 2783 | }
|
| 2784 |
|
| 2785 |
|
| 2786 | struct falloc_range {
|
| 2787 | struct list_head list;
|
| 2788 | u64 start;
|
| 2789 | u64 len;
|
| 2790 | };
|
| 2791 |
|
| 2792 | |
| 2793 | * Helper function to add falloc range
|
| 2794 | *
|
| 2795 | * Caller should have locked the larger range of extent containing
|
| 2796 | * [start, len)
|
| 2797 |
|
| 2798 | static int add_falloc_range(struct list_head *head, u64 start, u64 len)
|
| 2799 | {
|
| 2800 | struct falloc_range *range = NULL;
|
| 2801 |
|
| 2802 | if (!list_empty(head)) {
|
| 2803 | |
| 2804 | * As fallocate iterates by bytenr order, we only need to check
|
| 2805 | * the last range.
|
| 2806 |
|
| 2807 | range = list_last_entry(head, struct falloc_range, list);
|
| 2808 | if (range->start + range->len == start) {
|
| 2809 | range->len += len;
|
| 2810 | return 0;
|
| 2811 | }
|
| 2812 | }
|
| 2813 |
|
| 2814 | range = kmalloc_obj(*range);
|
| 2815 | if (!range)
|
| 2816 | return -ENOMEM;
|
| 2817 | range->start = start;
|
| 2818 | range->len = len;
|
| 2819 | list_add_tail(&range->list, head);
|
| 2820 | return 0;
|
| 2821 | }
|
| 2822 |
|
| 2823 | static int btrfs_fallocate_update_isize(struct inode *inode,
|
| 2824 | const u64 end,
|
| 2825 | const int mode)
|
| 2826 | {
|
| 2827 | struct btrfs_trans_handle *trans;
|
| 2828 | struct btrfs_root *root = BTRFS_I(inode)->root;
|
| 2829 | u64 range_start;
|
| 2830 | u64 range_end;
|
| 2831 | int ret;
|
| 2832 | int ret2;
|
| 2833 |
|
| 2834 | if (mode & FALLOC_FL_KEEP_SIZE || end <= i_size_read(inode))
|
| 2835 | return 0;
|
| 2836 |
|
| 2837 | range_start = round_down(i_size_read(inode), root->fs_info->sectorsize);
|
| 2838 | range_end = round_up(end, root->fs_info->sectorsize);
|
| 2839 |
|
| 2840 | ret = btrfs_inode_set_file_extent_range(BTRFS_I(inode), range_start,
|
| 2841 | range_end - range_start);
|
| 2842 | if (ret)
|
| 2843 | return ret;
|
| 2844 |
|
| 2845 | trans = btrfs_start_transaction(root, 1);
|
| 2846 | if (IS_ERR(trans))
|
| 2847 | return PTR_ERR(trans);
|
| 2848 |
|
| 2849 | inode_set_ctime_current(inode);
|
| 2850 | i_size_write(inode, end);
|
| 2851 | btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0);
|
| 2852 | ret = btrfs_update_inode(trans, BTRFS_I(inode));
|
| 2853 | ret2 = btrfs_end_transaction(trans);
|
| 2854 |
|
| 2855 | return ret ? ret : ret2;
|
| 2856 | }
|
| 2857 |
|
| 2858 | enum {
|
| 2859 | RANGE_BOUNDARY_WRITTEN_EXTENT,
|
| 2860 | RANGE_BOUNDARY_PREALLOC_EXTENT,
|
| 2861 | RANGE_BOUNDARY_HOLE,
|
| 2862 | };
|
| 2863 |
|
| 2864 | static int btrfs_zero_range_check_range_boundary(struct btrfs_inode *inode,
|
| 2865 | u64 offset)
|
| 2866 | {
|
| 2867 | const u32 sectorsize = inode->root->fs_info->sectorsize;
|
| 2868 | struct extent_map *em;
|
| 2869 | int ret;
|
| 2870 |
|
| 2871 | offset = round_down(offset, sectorsize);
|
| 2872 | em = btrfs_get_extent(inode, NULL, offset, sectorsize);
|
| 2873 | if (IS_ERR(em))
|
| 2874 | return PTR_ERR(em);
|
| 2875 |
|
| 2876 | if (em->disk_bytenr == EXTENT_MAP_HOLE)
|
| 2877 | ret = RANGE_BOUNDARY_HOLE;
|
| 2878 | else if (em->flags & EXTENT_FLAG_PREALLOC)
|
| 2879 | ret = RANGE_BOUNDARY_PREALLOC_EXTENT;
|
| 2880 | else
|
| 2881 | ret = RANGE_BOUNDARY_WRITTEN_EXTENT;
|
| 2882 |
|
| 2883 | btrfs_free_extent_map(em);
|
| 2884 | return ret;
|
| 2885 | }
|
| 2886 |
|
| 2887 | static int btrfs_zero_range(struct inode *inode,
|
| 2888 | loff_t offset,
|
| 2889 | loff_t len,
|
| 2890 | const int mode)
|
| 2891 | {
|
| 2892 | struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
|
| 2893 | struct extent_map *em;
|
| 2894 | struct extent_changeset *data_reserved = NULL;
|
| 2895 | int ret;
|
| 2896 | u64 alloc_hint = 0;
|
| 2897 | const u32 sectorsize = fs_info->sectorsize;
|
| 2898 | const u64 orig_start = offset;
|
| 2899 | const u64 orig_end = offset + len - 1;
|
| 2900 | u64 alloc_start = round_down(offset, sectorsize);
|
| 2901 | u64 alloc_end = round_up(offset + len, sectorsize);
|
| 2902 | u64 bytes_to_reserve = 0;
|
| 2903 | bool space_reserved = false;
|
| 2904 |
|
| 2905 | em = btrfs_get_extent(BTRFS_I(inode), NULL, alloc_start,
|
| 2906 | alloc_end - alloc_start);
|
| 2907 | if (IS_ERR(em)) {
|
| 2908 | ret = PTR_ERR(em);
|
| 2909 | goto out;
|
| 2910 | }
|
| 2911 |
|
| 2912 | |
| 2913 | * Avoid hole punching and extent allocation for some cases. More cases
|
| 2914 | * could be considered, but these are unlikely common and we keep things
|
| 2915 | * as simple as possible for now. Also, intentionally, if the target
|
| 2916 | * range contains one or more prealloc extents together with regular
|
| 2917 | * extents and holes, we drop all the existing extents and allocate a
|
| 2918 | * new prealloc extent, so that we get a larger contiguous disk extent.
|
| 2919 |
|
| 2920 | if (em->start <= alloc_start && (em->flags & EXTENT_FLAG_PREALLOC)) {
|
| 2921 | const u64 em_end = btrfs_extent_map_end(em);
|
| 2922 |
|
| 2923 | if (em_end >= offset + len) {
|
| 2924 | |
| 2925 | * The whole range is already a prealloc extent,
|
| 2926 | * do nothing except updating the inode's i_size if
|
| 2927 | * needed.
|
| 2928 |
|
| 2929 | btrfs_free_extent_map(em);
|
| 2930 | ret = btrfs_fallocate_update_isize(inode, offset + len,
|
| 2931 | mode);
|
| 2932 | goto out;
|
| 2933 | }
|
| 2934 | |
| 2935 | * Part of the range is already a prealloc extent, so operate
|
| 2936 | * only on the remaining part of the range.
|
| 2937 |
|
| 2938 | alloc_start = em_end;
|
| 2939 | ASSERT(IS_ALIGNED(alloc_start, sectorsize));
|
| 2940 | len = offset + len - alloc_start;
|
| 2941 | offset = alloc_start;
|
| 2942 | alloc_hint = btrfs_extent_map_block_start(em) + em->len;
|
| 2943 | }
|
| 2944 | btrfs_free_extent_map(em);
|
| 2945 |
|
| 2946 | if ((offset >> fs_info->sectorsize_bits) ==
|
| 2947 | ((offset + len - 1) >> fs_info->sectorsize_bits)) {
|
| 2948 | em = btrfs_get_extent(BTRFS_I(inode), NULL, alloc_start, sectorsize);
|
| 2949 | if (IS_ERR(em)) {
|
| 2950 | ret = PTR_ERR(em);
|
| 2951 | goto out;
|
| 2952 | }
|
| 2953 |
|
| 2954 | if (em->flags & EXTENT_FLAG_PREALLOC) {
|
| 2955 | btrfs_free_extent_map(em);
|
| 2956 | ret = btrfs_fallocate_update_isize(inode, offset + len,
|
| 2957 | mode);
|
| 2958 | goto out;
|
| 2959 | }
|
| 2960 | if (len < sectorsize && em->disk_bytenr != EXTENT_MAP_HOLE) {
|
| 2961 | btrfs_free_extent_map(em);
|
| 2962 | ret = btrfs_truncate_block(BTRFS_I(inode), offset + len - 1,
|
| 2963 | orig_start, orig_end);
|
| 2964 | if (!ret)
|
| 2965 | ret = btrfs_fallocate_update_isize(inode,
|
| 2966 | offset + len,
|
| 2967 | mode);
|
| 2968 | return ret;
|
| 2969 | }
|
| 2970 | btrfs_free_extent_map(em);
|
| 2971 | alloc_start = round_down(offset, sectorsize);
|
| 2972 | alloc_end = alloc_start + sectorsize;
|
| 2973 | goto reserve_space;
|
| 2974 | }
|
| 2975 |
|
| 2976 | alloc_start = round_up(offset, sectorsize);
|
| 2977 | alloc_end = round_down(offset + len, sectorsize);
|
| 2978 |
|
| 2979 | |
| 2980 | * For unaligned ranges, check the pages at the boundaries, they might
|
| 2981 | * map to an extent, in which case we need to partially zero them, or
|
| 2982 | * they might map to a hole, in which case we need our allocation range
|
| 2983 | * to cover them.
|
| 2984 |
|
| 2985 | if (!IS_ALIGNED(offset, sectorsize)) {
|
| 2986 | ret = btrfs_zero_range_check_range_boundary(BTRFS_I(inode),
|
| 2987 | offset);
|
| 2988 | if (ret < 0)
|
| 2989 | goto out;
|
| 2990 | if (ret == RANGE_BOUNDARY_HOLE) {
|
| 2991 | alloc_start = round_down(offset, sectorsize);
|
| 2992 | ret = 0;
|
| 2993 | } else if (ret == RANGE_BOUNDARY_WRITTEN_EXTENT) {
|
| 2994 | ret = btrfs_truncate_block(BTRFS_I(inode), offset,
|
| 2995 | orig_start, orig_end);
|
| 2996 | if (ret)
|
| 2997 | goto out;
|
| 2998 | } else {
|
| 2999 | ret = 0;
|
| 3000 | }
|
| 3001 | }
|
| 3002 |
|
| 3003 | if (!IS_ALIGNED(offset + len, sectorsize)) {
|
| 3004 | ret = btrfs_zero_range_check_range_boundary(BTRFS_I(inode),
|
| 3005 | offset + len);
|
| 3006 | if (ret < 0)
|
| 3007 | goto out;
|
| 3008 | if (ret == RANGE_BOUNDARY_HOLE) {
|
| 3009 | alloc_end = round_up(offset + len, sectorsize);
|
| 3010 | ret = 0;
|
| 3011 | } else if (ret == RANGE_BOUNDARY_WRITTEN_EXTENT) {
|
| 3012 | ret = btrfs_truncate_block(BTRFS_I(inode), offset + len - 1,
|
| 3013 | orig_start, orig_end);
|
| 3014 | if (ret)
|
| 3015 | goto out;
|
| 3016 | } else {
|
| 3017 | ret = 0;
|
| 3018 | }
|
| 3019 | }
|
| 3020 |
|
| 3021 | reserve_space:
|
| 3022 | if (alloc_start < alloc_end) {
|
| 3023 | struct extent_state *cached_state = NULL;
|
| 3024 | const u64 lockstart = alloc_start;
|
| 3025 | const u64 lockend = alloc_end - 1;
|
| 3026 |
|
| 3027 | bytes_to_reserve = alloc_end - alloc_start;
|
| 3028 | ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode),
|
| 3029 | bytes_to_reserve);
|
| 3030 | if (ret < 0)
|
| 3031 | goto out;
|
| 3032 | space_reserved = true;
|
| 3033 | btrfs_punch_hole_lock_range(inode, lockstart, lockend,
|
| 3034 | &cached_state);
|
| 3035 | ret = btrfs_qgroup_reserve_data(BTRFS_I(inode), &data_reserved,
|
| 3036 | alloc_start, bytes_to_reserve);
|
| 3037 | if (ret) {
|
| 3038 | btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, lockstart,
|
| 3039 | lockend, &cached_state);
|
| 3040 | goto out;
|
| 3041 | }
|
| 3042 | ret = btrfs_prealloc_file_range(inode, mode, alloc_start,
|
| 3043 | alloc_end - alloc_start,
|
| 3044 | fs_info->sectorsize,
|
| 3045 | offset + len, &alloc_hint);
|
| 3046 | btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend,
|
| 3047 | &cached_state);
|
| 3048 |
|
| 3049 | if (ret) {
|
| 3050 | space_reserved = false;
|
| 3051 | goto out;
|
| 3052 | }
|
| 3053 | }
|
| 3054 | ret = btrfs_fallocate_update_isize(inode, offset + len, mode);
|
| 3055 | out:
|
| 3056 | if (ret && space_reserved)
|
| 3057 | btrfs_free_reserved_data_space(BTRFS_I(inode), data_reserved,
|
| 3058 | alloc_start, bytes_to_reserve);
|
| 3059 | extent_changeset_free(data_reserved);
|
| 3060 |
|
| 3061 | return ret;
|
| 3062 | }
|
| 3063 |
|
| 3064 | static long btrfs_fallocate(struct file *file, int mode,
|
| 3065 | loff_t offset, loff_t len)
|
| 3066 | {
|
| 3067 | struct inode *inode = file_inode(file);
|
| 3068 | struct extent_state *cached_state = NULL;
|
| 3069 | struct extent_changeset *data_reserved = NULL;
|
| 3070 | struct falloc_range *range;
|
| 3071 | struct falloc_range *tmp;
|
| 3072 | LIST_HEAD(reserve_list);
|
| 3073 | u64 cur_offset;
|
| 3074 | u64 last_byte;
|
| 3075 | u64 alloc_start;
|
| 3076 | u64 alloc_end;
|
| 3077 | u64 alloc_hint = 0;
|
| 3078 | u64 locked_end;
|
| 3079 | u64 actual_end = 0;
|
| 3080 | u64 data_space_needed = 0;
|
| 3081 | u64 data_space_reserved = 0;
|
| 3082 | u64 qgroup_reserved = 0;
|
| 3083 | struct extent_map *em;
|
| 3084 | int blocksize = BTRFS_I(inode)->root->fs_info->sectorsize;
|
| 3085 | int ret;
|
| 3086 |
|
| 3087 | if (btrfs_is_shutdown(inode_to_fs_info(inode)))
|
| 3088 | return -EIO;
|
| 3089 |
|
| 3090 |
|
| 3091 | if (btrfs_is_zoned(inode_to_fs_info(inode)))
|
| 3092 | return -EOPNOTSUPP;
|
| 3093 |
|
| 3094 | alloc_start = round_down(offset, blocksize);
|
| 3095 | alloc_end = round_up(offset + len, blocksize);
|
| 3096 | cur_offset = alloc_start;
|
| 3097 |
|
| 3098 |
|
| 3099 | if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |
|
| 3100 | FALLOC_FL_ZERO_RANGE))
|
| 3101 | return -EOPNOTSUPP;
|
| 3102 |
|
| 3103 | if (mode & FALLOC_FL_PUNCH_HOLE)
|
| 3104 | return btrfs_punch_hole(file, offset, len);
|
| 3105 |
|
| 3106 | btrfs_inode_lock(BTRFS_I(inode), BTRFS_ILOCK_MMAP);
|
| 3107 |
|
| 3108 | if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) {
|
| 3109 | ret = inode_newsize_ok(inode, offset + len);
|
| 3110 | if (ret)
|
| 3111 | goto out;
|
| 3112 | }
|
| 3113 |
|
| 3114 | ret = file_modified(file);
|
| 3115 | if (ret)
|
| 3116 | goto out;
|
| 3117 |
|
| 3118 | |
| 3119 | * TODO: Move these two operations after we have checked
|
| 3120 | * accurate reserved space, or fallocate can still fail but
|
| 3121 | * with page truncated or size expanded.
|
| 3122 | *
|
| 3123 | * But that's a minor problem and won't do much harm BTW.
|
| 3124 |
|
| 3125 | if (alloc_start > inode->i_size) {
|
| 3126 | ret = btrfs_cont_expand(BTRFS_I(inode), i_size_read(inode),
|
| 3127 | alloc_start);
|
| 3128 | if (ret)
|
| 3129 | goto out;
|
| 3130 | } else if (offset + len > inode->i_size) {
|
| 3131 | |
| 3132 | * If we are fallocating from the end of the file onward we
|
| 3133 | * need to zero out the end of the block if i_size lands in the
|
| 3134 | * middle of a block.
|
| 3135 |
|
| 3136 | ret = btrfs_truncate_block(BTRFS_I(inode), inode->i_size,
|
| 3137 | inode->i_size, (u64)-1);
|
| 3138 | if (ret)
|
| 3139 | goto out;
|
| 3140 | }
|
| 3141 |
|
| 3142 | |
| 3143 | * We have locked the inode at the VFS level (in exclusive mode) and we
|
| 3144 | * have locked the i_mmap_lock lock (in exclusive mode). Now before
|
| 3145 | * locking the file range, flush all dealloc in the range and wait for
|
| 3146 | * all ordered extents in the range to complete. After this we can lock
|
| 3147 | * the file range and, due to the previous locking we did, we know there
|
| 3148 | * can't be more delalloc or ordered extents in the range.
|
| 3149 |
|
| 3150 | ret = btrfs_wait_ordered_range(BTRFS_I(inode), alloc_start,
|
| 3151 | alloc_end - alloc_start);
|
| 3152 | if (ret)
|
| 3153 | goto out;
|
| 3154 |
|
| 3155 | if (mode & FALLOC_FL_ZERO_RANGE) {
|
| 3156 | ret = btrfs_zero_range(inode, offset, len, mode);
|
| 3157 | btrfs_inode_unlock(BTRFS_I(inode), BTRFS_ILOCK_MMAP);
|
| 3158 | return ret;
|
| 3159 | }
|
| 3160 |
|
| 3161 | locked_end = alloc_end - 1;
|
| 3162 | btrfs_lock_extent(&BTRFS_I(inode)->io_tree, alloc_start, locked_end,
|
| 3163 | &cached_state);
|
| 3164 |
|
| 3165 | btrfs_assert_inode_range_clean(BTRFS_I(inode), alloc_start, locked_end);
|
| 3166 |
|
| 3167 |
|
| 3168 | while (cur_offset < alloc_end) {
|
| 3169 | em = btrfs_get_extent(BTRFS_I(inode), NULL, cur_offset,
|
| 3170 | alloc_end - cur_offset);
|
| 3171 | if (IS_ERR(em)) {
|
| 3172 | ret = PTR_ERR(em);
|
| 3173 | break;
|
| 3174 | }
|
| 3175 | last_byte = min(btrfs_extent_map_end(em), alloc_end);
|
| 3176 | actual_end = min_t(u64, btrfs_extent_map_end(em), offset + len);
|
| 3177 | last_byte = ALIGN(last_byte, blocksize);
|
| 3178 | if (em->disk_bytenr == EXTENT_MAP_HOLE ||
|
| 3179 | (cur_offset >= inode->i_size &&
|
| 3180 | !(em->flags & EXTENT_FLAG_PREALLOC))) {
|
| 3181 | const u64 range_len = last_byte - cur_offset;
|
| 3182 |
|
| 3183 | ret = add_falloc_range(&reserve_list, cur_offset, range_len);
|
| 3184 | if (ret < 0) {
|
| 3185 | btrfs_free_extent_map(em);
|
| 3186 | break;
|
| 3187 | }
|
| 3188 | ret = btrfs_qgroup_reserve_data(BTRFS_I(inode),
|
| 3189 | &data_reserved, cur_offset, range_len);
|
| 3190 | if (ret < 0) {
|
| 3191 | btrfs_free_extent_map(em);
|
| 3192 | break;
|
| 3193 | }
|
| 3194 | qgroup_reserved += range_len;
|
| 3195 | data_space_needed += range_len;
|
| 3196 | }
|
| 3197 | btrfs_free_extent_map(em);
|
| 3198 | cur_offset = last_byte;
|
| 3199 | }
|
| 3200 |
|
| 3201 | if (!ret && data_space_needed > 0) {
|
| 3202 | |
| 3203 | * We are safe to reserve space here as we can't have delalloc
|
| 3204 | * in the range, see above.
|
| 3205 |
|
| 3206 | ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode),
|
| 3207 | data_space_needed);
|
| 3208 | if (!ret)
|
| 3209 | data_space_reserved = data_space_needed;
|
| 3210 | }
|
| 3211 |
|
| 3212 | |
| 3213 | * If ret is still 0, means we're OK to fallocate.
|
| 3214 | * Or just cleanup the list and exit.
|
| 3215 |
|
| 3216 | list_for_each_entry_safe(range, tmp, &reserve_list, list) {
|
| 3217 | if (!ret) {
|
| 3218 | ret = btrfs_prealloc_file_range(inode, mode,
|
| 3219 | range->start,
|
| 3220 | range->len, blocksize,
|
| 3221 | offset + len, &alloc_hint);
|
| 3222 | |
| 3223 | * btrfs_prealloc_file_range() releases space even
|
| 3224 | * if it returns an error.
|
| 3225 |
|
| 3226 | data_space_reserved -= range->len;
|
| 3227 | qgroup_reserved -= range->len;
|
| 3228 | } else if (data_space_reserved > 0) {
|
| 3229 | btrfs_free_reserved_data_space(BTRFS_I(inode),
|
| 3230 | data_reserved, range->start,
|
| 3231 | range->len);
|
| 3232 | data_space_reserved -= range->len;
|
| 3233 | qgroup_reserved -= range->len;
|
| 3234 | } else if (qgroup_reserved > 0) {
|
| 3235 | btrfs_qgroup_free_data(BTRFS_I(inode), data_reserved,
|
| 3236 | range->start, range->len, NULL);
|
| 3237 | qgroup_reserved -= range->len;
|
| 3238 | }
|
| 3239 | list_del(&range->list);
|
| 3240 | kfree(range);
|
| 3241 | }
|
| 3242 | if (ret < 0)
|
| 3243 | goto out_unlock;
|
| 3244 |
|
| 3245 | |
| 3246 | * We didn't need to allocate any more space, but we still extended the
|
| 3247 | * size of the file so we need to update i_size and the inode item.
|
| 3248 |
|
| 3249 | ret = btrfs_fallocate_update_isize(inode, actual_end, mode);
|
| 3250 | out_unlock:
|
| 3251 | btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, alloc_start, locked_end,
|
| 3252 | &cached_state);
|
| 3253 | out:
|
| 3254 | btrfs_inode_unlock(BTRFS_I(inode), BTRFS_ILOCK_MMAP);
|
| 3255 | extent_changeset_free(data_reserved);
|
| 3256 | return ret;
|
| 3257 | }
|
| 3258 |
|
| 3259 | |
| 3260 | * Helper for btrfs_find_delalloc_in_range(). Find a subrange in a given range
|
| 3261 | * that has unflushed and/or flushing delalloc. There might be other adjacent
|
| 3262 | * subranges after the one it found, so btrfs_find_delalloc_in_range() keeps
|
| 3263 | * looping while it gets adjacent subranges, and merging them together.
|
| 3264 |
|
| 3265 | static bool find_delalloc_subrange(struct btrfs_inode *inode, u64 start, u64 end,
|
| 3266 | struct extent_state **cached_state,
|
| 3267 | bool *search_io_tree,
|
| 3268 | u64 *delalloc_start_ret, u64 *delalloc_end_ret)
|
| 3269 | {
|
| 3270 | u64 len = end + 1 - start;
|
| 3271 | u64 delalloc_len = 0;
|
| 3272 | struct btrfs_ordered_extent *oe;
|
| 3273 | u64 oe_start;
|
| 3274 | u64 oe_end;
|
| 3275 |
|
| 3276 | |
| 3277 | * Search the io tree first for EXTENT_DELALLOC. If we find any, it
|
| 3278 | * means we have delalloc (dirty pages) for which writeback has not
|
| 3279 | * started yet.
|
| 3280 |
|
| 3281 | if (*search_io_tree) {
|
| 3282 | spin_lock(&inode->lock);
|
| 3283 | if (inode->delalloc_bytes > 0) {
|
| 3284 | spin_unlock(&inode->lock);
|
| 3285 | *delalloc_start_ret = start;
|
| 3286 | delalloc_len = btrfs_count_range_bits(&inode->io_tree,
|
| 3287 | delalloc_start_ret, end,
|
| 3288 | len, EXTENT_DELALLOC,
|
| 3289 | true, cached_state);
|
| 3290 | } else {
|
| 3291 | spin_unlock(&inode->lock);
|
| 3292 | }
|
| 3293 | }
|
| 3294 |
|
| 3295 | if (delalloc_len > 0) {
|
| 3296 | |
| 3297 | * If delalloc was found then *delalloc_start_ret has a sector size
|
| 3298 | * aligned value (rounded down).
|
| 3299 |
|
| 3300 | *delalloc_end_ret = *delalloc_start_ret + delalloc_len - 1;
|
| 3301 |
|
| 3302 | if (*delalloc_start_ret == start) {
|
| 3303 |
|
| 3304 | if (*delalloc_end_ret == end)
|
| 3305 | return true;
|
| 3306 |
|
| 3307 | start = *delalloc_end_ret + 1;
|
| 3308 | len = end + 1 - start;
|
| 3309 | }
|
| 3310 | } else {
|
| 3311 |
|
| 3312 | *search_io_tree = false;
|
| 3313 | }
|
| 3314 |
|
| 3315 | |
| 3316 | * Now also check if there's any ordered extent in the range.
|
| 3317 | * We do this because:
|
| 3318 | *
|
| 3319 | * 1) When delalloc is flushed, the file range is locked, we clear the
|
| 3320 | * EXTENT_DELALLOC bit from the io tree and create an extent map and
|
| 3321 | * an ordered extent for the write. So we might just have been called
|
| 3322 | * after delalloc is flushed and before the ordered extent completes
|
| 3323 | * and inserts the new file extent item in the subvolume's btree;
|
| 3324 | *
|
| 3325 | * 2) We may have an ordered extent created by flushing delalloc for a
|
| 3326 | * subrange that starts before the subrange we found marked with
|
| 3327 | * EXTENT_DELALLOC in the io tree.
|
| 3328 | *
|
| 3329 | * We could also use the extent map tree to find such delalloc that is
|
| 3330 | * being flushed, but using the ordered extents tree is more efficient
|
| 3331 | * because it's usually much smaller as ordered extents are removed from
|
| 3332 | * the tree once they complete. With the extent maps, we may have them
|
| 3333 | * in the extent map tree for a very long time, and they were either
|
| 3334 | * created by previous writes or loaded by read operations.
|
| 3335 |
|
| 3336 | oe = btrfs_lookup_first_ordered_range(inode, start, len);
|
| 3337 | if (!oe)
|
| 3338 | return (delalloc_len > 0);
|
| 3339 |
|
| 3340 |
|
| 3341 | oe_start = max(oe->file_offset, start);
|
| 3342 | oe_end = min(oe->file_offset + oe->num_bytes - 1, end);
|
| 3343 |
|
| 3344 | btrfs_put_ordered_extent(oe);
|
| 3345 |
|
| 3346 |
|
| 3347 | if (delalloc_len == 0) {
|
| 3348 | *delalloc_start_ret = oe_start;
|
| 3349 | *delalloc_end_ret = oe_end;
|
| 3350 | return true;
|
| 3351 | }
|
| 3352 |
|
| 3353 | |
| 3354 | * We have both unflushed delalloc (io_tree) and an ordered extent.
|
| 3355 | * If the ranges are adjacent returned a combined range, otherwise
|
| 3356 | * return the leftmost range.
|
| 3357 |
|
| 3358 | if (oe_start < *delalloc_start_ret) {
|
| 3359 | if (oe_end < *delalloc_start_ret)
|
| 3360 | *delalloc_end_ret = oe_end;
|
| 3361 | *delalloc_start_ret = oe_start;
|
| 3362 | } else if (*delalloc_end_ret + 1 == oe_start) {
|
| 3363 | *delalloc_end_ret = oe_end;
|
| 3364 | }
|
| 3365 |
|
| 3366 | return true;
|
| 3367 | }
|
| 3368 |
|
| 3369 | |
| 3370 | * Check if there's delalloc in a given range.
|
| 3371 | *
|
| 3372 | * @inode: The inode.
|
| 3373 | * @start: The start offset of the range. It does not need to be
|
| 3374 | * sector size aligned.
|
| 3375 | * @end: The end offset (inclusive value) of the search range.
|
| 3376 | * It does not need to be sector size aligned.
|
| 3377 | * @cached_state: Extent state record used for speeding up delalloc
|
| 3378 | * searches in the inode's io_tree. Can be NULL.
|
| 3379 | * @delalloc_start_ret: Output argument, set to the start offset of the
|
| 3380 | * subrange found with delalloc (may not be sector size
|
| 3381 | * aligned).
|
| 3382 | * @delalloc_end_ret: Output argument, set to he end offset (inclusive value)
|
| 3383 | * of the subrange found with delalloc.
|
| 3384 | *
|
| 3385 | * Returns true if a subrange with delalloc is found within the given range, and
|
| 3386 | * if so it sets @delalloc_start_ret and @delalloc_end_ret with the start and
|
| 3387 | * end offsets of the subrange.
|
| 3388 |
|
| 3389 | bool btrfs_find_delalloc_in_range(struct btrfs_inode *inode, u64 start, u64 end,
|
| 3390 | struct extent_state **cached_state,
|
| 3391 | u64 *delalloc_start_ret, u64 *delalloc_end_ret)
|
| 3392 | {
|
| 3393 | u64 cur_offset = round_down(start, inode->root->fs_info->sectorsize);
|
| 3394 | u64 prev_delalloc_end = 0;
|
| 3395 | bool search_io_tree = true;
|
| 3396 | bool ret = false;
|
| 3397 |
|
| 3398 | while (cur_offset <= end) {
|
| 3399 | u64 delalloc_start;
|
| 3400 | u64 delalloc_end;
|
| 3401 | bool delalloc;
|
| 3402 |
|
| 3403 | delalloc = find_delalloc_subrange(inode, cur_offset, end,
|
| 3404 | cached_state, &search_io_tree,
|
| 3405 | &delalloc_start,
|
| 3406 | &delalloc_end);
|
| 3407 | if (!delalloc)
|
| 3408 | break;
|
| 3409 |
|
| 3410 | if (prev_delalloc_end == 0) {
|
| 3411 |
|
| 3412 | *delalloc_start_ret = max(delalloc_start, start);
|
| 3413 | *delalloc_end_ret = delalloc_end;
|
| 3414 | ret = true;
|
| 3415 | } else if (delalloc_start == prev_delalloc_end + 1) {
|
| 3416 |
|
| 3417 | *delalloc_end_ret = delalloc_end;
|
| 3418 | } else {
|
| 3419 |
|
| 3420 | break;
|
| 3421 | }
|
| 3422 |
|
| 3423 | prev_delalloc_end = delalloc_end;
|
| 3424 | cur_offset = delalloc_end + 1;
|
| 3425 | cond_resched();
|
| 3426 | }
|
| 3427 |
|
| 3428 | return ret;
|
| 3429 | }
|
| 3430 |
|
| 3431 | |
| 3432 | * Check if there's a hole or delalloc range in a range representing a hole (or
|
| 3433 | * prealloc extent) found in the inode's subvolume btree.
|
| 3434 | *
|
| 3435 | * @inode: The inode.
|
| 3436 | * @whence: Seek mode (SEEK_DATA or SEEK_HOLE).
|
| 3437 | * @start: Start offset of the hole region. It does not need to be sector
|
| 3438 | * size aligned.
|
| 3439 | * @end: End offset (inclusive value) of the hole region. It does not
|
| 3440 | * need to be sector size aligned.
|
| 3441 | * @start_ret: Return parameter, used to set the start of the subrange in the
|
| 3442 | * hole that matches the search criteria (seek mode), if such
|
| 3443 | * subrange is found (return value of the function is true).
|
| 3444 | * The value returned here may not be sector size aligned.
|
| 3445 | *
|
| 3446 | * Returns true if a subrange matching the given seek mode is found, and if one
|
| 3447 | * is found, it updates @start_ret with the start of the subrange.
|
| 3448 |
|
| 3449 | static bool find_desired_extent_in_hole(struct btrfs_inode *inode, int whence,
|
| 3450 | struct extent_state **cached_state,
|
| 3451 | u64 start, u64 end, u64 *start_ret)
|
| 3452 | {
|
| 3453 | u64 delalloc_start;
|
| 3454 | u64 delalloc_end;
|
| 3455 | bool delalloc;
|
| 3456 |
|
| 3457 | delalloc = btrfs_find_delalloc_in_range(inode, start, end, cached_state,
|
| 3458 | &delalloc_start, &delalloc_end);
|
| 3459 | if (delalloc && whence == SEEK_DATA) {
|
| 3460 | *start_ret = delalloc_start;
|
| 3461 | return true;
|
| 3462 | }
|
| 3463 |
|
| 3464 | if (delalloc && whence == SEEK_HOLE) {
|
| 3465 | |
| 3466 | * We found delalloc but it starts after out start offset. So we
|
| 3467 | * have a hole between our start offset and the delalloc start.
|
| 3468 |
|
| 3469 | if (start < delalloc_start) {
|
| 3470 | *start_ret = start;
|
| 3471 | return true;
|
| 3472 | }
|
| 3473 | |
| 3474 | * Delalloc range starts at our start offset.
|
| 3475 | * If the delalloc range's length is smaller than our range,
|
| 3476 | * then it means we have a hole that starts where the delalloc
|
| 3477 | * subrange ends.
|
| 3478 |
|
| 3479 | if (delalloc_end < end) {
|
| 3480 | *start_ret = delalloc_end + 1;
|
| 3481 | return true;
|
| 3482 | }
|
| 3483 |
|
| 3484 |
|
| 3485 | return false;
|
| 3486 | }
|
| 3487 |
|
| 3488 | if (!delalloc && whence == SEEK_HOLE) {
|
| 3489 | *start_ret = start;
|
| 3490 | return true;
|
| 3491 | }
|
| 3492 |
|
| 3493 | |
| 3494 | * No delalloc in the range and we are seeking for data. The caller has
|
| 3495 | * to iterate to the next extent item in the subvolume btree.
|
| 3496 |
|
| 3497 | return false;
|
| 3498 | }
|
| 3499 |
|
| 3500 | static loff_t find_desired_extent(struct file *file, loff_t offset, int whence)
|
| 3501 | {
|
| 3502 | struct btrfs_inode *inode = BTRFS_I(file->f_mapping->host);
|
| 3503 | struct btrfs_file_private *private;
|
| 3504 | struct btrfs_fs_info *fs_info = inode->root->fs_info;
|
| 3505 | struct extent_state *cached_state = NULL;
|
| 3506 | struct extent_state **delalloc_cached_state;
|
| 3507 | const loff_t i_size = i_size_read(&inode->vfs_inode);
|
| 3508 | const u64 ino = btrfs_ino(inode);
|
| 3509 | struct btrfs_root *root = inode->root;
|
| 3510 | struct btrfs_path *path;
|
| 3511 | struct btrfs_key key;
|
| 3512 | u64 last_extent_end;
|
| 3513 | u64 lockstart;
|
| 3514 | u64 lockend;
|
| 3515 | u64 start;
|
| 3516 | int ret;
|
| 3517 | bool found = false;
|
| 3518 |
|
| 3519 | if (i_size == 0 || offset >= i_size)
|
| 3520 | return -ENXIO;
|
| 3521 |
|
| 3522 | |
| 3523 | * Quick path. If the inode has no prealloc extents and its number of
|
| 3524 | * bytes used matches its i_size, then it can not have holes.
|
| 3525 |
|
| 3526 | if (whence == SEEK_HOLE &&
|
| 3527 | !(inode->flags & BTRFS_INODE_PREALLOC) &&
|
| 3528 | inode_get_bytes(&inode->vfs_inode) == i_size)
|
| 3529 | return i_size;
|
| 3530 |
|
| 3531 | spin_lock(&inode->lock);
|
| 3532 | private = file->private_data;
|
| 3533 | spin_unlock(&inode->lock);
|
| 3534 |
|
| 3535 | if (private && private->owner_task != current) {
|
| 3536 | |
| 3537 | * Not allocated by us, don't use it as its cached state is used
|
| 3538 | * by the task that allocated it and we don't want neither to
|
| 3539 | * mess with it nor get incorrect results because it reflects an
|
| 3540 | * invalid state for the current task.
|
| 3541 |
|
| 3542 | private = NULL;
|
| 3543 | } else if (!private) {
|
| 3544 | private = kzalloc_obj(*private);
|
| 3545 | |
| 3546 | * No worries if memory allocation failed.
|
| 3547 | * The private structure is used only for speeding up multiple
|
| 3548 | * lseek SEEK_HOLE/DATA calls to a file when there's delalloc,
|
| 3549 | * so everything will still be correct.
|
| 3550 |
|
| 3551 | if (private) {
|
| 3552 | bool free = false;
|
| 3553 |
|
| 3554 | private->owner_task = current;
|
| 3555 |
|
| 3556 | spin_lock(&inode->lock);
|
| 3557 | if (file->private_data)
|
| 3558 | free = true;
|
| 3559 | else
|
| 3560 | file->private_data = private;
|
| 3561 | spin_unlock(&inode->lock);
|
| 3562 |
|
| 3563 | if (free) {
|
| 3564 | kfree(private);
|
| 3565 | private = NULL;
|
| 3566 | }
|
| 3567 | }
|
| 3568 | }
|
| 3569 |
|
| 3570 | if (private)
|
| 3571 | delalloc_cached_state = &private->llseek_cached_state;
|
| 3572 | else
|
| 3573 | delalloc_cached_state = NULL;
|
| 3574 |
|
| 3575 | |
| 3576 | * offset can be negative, in this case we start finding DATA/HOLE from
|
| 3577 | * the very start of the file.
|
| 3578 |
|
| 3579 | start = max_t(loff_t, 0, offset);
|
| 3580 |
|
| 3581 | lockstart = round_down(start, fs_info->sectorsize);
|
| 3582 | lockend = round_up(i_size, fs_info->sectorsize);
|
| 3583 | if (lockend <= lockstart)
|
| 3584 | lockend = lockstart + fs_info->sectorsize;
|
| 3585 | lockend--;
|
| 3586 |
|
| 3587 | path = btrfs_alloc_path();
|
| 3588 | if (!path)
|
| 3589 | return -ENOMEM;
|
| 3590 | path->reada = READA_FORWARD;
|
| 3591 |
|
| 3592 | key.objectid = ino;
|
| 3593 | key.type = BTRFS_EXTENT_DATA_KEY;
|
| 3594 | key.offset = start;
|
| 3595 |
|
| 3596 | last_extent_end = lockstart;
|
| 3597 |
|
| 3598 | btrfs_lock_extent(&inode->io_tree, lockstart, lockend, &cached_state);
|
| 3599 |
|
| 3600 | ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
|
| 3601 | if (ret < 0) {
|
| 3602 | goto out;
|
| 3603 | } else if (ret > 0 && path->slots[0] > 0) {
|
| 3604 | btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1);
|
| 3605 | if (key.objectid == ino && key.type == BTRFS_EXTENT_DATA_KEY)
|
| 3606 | path->slots[0]--;
|
| 3607 | }
|
| 3608 |
|
| 3609 | while (start < i_size) {
|
| 3610 | struct extent_buffer *leaf = path->nodes[0];
|
| 3611 | struct btrfs_file_extent_item *extent;
|
| 3612 | u64 extent_end;
|
| 3613 | u8 type;
|
| 3614 |
|
| 3615 | if (path->slots[0] >= btrfs_header_nritems(leaf)) {
|
| 3616 | ret = btrfs_next_leaf(root, path);
|
| 3617 | if (ret < 0)
|
| 3618 | goto out;
|
| 3619 | else if (ret > 0)
|
| 3620 | break;
|
| 3621 |
|
| 3622 | leaf = path->nodes[0];
|
| 3623 | }
|
| 3624 |
|
| 3625 | btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
|
| 3626 | if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY)
|
| 3627 | break;
|
| 3628 |
|
| 3629 | extent_end = btrfs_file_extent_end(path);
|
| 3630 |
|
| 3631 | |
| 3632 | * In the first iteration we may have a slot that points to an
|
| 3633 | * extent that ends before our start offset, so skip it.
|
| 3634 |
|
| 3635 | if (extent_end <= start) {
|
| 3636 | path->slots[0]++;
|
| 3637 | continue;
|
| 3638 | }
|
| 3639 |
|
| 3640 |
|
| 3641 | if (last_extent_end < key.offset) {
|
| 3642 | u64 search_start = last_extent_end;
|
| 3643 | u64 found_start;
|
| 3644 |
|
| 3645 | |
| 3646 | * First iteration, @start matches @offset and it's
|
| 3647 | * within the hole.
|
| 3648 |
|
| 3649 | if (start == offset)
|
| 3650 | search_start = offset;
|
| 3651 |
|
| 3652 | found = find_desired_extent_in_hole(inode, whence,
|
| 3653 | delalloc_cached_state,
|
| 3654 | search_start,
|
| 3655 | key.offset - 1,
|
| 3656 | &found_start);
|
| 3657 | if (found) {
|
| 3658 | start = found_start;
|
| 3659 | break;
|
| 3660 | }
|
| 3661 | |
| 3662 | * Didn't find data or a hole (due to delalloc) in the
|
| 3663 | * implicit hole range, so need to analyze the extent.
|
| 3664 |
|
| 3665 | }
|
| 3666 |
|
| 3667 | extent = btrfs_item_ptr(leaf, path->slots[0],
|
| 3668 | struct btrfs_file_extent_item);
|
| 3669 | type = btrfs_file_extent_type(leaf, extent);
|
| 3670 |
|
| 3671 | |
| 3672 | * Can't access the extent's disk_bytenr field if this is an
|
| 3673 | * inline extent, since at that offset, it's where the extent
|
| 3674 | * data starts.
|
| 3675 |
|
| 3676 | if (type == BTRFS_FILE_EXTENT_PREALLOC ||
|
| 3677 | (type == BTRFS_FILE_EXTENT_REG &&
|
| 3678 | btrfs_file_extent_disk_bytenr(leaf, extent) == 0)) {
|
| 3679 | |
| 3680 | * Explicit hole or prealloc extent, search for delalloc.
|
| 3681 | * A prealloc extent is treated like a hole.
|
| 3682 |
|
| 3683 | u64 search_start = key.offset;
|
| 3684 | u64 found_start;
|
| 3685 |
|
| 3686 | |
| 3687 | * First iteration, @start matches @offset and it's
|
| 3688 | * within the hole.
|
| 3689 |
|
| 3690 | if (start == offset)
|
| 3691 | search_start = offset;
|
| 3692 |
|
| 3693 | found = find_desired_extent_in_hole(inode, whence,
|
| 3694 | delalloc_cached_state,
|
| 3695 | search_start,
|
| 3696 | extent_end - 1,
|
| 3697 | &found_start);
|
| 3698 | if (found) {
|
| 3699 | start = found_start;
|
| 3700 | break;
|
| 3701 | }
|
| 3702 | |
| 3703 | * Didn't find data or a hole (due to delalloc) in the
|
| 3704 | * implicit hole range, so need to analyze the next
|
| 3705 | * extent item.
|
| 3706 |
|
| 3707 | } else {
|
| 3708 | |
| 3709 | * Found a regular or inline extent.
|
| 3710 | * If we are seeking for data, adjust the start offset
|
| 3711 | * and stop, we're done.
|
| 3712 |
|
| 3713 | if (whence == SEEK_DATA) {
|
| 3714 | start = max_t(u64, key.offset, offset);
|
| 3715 | found = true;
|
| 3716 | break;
|
| 3717 | }
|
| 3718 | |
| 3719 | * Else, we are seeking for a hole, check the next file
|
| 3720 | * extent item.
|
| 3721 |
|
| 3722 | }
|
| 3723 |
|
| 3724 | start = extent_end;
|
| 3725 | last_extent_end = extent_end;
|
| 3726 | path->slots[0]++;
|
| 3727 | if (fatal_signal_pending(current)) {
|
| 3728 | ret = -EINTR;
|
| 3729 | goto out;
|
| 3730 | }
|
| 3731 | cond_resched();
|
| 3732 | }
|
| 3733 |
|
| 3734 |
|
| 3735 | if (!found && start < i_size) {
|
| 3736 | found = find_desired_extent_in_hole(inode, whence,
|
| 3737 | delalloc_cached_state, start,
|
| 3738 | i_size - 1, &start);
|
| 3739 | if (!found)
|
| 3740 | start = i_size;
|
| 3741 | }
|
| 3742 |
|
| 3743 | out:
|
| 3744 | btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state);
|
| 3745 | btrfs_free_path(path);
|
| 3746 |
|
| 3747 | if (ret < 0)
|
| 3748 | return ret;
|
| 3749 |
|
| 3750 | if (whence == SEEK_DATA && start >= i_size)
|
| 3751 | return -ENXIO;
|
| 3752 |
|
| 3753 | return min_t(loff_t, start, i_size);
|
| 3754 | }
|
| 3755 |
|
| 3756 | static loff_t btrfs_file_llseek(struct file *file, loff_t offset, int whence)
|
| 3757 | {
|
| 3758 | struct inode *inode = file->f_mapping->host;
|
| 3759 |
|
| 3760 | switch (whence) {
|
| 3761 | default:
|
| 3762 | return generic_file_llseek(file, offset, whence);
|
| 3763 | case SEEK_DATA:
|
| 3764 | case SEEK_HOLE:
|
| 3765 | btrfs_inode_lock(BTRFS_I(inode), BTRFS_ILOCK_SHARED);
|
| 3766 | offset = find_desired_extent(file, offset, whence);
|
| 3767 | btrfs_inode_unlock(BTRFS_I(inode), BTRFS_ILOCK_SHARED);
|
| 3768 | break;
|
| 3769 | }
|
| 3770 |
|
| 3771 | if (offset < 0)
|
| 3772 | return offset;
|
| 3773 |
|
| 3774 | return vfs_setpos(file, offset, inode->i_sb->s_maxbytes);
|
| 3775 | }
|
| 3776 |
|
| 3777 | static int btrfs_file_open(struct inode *inode, struct file *filp)
|
| 3778 | {
|
| 3779 | int ret;
|
| 3780 |
|
| 3781 | if (btrfs_is_shutdown(inode_to_fs_info(inode)))
|
| 3782 | return -EIO;
|
| 3783 |
|
| 3784 | filp->f_mode |= FMODE_NOWAIT | FMODE_CAN_ODIRECT;
|
| 3785 |
|
| 3786 | ret = fsverity_file_open(inode, filp);
|
| 3787 | if (ret)
|
| 3788 | return ret;
|
| 3789 | return generic_file_open(inode, filp);
|
| 3790 | }
|
| 3791 |
|
| 3792 | static ssize_t btrfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
|
| 3793 | {
|
| 3794 | ssize_t ret = 0;
|
| 3795 |
|
| 3796 | if (btrfs_is_shutdown(inode_to_fs_info(file_inode(iocb->ki_filp))))
|
| 3797 | return -EIO;
|
| 3798 |
|
| 3799 | if (iocb->ki_flags & IOCB_DIRECT) {
|
| 3800 | ret = btrfs_direct_read(iocb, to);
|
| 3801 | if (ret < 0 || !iov_iter_count(to) ||
|
| 3802 | iocb->ki_pos >= i_size_read(file_inode(iocb->ki_filp)))
|
| 3803 | return ret;
|
| 3804 | }
|
| 3805 |
|
| 3806 | return filemap_read(iocb, to, ret);
|
| 3807 | }
|
| 3808 |
|
| 3809 | static ssize_t btrfs_file_splice_read(struct file *in, loff_t *ppos,
|
| 3810 | struct pipe_inode_info *pipe,
|
| 3811 | size_t len, unsigned int flags)
|
| 3812 | {
|
| 3813 | if (btrfs_is_shutdown(inode_to_fs_info(file_inode(in))))
|
| 3814 | return -EIO;
|
| 3815 |
|
| 3816 | return filemap_splice_read(in, ppos, pipe, len, flags);
|
| 3817 | }
|
| 3818 |
|
| 3819 | const struct file_operations btrfs_file_operations = {
|
| 3820 | .llseek = btrfs_file_llseek,
|
| 3821 | .read_iter = btrfs_file_read_iter,
|
| 3822 | .splice_read = btrfs_file_splice_read,
|
| 3823 | .write_iter = btrfs_file_write_iter,
|
| 3824 | .splice_write = iter_file_splice_write,
|
| 3825 | .mmap_prepare = btrfs_file_mmap_prepare,
|
| 3826 | .open = btrfs_file_open,
|
| 3827 | .release = btrfs_release_file,
|
| 3828 | .get_unmapped_area = thp_get_unmapped_area,
|
| 3829 | .fsync = btrfs_sync_file,
|
| 3830 | .fallocate = btrfs_fallocate,
|
| 3831 | .unlocked_ioctl = btrfs_ioctl,
|
| 3832 | #ifdef CONFIG_COMPAT
|
| 3833 | .compat_ioctl = btrfs_compat_ioctl,
|
| 3834 | #endif
|
| 3835 | .remap_file_range = btrfs_remap_file_range,
|
| 3836 | .uring_cmd = btrfs_uring_cmd,
|
| 3837 | .fop_flags = FOP_BUFFER_RASYNC | FOP_BUFFER_WASYNC,
|
| 3838 | .setlease = generic_setlease,
|
| 3839 | };
|
| 3840 |
|
| 3841 | int btrfs_fdatawrite_range(struct btrfs_inode *inode, loff_t start, loff_t end)
|
| 3842 | {
|
| 3843 | struct address_space *mapping = inode->vfs_inode.i_mapping;
|
| 3844 | int ret;
|
| 3845 |
|
| 3846 | |
| 3847 | * So with compression we will find and lock a dirty page and clear the
|
| 3848 | * first one as dirty, setup an async extent, and immediately return
|
| 3849 | * with the entire range locked but with nobody actually marked with
|
| 3850 | * writeback. So we can't just filemap_write_and_wait_range() and
|
| 3851 | * expect it to work since it will just kick off a thread to do the
|
| 3852 | * actual work. So we need to call filemap_fdatawrite_range _again_
|
| 3853 | * since it will wait on the page lock, which won't be unlocked until
|
| 3854 | * after the pages have been marked as writeback and so we're good to go
|
| 3855 | * from there. We have to do this otherwise we'll miss the ordered
|
| 3856 | * extents and that results in badness. Please Josef, do not think you
|
| 3857 | * know better and pull this out at some point in the future, it is
|
| 3858 | * right and you are wrong.
|
| 3859 |
|
| 3860 | ret = filemap_fdatawrite_range(mapping, start, end);
|
| 3861 | if (!ret && test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, &inode->runtime_flags))
|
| 3862 | ret = filemap_fdatawrite_range(mapping, start, end);
|
| 3863 |
|
| 3864 | return ret;
|
| 3865 | }
|