| 1 | |
| 2 | |
| 3 | * linux/fs/ext4/inode.c
|
| 4 | *
|
| 5 | * Copyright (C) 1992, 1993, 1994, 1995
|
| 6 | * Remy Card ([email protected])
|
| 7 | * Laboratoire MASI - Institut Blaise Pascal
|
| 8 | * Universite Pierre et Marie Curie (Paris VI)
|
| 9 | *
|
| 10 | * from
|
| 11 | *
|
| 12 | * linux/fs/minix/inode.c
|
| 13 | *
|
| 14 | * Copyright (C) 1991, 1992 Linus Torvalds
|
| 15 | *
|
| 16 | * 64-bit file support on 64-bit platforms by Jakub Jelinek
|
| 17 | * ([email protected])
|
| 18 | *
|
| 19 | * Assorted race fixes, rewrite of ext4_get_block() by Al Viro, 2000
|
| 20 |
|
| 21 |
|
| 22 | #include <linux/fs.h>
|
| 23 | #include <linux/mount.h>
|
| 24 | #include <linux/time.h>
|
| 25 | #include <linux/highuid.h>
|
| 26 | #include <linux/pagemap.h>
|
| 27 | #include <linux/dax.h>
|
| 28 | #include <linux/quotaops.h>
|
| 29 | #include <linux/string.h>
|
| 30 | #include <linux/buffer_head.h>
|
| 31 | #include <linux/writeback.h>
|
| 32 | #include <linux/folio_batch.h>
|
| 33 | #include <linux/mpage.h>
|
| 34 | #include <linux/rmap.h>
|
| 35 | #include <linux/namei.h>
|
| 36 | #include <linux/uio.h>
|
| 37 | #include <linux/bio.h>
|
| 38 | #include <linux/workqueue.h>
|
| 39 | #include <linux/kernel.h>
|
| 40 | #include <linux/printk.h>
|
| 41 | #include <linux/slab.h>
|
| 42 | #include <linux/bitops.h>
|
| 43 | #include <linux/iomap.h>
|
| 44 | #include <linux/iversion.h>
|
| 45 |
|
| 46 | #include "ext4_jbd2.h"
|
| 47 | #include "xattr.h"
|
| 48 | #include "acl.h"
|
| 49 | #include "truncate.h"
|
| 50 |
|
| 51 | #include <kunit/static_stub.h>
|
| 52 |
|
| 53 | #include <trace/events/ext4.h>
|
| 54 |
|
| 55 | static void ext4_journalled_zero_new_buffers(handle_t *handle,
|
| 56 | struct inode *inode,
|
| 57 | struct folio *folio,
|
| 58 | unsigned from, unsigned to);
|
| 59 |
|
| 60 | static __u32 ext4_inode_csum(struct inode *inode, struct ext4_inode *raw,
|
| 61 | struct ext4_inode_info *ei)
|
| 62 | {
|
| 63 | __u32 csum;
|
| 64 | __u16 dummy_csum = 0;
|
| 65 | int offset = offsetof(struct ext4_inode, i_checksum_lo);
|
| 66 | unsigned int csum_size = sizeof(dummy_csum);
|
| 67 |
|
| 68 | csum = ext4_chksum(ei->i_csum_seed, (__u8 *)raw, offset);
|
| 69 | csum = ext4_chksum(csum, (__u8 *)&dummy_csum, csum_size);
|
| 70 | offset += csum_size;
|
| 71 | csum = ext4_chksum(csum, (__u8 *)raw + offset,
|
| 72 | EXT4_GOOD_OLD_INODE_SIZE - offset);
|
| 73 |
|
| 74 | if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
|
| 75 | offset = offsetof(struct ext4_inode, i_checksum_hi);
|
| 76 | csum = ext4_chksum(csum, (__u8 *)raw + EXT4_GOOD_OLD_INODE_SIZE,
|
| 77 | offset - EXT4_GOOD_OLD_INODE_SIZE);
|
| 78 | if (EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) {
|
| 79 | csum = ext4_chksum(csum, (__u8 *)&dummy_csum,
|
| 80 | csum_size);
|
| 81 | offset += csum_size;
|
| 82 | }
|
| 83 | csum = ext4_chksum(csum, (__u8 *)raw + offset,
|
| 84 | EXT4_INODE_SIZE(inode->i_sb) - offset);
|
| 85 | }
|
| 86 |
|
| 87 | return csum;
|
| 88 | }
|
| 89 |
|
| 90 | static int ext4_inode_csum_verify(struct inode *inode, struct ext4_inode *raw,
|
| 91 | struct ext4_inode_info *ei)
|
| 92 | {
|
| 93 | __u32 provided, calculated;
|
| 94 |
|
| 95 | if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
|
| 96 | cpu_to_le32(EXT4_OS_LINUX) ||
|
| 97 | !ext4_has_feature_metadata_csum(inode->i_sb))
|
| 98 | return 1;
|
| 99 |
|
| 100 | provided = le16_to_cpu(raw->i_checksum_lo);
|
| 101 | calculated = ext4_inode_csum(inode, raw, ei);
|
| 102 | if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
|
| 103 | EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
|
| 104 | provided |= ((__u32)le16_to_cpu(raw->i_checksum_hi)) << 16;
|
| 105 | else
|
| 106 | calculated &= 0xFFFF;
|
| 107 |
|
| 108 | return provided == calculated;
|
| 109 | }
|
| 110 |
|
| 111 | void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw,
|
| 112 | struct ext4_inode_info *ei)
|
| 113 | {
|
| 114 | __u32 csum;
|
| 115 |
|
| 116 | if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
|
| 117 | cpu_to_le32(EXT4_OS_LINUX) ||
|
| 118 | !ext4_has_feature_metadata_csum(inode->i_sb))
|
| 119 | return;
|
| 120 |
|
| 121 | csum = ext4_inode_csum(inode, raw, ei);
|
| 122 | raw->i_checksum_lo = cpu_to_le16(csum & 0xFFFF);
|
| 123 | if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
|
| 124 | EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
|
| 125 | raw->i_checksum_hi = cpu_to_le16(csum >> 16);
|
| 126 | }
|
| 127 |
|
| 128 | static inline int ext4_begin_ordered_truncate(struct inode *inode,
|
| 129 | loff_t new_size)
|
| 130 | {
|
| 131 | struct jbd2_inode *jinode = READ_ONCE(EXT4_I(inode)->jinode);
|
| 132 |
|
| 133 | trace_ext4_begin_ordered_truncate(inode, new_size);
|
| 134 | |
| 135 | * If jinode is zero, then we never opened the file for
|
| 136 | * writing, so there's no need to call
|
| 137 | * jbd2_journal_begin_ordered_truncate() since there's no
|
| 138 | * outstanding writes we need to flush.
|
| 139 |
|
| 140 | if (!jinode)
|
| 141 | return 0;
|
| 142 | return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode),
|
| 143 | jinode,
|
| 144 | new_size);
|
| 145 | }
|
| 146 |
|
| 147 | |
| 148 | * Test whether an inode is a fast symlink.
|
| 149 | * A fast symlink has its symlink data stored in ext4_inode_info->i_data.
|
| 150 |
|
| 151 | int ext4_inode_is_fast_symlink(struct inode *inode)
|
| 152 | {
|
| 153 | if (!ext4_has_feature_ea_inode(inode->i_sb)) {
|
| 154 | int ea_blocks = EXT4_I(inode)->i_file_acl ?
|
| 155 | EXT4_CLUSTER_SIZE(inode->i_sb) >> 9 : 0;
|
| 156 |
|
| 157 | if (ext4_has_inline_data(inode))
|
| 158 | return 0;
|
| 159 |
|
| 160 | return (S_ISLNK(inode->i_mode) && inode->i_blocks - ea_blocks == 0);
|
| 161 | }
|
| 162 | return S_ISLNK(inode->i_mode) && inode->i_size &&
|
| 163 | (inode->i_size < EXT4_N_BLOCKS * 4);
|
| 164 | }
|
| 165 |
|
| 166 | |
| 167 | * Called at the last iput() if i_nlink is zero.
|
| 168 |
|
| 169 | void ext4_evict_inode(struct inode *inode)
|
| 170 | {
|
| 171 | handle_t *handle;
|
| 172 | int err;
|
| 173 | |
| 174 | * Credits for final inode cleanup and freeing:
|
| 175 | * sb + inode (ext4_orphan_del()), block bitmap, group descriptor
|
| 176 | * (xattr block freeing), bitmap, group descriptor (inode freeing)
|
| 177 |
|
| 178 | int extra_credits = 6;
|
| 179 | bool freeze_protected = false;
|
| 180 |
|
| 181 | trace_ext4_evict_inode(inode);
|
| 182 |
|
| 183 | dax_break_layout_final(inode);
|
| 184 |
|
| 185 | if (EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)
|
| 186 | ext4_evict_ea_inode(inode);
|
| 187 | if (inode->i_nlink) {
|
| 188 | struct mapping_metadata_bhs *mmb;
|
| 189 |
|
| 190 | |
| 191 | * If there's dirty page will lead to data loss, user
|
| 192 | * could see stale data.
|
| 193 |
|
| 194 | if (unlikely(!ext4_emergency_state(inode->i_sb) &&
|
| 195 | mapping_tagged(&inode->i_data, PAGECACHE_TAG_DIRTY)))
|
| 196 | ext4_warning_inode(inode, "data will be lost");
|
| 197 |
|
| 198 | truncate_inode_pages_final(&inode->i_data);
|
| 199 | mmb = ext4_i_metadata_bhs(inode);
|
| 200 | if (mmb)
|
| 201 | mmb_sync(mmb);
|
| 202 | goto no_delete;
|
| 203 | }
|
| 204 |
|
| 205 | if (is_bad_inode(inode))
|
| 206 | goto no_delete;
|
| 207 | dquot_initialize(inode);
|
| 208 |
|
| 209 | if (ext4_should_order_data(inode))
|
| 210 | ext4_begin_ordered_truncate(inode, 0);
|
| 211 | truncate_inode_pages_final(&inode->i_data);
|
| 212 |
|
| 213 | |
| 214 | * For inodes with journalled data, transaction commit could have
|
| 215 | * dirtied the inode. And for inodes with dioread_nolock, unwritten
|
| 216 | * extents converting worker could merge extents and also have dirtied
|
| 217 | * the inode. Flush worker is ignoring it because of I_FREEING flag but
|
| 218 | * we still need to remove the inode from the writeback lists.
|
| 219 |
|
| 220 | inode_io_list_del(inode);
|
| 221 |
|
| 222 | |
| 223 | * Protect us against freezing - iput() caller didn't have to have any
|
| 224 | * protection against it. When we are in a running transaction though,
|
| 225 | * we are already protected against freezing and we cannot grab further
|
| 226 | * protection due to lock ordering constraints.
|
| 227 |
|
| 228 | if (!ext4_journal_current_handle()) {
|
| 229 | sb_start_intwrite(inode->i_sb);
|
| 230 | freeze_protected = true;
|
| 231 | }
|
| 232 |
|
| 233 | if (!IS_NOQUOTA(inode))
|
| 234 | extra_credits += EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb);
|
| 235 |
|
| 236 | |
| 237 | * Block bitmap, group descriptor, and inode are accounted in both
|
| 238 | * ext4_blocks_for_truncate() and extra_credits. So subtract 3.
|
| 239 |
|
| 240 | handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE,
|
| 241 | ext4_blocks_for_truncate(inode) + extra_credits - 3);
|
| 242 | if (IS_ERR(handle)) {
|
| 243 | ext4_std_error(inode->i_sb, PTR_ERR(handle));
|
| 244 | |
| 245 | * If we're going to skip the normal cleanup, we still need to
|
| 246 | * make sure that the in-core orphan linked list is properly
|
| 247 | * cleaned up.
|
| 248 |
|
| 249 | ext4_orphan_del(NULL, inode);
|
| 250 | if (freeze_protected)
|
| 251 | sb_end_intwrite(inode->i_sb);
|
| 252 | goto no_delete;
|
| 253 | }
|
| 254 |
|
| 255 | if (IS_SYNC(inode))
|
| 256 | ext4_handle_sync(handle);
|
| 257 |
|
| 258 | |
| 259 | * Set inode->i_size to 0 before calling ext4_truncate(). We need
|
| 260 | * special handling of symlinks here because i_size is used to
|
| 261 | * determine whether ext4_inode_info->i_data contains symlink data or
|
| 262 | * block mappings. Setting i_size to 0 will remove its fast symlink
|
| 263 | * status. Erase i_data so that it becomes a valid empty block map.
|
| 264 |
|
| 265 | if (ext4_inode_is_fast_symlink(inode))
|
| 266 | memset(EXT4_I(inode)->i_data, 0, sizeof(EXT4_I(inode)->i_data));
|
| 267 | inode->i_size = 0;
|
| 268 | ext4_set_inode_state(inode, EXT4_STATE_NO_EXPAND);
|
| 269 | err = ext4_mark_inode_dirty(handle, inode);
|
| 270 | if (err) {
|
| 271 | ext4_warning(inode->i_sb,
|
| 272 | "couldn't mark inode dirty (err %d)", err);
|
| 273 | goto stop_handle;
|
| 274 | }
|
| 275 | if (inode->i_blocks) {
|
| 276 | err = ext4_truncate(inode);
|
| 277 | if (err) {
|
| 278 | ext4_error_err(inode->i_sb, -err,
|
| 279 | "couldn't truncate inode %llu (err %d)",
|
| 280 | inode->i_ino, err);
|
| 281 | goto stop_handle;
|
| 282 | }
|
| 283 | }
|
| 284 |
|
| 285 |
|
| 286 | err = ext4_xattr_delete_inode(handle, inode, extra_credits);
|
| 287 | if (err) {
|
| 288 | ext4_warning(inode->i_sb, "xattr delete (err %d)", err);
|
| 289 | stop_handle:
|
| 290 | ext4_journal_stop(handle);
|
| 291 | ext4_orphan_del(NULL, inode);
|
| 292 | if (freeze_protected)
|
| 293 | sb_end_intwrite(inode->i_sb);
|
| 294 | goto no_delete;
|
| 295 | }
|
| 296 |
|
| 297 | |
| 298 | * Kill off the orphan record which ext4_truncate created.
|
| 299 | * AKPM: I think this can be inside the above `if'.
|
| 300 | * Note that ext4_orphan_del() has to be able to cope with the
|
| 301 | * deletion of a non-existent orphan - this is because we don't
|
| 302 | * know if ext4_truncate() actually created an orphan record.
|
| 303 | * (Well, we could do this if we need to, but heck - it works)
|
| 304 |
|
| 305 | ext4_orphan_del(handle, inode);
|
| 306 | EXT4_I(inode)->i_dtime = (__u32)ktime_get_real_seconds();
|
| 307 |
|
| 308 | |
| 309 | * One subtle ordering requirement: if anything has gone wrong
|
| 310 | * (transaction abort, IO errors, whatever), then we can still
|
| 311 | * do these next steps (the fs will already have been marked as
|
| 312 | * having errors), but we can't free the inode if the mark_dirty
|
| 313 | * fails.
|
| 314 |
|
| 315 | if (ext4_mark_inode_dirty(handle, inode))
|
| 316 |
|
| 317 | ext4_clear_inode(inode);
|
| 318 | else
|
| 319 | ext4_free_inode(handle, inode);
|
| 320 | ext4_journal_stop(handle);
|
| 321 | if (freeze_protected)
|
| 322 | sb_end_intwrite(inode->i_sb);
|
| 323 | return;
|
| 324 | no_delete:
|
| 325 | |
| 326 | * Check out some where else accidentally dirty the evicting inode,
|
| 327 | * which may probably cause inode use-after-free issues later.
|
| 328 |
|
| 329 | WARN_ON_ONCE(!list_empty_careful(&inode->i_io_list));
|
| 330 |
|
| 331 | if (!list_empty(&EXT4_I(inode)->i_fc_list))
|
| 332 | ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_NOMEM, NULL);
|
| 333 | ext4_clear_inode(inode);
|
| 334 | }
|
| 335 |
|
| 336 | #ifdef CONFIG_QUOTA
|
| 337 | qsize_t *ext4_get_reserved_space(struct inode *inode)
|
| 338 | {
|
| 339 | return &EXT4_I(inode)->i_reserved_quota;
|
| 340 | }
|
| 341 | #endif
|
| 342 |
|
| 343 | |
| 344 | * Called with i_data_sem down, which is important since we can call
|
| 345 | * ext4_discard_preallocations() from here.
|
| 346 |
|
| 347 | void ext4_da_update_reserve_space(struct inode *inode,
|
| 348 | int used, int quota_claim)
|
| 349 | {
|
| 350 | struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
|
| 351 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 352 |
|
| 353 | spin_lock(&ei->i_block_reservation_lock);
|
| 354 | trace_ext4_da_update_reserve_space(inode, used, quota_claim);
|
| 355 | if (unlikely(used > ei->i_reserved_data_blocks)) {
|
| 356 | ext4_warning(inode->i_sb, "%s: ino %llu, used %d "
|
| 357 | "with only %d reserved data blocks",
|
| 358 | __func__, inode->i_ino, used,
|
| 359 | ei->i_reserved_data_blocks);
|
| 360 | WARN_ON(1);
|
| 361 | used = ei->i_reserved_data_blocks;
|
| 362 | }
|
| 363 |
|
| 364 |
|
| 365 | ei->i_reserved_data_blocks -= used;
|
| 366 | percpu_counter_sub(&sbi->s_dirtyclusters_counter, used);
|
| 367 |
|
| 368 | spin_unlock(&ei->i_block_reservation_lock);
|
| 369 |
|
| 370 |
|
| 371 | if (quota_claim)
|
| 372 | dquot_claim_block(inode, EXT4_C2B(sbi, used));
|
| 373 | else {
|
| 374 | |
| 375 | * We did fallocate with an offset that is already delayed
|
| 376 | * allocated. So on delayed allocated writeback we should
|
| 377 | * not re-claim the quota for fallocated blocks.
|
| 378 |
|
| 379 | dquot_release_reservation_block(inode, EXT4_C2B(sbi, used));
|
| 380 | }
|
| 381 |
|
| 382 | |
| 383 | * If we have done all the pending block allocations and if
|
| 384 | * there aren't any writers on the inode, we can discard the
|
| 385 | * inode's preallocations.
|
| 386 |
|
| 387 | if ((ei->i_reserved_data_blocks == 0) &&
|
| 388 | !inode_is_open_for_write(inode))
|
| 389 | ext4_discard_preallocations(inode);
|
| 390 | }
|
| 391 |
|
| 392 | static int __check_block_validity(struct inode *inode, const char *func,
|
| 393 | unsigned int line,
|
| 394 | struct ext4_map_blocks *map)
|
| 395 | {
|
| 396 | journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
|
| 397 |
|
| 398 | if (journal && inode == journal->j_inode)
|
| 399 | return 0;
|
| 400 |
|
| 401 | if (!ext4_inode_block_valid(inode, map->m_pblk, map->m_len)) {
|
| 402 | ext4_error_inode(inode, func, line, map->m_pblk,
|
| 403 | "lblock %lu mapped to illegal pblock %llu "
|
| 404 | "(length %d)", (unsigned long) map->m_lblk,
|
| 405 | map->m_pblk, map->m_len);
|
| 406 | return -EFSCORRUPTED;
|
| 407 | }
|
| 408 | return 0;
|
| 409 | }
|
| 410 |
|
| 411 | int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk,
|
| 412 | ext4_lblk_t len)
|
| 413 | {
|
| 414 | int ret;
|
| 415 |
|
| 416 | KUNIT_STATIC_STUB_REDIRECT(ext4_issue_zeroout, inode, lblk, pblk, len);
|
| 417 |
|
| 418 | if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode))
|
| 419 | return fscrypt_zeroout_range(inode,
|
| 420 | (loff_t)lblk << inode->i_blkbits,
|
| 421 | pblk << (inode->i_blkbits - SECTOR_SHIFT),
|
| 422 | (u64)len << inode->i_blkbits);
|
| 423 |
|
| 424 | ret = sb_issue_zeroout(inode->i_sb, pblk, len, GFP_NOFS);
|
| 425 | if (ret > 0)
|
| 426 | ret = 0;
|
| 427 |
|
| 428 | return ret;
|
| 429 | }
|
| 430 |
|
| 431 | |
| 432 | * For generic regular files, when updating the extent tree, Ext4 should
|
| 433 | * hold the i_rwsem and invalidate_lock exclusively. This ensures
|
| 434 | * exclusion against concurrent page faults, as well as reads and writes.
|
| 435 |
|
| 436 | #ifdef CONFIG_EXT4_DEBUG
|
| 437 | void ext4_check_map_extents_env(struct inode *inode)
|
| 438 | {
|
| 439 | if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
|
| 440 | return;
|
| 441 |
|
| 442 | if (!S_ISREG(inode->i_mode) ||
|
| 443 | IS_NOQUOTA(inode) || IS_VERITY(inode) ||
|
| 444 | is_special_ino(inode->i_sb, inode->i_ino) ||
|
| 445 | (inode_state_read_once(inode) & (I_FREEING | I_WILL_FREE | I_NEW)) ||
|
| 446 | ext4_test_inode_flag(inode, EXT4_INODE_EA_INODE) ||
|
| 447 | ext4_verity_in_progress(inode))
|
| 448 | return;
|
| 449 |
|
| 450 | WARN_ON_ONCE(!inode_is_locked(inode) &&
|
| 451 | !rwsem_is_locked(&inode->i_mapping->invalidate_lock));
|
| 452 | }
|
| 453 | #else
|
| 454 | void ext4_check_map_extents_env(struct inode *inode) {}
|
| 455 | #endif
|
| 456 |
|
| 457 | #define check_block_validity(inode, map) \
|
| 458 | __check_block_validity((inode), __func__, __LINE__, (map))
|
| 459 |
|
| 460 | #ifdef ES_AGGRESSIVE_TEST
|
| 461 | static void ext4_map_blocks_es_recheck(handle_t *handle,
|
| 462 | struct inode *inode,
|
| 463 | struct ext4_map_blocks *es_map,
|
| 464 | struct ext4_map_blocks *map,
|
| 465 | int flags)
|
| 466 | {
|
| 467 | int retval;
|
| 468 |
|
| 469 | map->m_flags = 0;
|
| 470 | |
| 471 | * There is a race window that the result is not the same.
|
| 472 | * e.g. xfstests #223 when dioread_nolock enables. The reason
|
| 473 | * is that we lookup a block mapping in extent status tree with
|
| 474 | * out taking i_data_sem. So at the time the unwritten extent
|
| 475 | * could be converted.
|
| 476 |
|
| 477 | down_read(&EXT4_I(inode)->i_data_sem);
|
| 478 | if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
|
| 479 | retval = ext4_ext_map_blocks(handle, inode, map, 0);
|
| 480 | } else {
|
| 481 | retval = ext4_ind_map_blocks(handle, inode, map, 0);
|
| 482 | }
|
| 483 | up_read((&EXT4_I(inode)->i_data_sem));
|
| 484 |
|
| 485 | |
| 486 | * We don't check m_len because extent will be collpased in status
|
| 487 | * tree. So the m_len might not equal.
|
| 488 |
|
| 489 | if (es_map->m_lblk != map->m_lblk ||
|
| 490 | es_map->m_flags != map->m_flags ||
|
| 491 | es_map->m_pblk != map->m_pblk) {
|
| 492 | printk("ES cache assertion failed for inode: %llu "
|
| 493 | "es_cached ex [%d/%d/%llu/%x] != "
|
| 494 | "found ex [%d/%d/%llu/%x] retval %d flags %x\n",
|
| 495 | inode->i_ino, es_map->m_lblk, es_map->m_len,
|
| 496 | es_map->m_pblk, es_map->m_flags, map->m_lblk,
|
| 497 | map->m_len, map->m_pblk, map->m_flags,
|
| 498 | retval, flags);
|
| 499 | }
|
| 500 | }
|
| 501 | #endif
|
| 502 |
|
| 503 | static int ext4_map_query_blocks_next_in_leaf(handle_t *handle,
|
| 504 | struct inode *inode, struct ext4_map_blocks *map,
|
| 505 | unsigned int orig_mlen)
|
| 506 | {
|
| 507 | struct ext4_map_blocks map2;
|
| 508 | unsigned int status, status2;
|
| 509 | int retval;
|
| 510 |
|
| 511 | status = map->m_flags & EXT4_MAP_UNWRITTEN ?
|
| 512 | EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
|
| 513 |
|
| 514 | WARN_ON_ONCE(!(map->m_flags & EXT4_MAP_QUERY_LAST_IN_LEAF));
|
| 515 | WARN_ON_ONCE(orig_mlen <= map->m_len);
|
| 516 |
|
| 517 |
|
| 518 | map2.m_lblk = map->m_lblk + map->m_len;
|
| 519 | map2.m_len = orig_mlen - map->m_len;
|
| 520 | map2.m_flags = 0;
|
| 521 | retval = ext4_ext_map_blocks(handle, inode, &map2, 0);
|
| 522 |
|
| 523 | if (retval <= 0) {
|
| 524 | ext4_es_cache_extent(inode, map->m_lblk, map->m_len,
|
| 525 | map->m_pblk, status);
|
| 526 | return map->m_len;
|
| 527 | }
|
| 528 |
|
| 529 | if (unlikely(retval != map2.m_len)) {
|
| 530 | ext4_warning(inode->i_sb,
|
| 531 | "ES len assertion failed for inode "
|
| 532 | "%llu: retval %d != map->m_len %d",
|
| 533 | inode->i_ino, retval, map2.m_len);
|
| 534 | WARN_ON(1);
|
| 535 | }
|
| 536 |
|
| 537 | status2 = map2.m_flags & EXT4_MAP_UNWRITTEN ?
|
| 538 | EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
|
| 539 |
|
| 540 | |
| 541 | * If map2 is contiguous with map, then let's insert it as a single
|
| 542 | * extent in es cache and return the combined length of both the maps.
|
| 543 |
|
| 544 | if (map->m_pblk + map->m_len == map2.m_pblk &&
|
| 545 | status == status2) {
|
| 546 | ext4_es_cache_extent(inode, map->m_lblk,
|
| 547 | map->m_len + map2.m_len, map->m_pblk,
|
| 548 | status);
|
| 549 | map->m_len += map2.m_len;
|
| 550 | } else {
|
| 551 | ext4_es_cache_extent(inode, map->m_lblk, map->m_len,
|
| 552 | map->m_pblk, status);
|
| 553 | }
|
| 554 |
|
| 555 | return map->m_len;
|
| 556 | }
|
| 557 |
|
| 558 | int ext4_map_query_blocks(handle_t *handle, struct inode *inode,
|
| 559 | struct ext4_map_blocks *map, int flags)
|
| 560 | {
|
| 561 | unsigned int status;
|
| 562 | int retval;
|
| 563 | unsigned int orig_mlen = map->m_len;
|
| 564 |
|
| 565 | flags &= EXT4_EX_QUERY_FILTER;
|
| 566 | if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
|
| 567 | retval = ext4_ext_map_blocks(handle, inode, map, flags);
|
| 568 | else
|
| 569 | retval = ext4_ind_map_blocks(handle, inode, map, flags);
|
| 570 | if (retval < 0)
|
| 571 | return retval;
|
| 572 |
|
| 573 |
|
| 574 | if (retval == 0)
|
| 575 | goto out;
|
| 576 |
|
| 577 | if (unlikely(retval != map->m_len)) {
|
| 578 | ext4_warning(inode->i_sb,
|
| 579 | "ES len assertion failed for inode "
|
| 580 | "%llu: retval %d != map->m_len %d",
|
| 581 | inode->i_ino, retval, map->m_len);
|
| 582 | WARN_ON(1);
|
| 583 | }
|
| 584 |
|
| 585 | |
| 586 | * No need to query next in leaf:
|
| 587 | * - if returned extent is not last in leaf or
|
| 588 | * - if the last in leaf is the full requested range
|
| 589 |
|
| 590 | if (!(map->m_flags & EXT4_MAP_QUERY_LAST_IN_LEAF) ||
|
| 591 | map->m_len == orig_mlen) {
|
| 592 | status = map->m_flags & EXT4_MAP_UNWRITTEN ?
|
| 593 | EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
|
| 594 | ext4_es_cache_extent(inode, map->m_lblk, map->m_len,
|
| 595 | map->m_pblk, status);
|
| 596 | } else {
|
| 597 | retval = ext4_map_query_blocks_next_in_leaf(handle, inode, map,
|
| 598 | orig_mlen);
|
| 599 | }
|
| 600 | out:
|
| 601 | map->m_seq = READ_ONCE(EXT4_I(inode)->i_es_seq);
|
| 602 | return retval;
|
| 603 | }
|
| 604 |
|
| 605 | int ext4_map_create_blocks(handle_t *handle, struct inode *inode,
|
| 606 | struct ext4_map_blocks *map, int flags)
|
| 607 | {
|
| 608 | unsigned int status;
|
| 609 | int err, retval = 0;
|
| 610 |
|
| 611 | |
| 612 | * We pass in the magic EXT4_GET_BLOCKS_DELALLOC_RESERVE
|
| 613 | * indicates that the blocks and quotas has already been
|
| 614 | * checked when the data was copied into the page cache.
|
| 615 |
|
| 616 | if (map->m_flags & EXT4_MAP_DELAYED)
|
| 617 | flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE;
|
| 618 |
|
| 619 | |
| 620 | * Here we clear m_flags because after allocating an new extent,
|
| 621 | * it will be set again.
|
| 622 |
|
| 623 | map->m_flags &= ~EXT4_MAP_FLAGS;
|
| 624 |
|
| 625 | |
| 626 | * We need to check for EXT4 here because migrate could have
|
| 627 | * changed the inode type in between.
|
| 628 |
|
| 629 | if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
|
| 630 | retval = ext4_ext_map_blocks(handle, inode, map, flags);
|
| 631 | } else {
|
| 632 | retval = ext4_ind_map_blocks(handle, inode, map, flags);
|
| 633 |
|
| 634 | |
| 635 | * We allocated new blocks which will result in i_data's
|
| 636 | * format changing. Force the migrate to fail by clearing
|
| 637 | * migrate flags.
|
| 638 |
|
| 639 | if (retval > 0 && map->m_flags & EXT4_MAP_NEW)
|
| 640 | ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE);
|
| 641 | }
|
| 642 | if (retval <= 0)
|
| 643 | return retval;
|
| 644 |
|
| 645 | if (unlikely(retval != map->m_len)) {
|
| 646 | ext4_warning(inode->i_sb,
|
| 647 | "ES len assertion failed for inode %llu: "
|
| 648 | "retval %d != map->m_len %d",
|
| 649 | inode->i_ino, retval, map->m_len);
|
| 650 | WARN_ON(1);
|
| 651 | }
|
| 652 |
|
| 653 | |
| 654 | * We have to zeroout blocks before inserting them into extent
|
| 655 | * status tree. Otherwise someone could look them up there and
|
| 656 | * use them before they are really zeroed. We also have to
|
| 657 | * unmap metadata before zeroing as otherwise writeback can
|
| 658 | * overwrite zeros with stale data from block device.
|
| 659 |
|
| 660 | if (flags & EXT4_GET_BLOCKS_ZERO &&
|
| 661 | map->m_flags & EXT4_MAP_MAPPED && map->m_flags & EXT4_MAP_NEW) {
|
| 662 | err = ext4_issue_zeroout(inode, map->m_lblk, map->m_pblk,
|
| 663 | map->m_len);
|
| 664 | if (err)
|
| 665 | return err;
|
| 666 | }
|
| 667 |
|
| 668 | status = map->m_flags & EXT4_MAP_UNWRITTEN ?
|
| 669 | EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
|
| 670 | ext4_es_insert_extent(inode, map->m_lblk, map->m_len, map->m_pblk,
|
| 671 | status, flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE);
|
| 672 | map->m_seq = READ_ONCE(EXT4_I(inode)->i_es_seq);
|
| 673 |
|
| 674 | return retval;
|
| 675 | }
|
| 676 |
|
| 677 | |
| 678 | * The ext4_map_blocks() function tries to look up the requested blocks,
|
| 679 | * and returns if the blocks are already mapped.
|
| 680 | *
|
| 681 | * Otherwise it takes the write lock of the i_data_sem and allocate blocks
|
| 682 | * and store the allocated blocks in the result buffer head and mark it
|
| 683 | * mapped.
|
| 684 | *
|
| 685 | * If file type is extents based, it will call ext4_ext_map_blocks(),
|
| 686 | * Otherwise, call with ext4_ind_map_blocks() to handle indirect mapping
|
| 687 | * based files
|
| 688 | *
|
| 689 | * On success, it returns the number of blocks being mapped or allocated.
|
| 690 | * If flags doesn't contain EXT4_GET_BLOCKS_CREATE the blocks are
|
| 691 | * pre-allocated and unwritten, the resulting @map is marked as unwritten.
|
| 692 | * If the flags contain EXT4_GET_BLOCKS_CREATE, it will mark @map as mapped.
|
| 693 | *
|
| 694 | * It returns 0 if plain look up failed (blocks have not been allocated), in
|
| 695 | * that case, @map is returned as unmapped but we still do fill map->m_len to
|
| 696 | * indicate the length of a hole starting at map->m_lblk.
|
| 697 | *
|
| 698 | * It returns the error in case of allocation failure.
|
| 699 |
|
| 700 | int ext4_map_blocks(handle_t *handle, struct inode *inode,
|
| 701 | struct ext4_map_blocks *map, int flags)
|
| 702 | {
|
| 703 | struct extent_status es;
|
| 704 | int retval;
|
| 705 | int ret = 0;
|
| 706 | unsigned int orig_mlen = map->m_len;
|
| 707 | #ifdef ES_AGGRESSIVE_TEST
|
| 708 | struct ext4_map_blocks orig_map;
|
| 709 |
|
| 710 | memcpy(&orig_map, map, sizeof(*map));
|
| 711 | #endif
|
| 712 |
|
| 713 | map->m_flags = 0;
|
| 714 | ext_debug(inode, "flag 0x%x, max_blocks %u, logical block %lu\n",
|
| 715 | flags, map->m_len, (unsigned long) map->m_lblk);
|
| 716 |
|
| 717 | |
| 718 | * ext4_map_blocks returns an int, and m_len is an unsigned int
|
| 719 |
|
| 720 | if (unlikely(map->m_len > INT_MAX))
|
| 721 | map->m_len = INT_MAX;
|
| 722 |
|
| 723 |
|
| 724 | if (unlikely(map->m_lblk >= EXT_MAX_BLOCKS))
|
| 725 | return -EFSCORRUPTED;
|
| 726 |
|
| 727 | |
| 728 | * Callers from the context of data submission are the only exceptions
|
| 729 | * for regular files that do not hold the i_rwsem or invalidate_lock.
|
| 730 | * However, caching unrelated ranges is not permitted.
|
| 731 |
|
| 732 | if (flags & EXT4_GET_BLOCKS_IO_SUBMIT)
|
| 733 | WARN_ON_ONCE(!(flags & EXT4_EX_NOCACHE));
|
| 734 | else
|
| 735 | ext4_check_map_extents_env(inode);
|
| 736 |
|
| 737 |
|
| 738 | if (ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es, &map->m_seq)) {
|
| 739 | if (ext4_es_is_written(&es) || ext4_es_is_unwritten(&es)) {
|
| 740 | map->m_pblk = ext4_es_pblock(&es) +
|
| 741 | map->m_lblk - es.es_lblk;
|
| 742 | map->m_flags |= ext4_es_is_written(&es) ?
|
| 743 | EXT4_MAP_MAPPED : EXT4_MAP_UNWRITTEN;
|
| 744 | retval = es.es_len - (map->m_lblk - es.es_lblk);
|
| 745 | if (retval > map->m_len)
|
| 746 | retval = map->m_len;
|
| 747 | map->m_len = retval;
|
| 748 | } else if (ext4_es_is_delayed(&es) || ext4_es_is_hole(&es)) {
|
| 749 | map->m_pblk = 0;
|
| 750 | map->m_flags |= ext4_es_is_delayed(&es) ?
|
| 751 | EXT4_MAP_DELAYED : 0;
|
| 752 | retval = es.es_len - (map->m_lblk - es.es_lblk);
|
| 753 | if (retval > map->m_len)
|
| 754 | retval = map->m_len;
|
| 755 | map->m_len = retval;
|
| 756 | retval = 0;
|
| 757 | } else {
|
| 758 | BUG();
|
| 759 | }
|
| 760 |
|
| 761 | if (flags & EXT4_GET_BLOCKS_CACHED_NOWAIT)
|
| 762 | return retval;
|
| 763 | #ifdef ES_AGGRESSIVE_TEST
|
| 764 | ext4_map_blocks_es_recheck(handle, inode, map,
|
| 765 | &orig_map, flags);
|
| 766 | #endif
|
| 767 | if (!(flags & EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF) ||
|
| 768 | orig_mlen == map->m_len)
|
| 769 | goto found;
|
| 770 |
|
| 771 | map->m_len = orig_mlen;
|
| 772 | }
|
| 773 | |
| 774 | * In the query cache no-wait mode, nothing we can do more if we
|
| 775 | * cannot find extent in the cache.
|
| 776 |
|
| 777 | if (flags & EXT4_GET_BLOCKS_CACHED_NOWAIT)
|
| 778 | return 0;
|
| 779 |
|
| 780 | |
| 781 | * Try to see if we can get the block without requesting a new
|
| 782 | * file system block.
|
| 783 |
|
| 784 | down_read(&EXT4_I(inode)->i_data_sem);
|
| 785 | retval = ext4_map_query_blocks(handle, inode, map, flags);
|
| 786 | up_read((&EXT4_I(inode)->i_data_sem));
|
| 787 |
|
| 788 | found:
|
| 789 | if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
|
| 790 | ret = check_block_validity(inode, map);
|
| 791 | if (ret != 0)
|
| 792 | return ret;
|
| 793 | }
|
| 794 |
|
| 795 |
|
| 796 | if ((flags & EXT4_GET_BLOCKS_CREATE) == 0)
|
| 797 | return retval;
|
| 798 |
|
| 799 | |
| 800 | * Returns if the blocks have already allocated
|
| 801 | *
|
| 802 | * Note that if blocks have been preallocated
|
| 803 | * ext4_ext_map_blocks() returns with buffer head unmapped
|
| 804 |
|
| 805 | if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED)
|
| 806 | |
| 807 | * If we need to convert extent to unwritten
|
| 808 | * we continue and do the actual work in
|
| 809 | * ext4_ext_map_blocks()
|
| 810 |
|
| 811 | if (!(flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN))
|
| 812 | return retval;
|
| 813 |
|
| 814 |
|
| 815 | ext4_fc_track_inode(handle, inode);
|
| 816 | |
| 817 | * New blocks allocate and/or writing to unwritten extent
|
| 818 | * will possibly result in updating i_data, so we take
|
| 819 | * the write lock of i_data_sem, and call get_block()
|
| 820 | * with create == 1 flag.
|
| 821 |
|
| 822 | down_write(&EXT4_I(inode)->i_data_sem);
|
| 823 | retval = ext4_map_create_blocks(handle, inode, map, flags);
|
| 824 | up_write((&EXT4_I(inode)->i_data_sem));
|
| 825 |
|
| 826 | if (retval < 0)
|
| 827 | ext_debug(inode, "failed with err %d\n", retval);
|
| 828 | if (retval <= 0)
|
| 829 | return retval;
|
| 830 |
|
| 831 | if (map->m_flags & EXT4_MAP_MAPPED) {
|
| 832 | ret = check_block_validity(inode, map);
|
| 833 | if (ret != 0)
|
| 834 | return ret;
|
| 835 |
|
| 836 | |
| 837 | * Inodes with freshly allocated blocks where contents will be
|
| 838 | * visible after transaction commit must be on transaction's
|
| 839 | * ordered data list.
|
| 840 |
|
| 841 | if (map->m_flags & EXT4_MAP_NEW &&
|
| 842 | !(map->m_flags & EXT4_MAP_UNWRITTEN) &&
|
| 843 | !(flags & EXT4_GET_BLOCKS_ZERO) &&
|
| 844 | !ext4_is_quota_file(inode) &&
|
| 845 | ext4_should_order_data(inode)) {
|
| 846 | loff_t start_byte = EXT4_LBLK_TO_B(inode, map->m_lblk);
|
| 847 | loff_t length = EXT4_LBLK_TO_B(inode, map->m_len);
|
| 848 |
|
| 849 | if (flags & EXT4_GET_BLOCKS_IO_SUBMIT)
|
| 850 | ret = ext4_jbd2_inode_add_wait(handle, inode,
|
| 851 | start_byte, length);
|
| 852 | else
|
| 853 | ret = ext4_jbd2_inode_add_write(handle, inode,
|
| 854 | start_byte, length);
|
| 855 | if (ret)
|
| 856 | return ret;
|
| 857 | }
|
| 858 | }
|
| 859 | ext4_fc_track_range(handle, inode, map->m_lblk, map->m_lblk +
|
| 860 | map->m_len - 1);
|
| 861 | return retval;
|
| 862 | }
|
| 863 |
|
| 864 | |
| 865 | * Update EXT4_MAP_FLAGS in bh->b_state. For buffer heads attached to pages
|
| 866 | * we have to be careful as someone else may be manipulating b_state as well.
|
| 867 |
|
| 868 | static void ext4_update_bh_state(struct buffer_head *bh, unsigned long flags)
|
| 869 | {
|
| 870 | unsigned long old_state;
|
| 871 | unsigned long new_state;
|
| 872 |
|
| 873 | flags &= EXT4_MAP_FLAGS;
|
| 874 |
|
| 875 |
|
| 876 | if (!bh->b_folio) {
|
| 877 | bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | flags;
|
| 878 | return;
|
| 879 | }
|
| 880 | |
| 881 | * Someone else may be modifying b_state. Be careful! This is ugly but
|
| 882 | * once we get rid of using bh as a container for mapping information
|
| 883 | * to pass to / from get_block functions, this can go away.
|
| 884 |
|
| 885 | old_state = READ_ONCE(bh->b_state);
|
| 886 | do {
|
| 887 | new_state = (old_state & ~EXT4_MAP_FLAGS) | flags;
|
| 888 | } while (unlikely(!try_cmpxchg(&bh->b_state, &old_state, new_state)));
|
| 889 | }
|
| 890 |
|
| 891 | |
| 892 | * Make sure that the current journal transaction has enough credits to map
|
| 893 | * one extent. Return -EAGAIN if it cannot extend the current running
|
| 894 | * transaction.
|
| 895 |
|
| 896 | static inline int ext4_journal_ensure_extent_credits(handle_t *handle,
|
| 897 | struct inode *inode)
|
| 898 | {
|
| 899 | int credits;
|
| 900 | int ret;
|
| 901 |
|
| 902 |
|
| 903 | if (!handle)
|
| 904 | return 0;
|
| 905 |
|
| 906 | credits = ext4_chunk_trans_blocks(inode, 1);
|
| 907 | ret = __ext4_journal_ensure_credits(handle, credits, credits, 0);
|
| 908 | return ret <= 0 ? ret : -EAGAIN;
|
| 909 | }
|
| 910 |
|
| 911 | static int _ext4_get_block(struct inode *inode, sector_t iblock,
|
| 912 | struct buffer_head *bh, int flags)
|
| 913 | {
|
| 914 | struct ext4_map_blocks map;
|
| 915 | int ret = 0;
|
| 916 |
|
| 917 | if (ext4_has_inline_data(inode))
|
| 918 | return -ERANGE;
|
| 919 |
|
| 920 | map.m_lblk = iblock;
|
| 921 | map.m_len = bh->b_size >> inode->i_blkbits;
|
| 922 |
|
| 923 | ret = ext4_map_blocks(ext4_journal_current_handle(), inode, &map,
|
| 924 | flags);
|
| 925 | if (ret > 0) {
|
| 926 | map_bh(bh, inode->i_sb, map.m_pblk);
|
| 927 | ext4_update_bh_state(bh, map.m_flags);
|
| 928 | bh->b_size = inode->i_sb->s_blocksize * map.m_len;
|
| 929 | ret = 0;
|
| 930 | } else if (ret == 0) {
|
| 931 |
|
| 932 | bh->b_size = inode->i_sb->s_blocksize * map.m_len;
|
| 933 | }
|
| 934 | return ret;
|
| 935 | }
|
| 936 |
|
| 937 | int ext4_get_block(struct inode *inode, sector_t iblock,
|
| 938 | struct buffer_head *bh, int create)
|
| 939 | {
|
| 940 | return _ext4_get_block(inode, iblock, bh,
|
| 941 | create ? EXT4_GET_BLOCKS_CREATE : 0);
|
| 942 | }
|
| 943 |
|
| 944 | |
| 945 | * Get block function used when preparing for buffered write if we require
|
| 946 | * creating an unwritten extent if blocks haven't been allocated. The extent
|
| 947 | * will be converted to written after the IO is complete.
|
| 948 |
|
| 949 | int ext4_get_block_unwritten(struct inode *inode, sector_t iblock,
|
| 950 | struct buffer_head *bh_result, int create)
|
| 951 | {
|
| 952 | int ret = 0;
|
| 953 |
|
| 954 | ext4_debug("ext4_get_block_unwritten: inode %llu, create flag %d\n",
|
| 955 | inode->i_ino, create);
|
| 956 | ret = _ext4_get_block(inode, iblock, bh_result,
|
| 957 | EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT);
|
| 958 |
|
| 959 | |
| 960 | * If the buffer is marked unwritten, mark it as new to make sure it is
|
| 961 | * zeroed out correctly in case of partial writes. Otherwise, there is
|
| 962 | * a chance of stale data getting exposed.
|
| 963 |
|
| 964 | if (ret == 0 && buffer_unwritten(bh_result))
|
| 965 | set_buffer_new(bh_result);
|
| 966 |
|
| 967 | return ret;
|
| 968 | }
|
| 969 |
|
| 970 |
|
| 971 | #define DIO_MAX_BLOCKS 4096
|
| 972 |
|
| 973 | |
| 974 | * `handle' can be NULL if create is zero
|
| 975 |
|
| 976 | struct buffer_head *ext4_getblk(handle_t *handle, struct inode *inode,
|
| 977 | ext4_lblk_t block, int map_flags)
|
| 978 | {
|
| 979 | struct ext4_map_blocks map;
|
| 980 | struct buffer_head *bh;
|
| 981 | int create = map_flags & EXT4_GET_BLOCKS_CREATE;
|
| 982 | bool nowait = map_flags & EXT4_GET_BLOCKS_CACHED_NOWAIT;
|
| 983 | int err;
|
| 984 |
|
| 985 | ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
|
| 986 | || handle != NULL || create == 0);
|
| 987 | ASSERT(create == 0 || !nowait);
|
| 988 |
|
| 989 | map.m_lblk = block;
|
| 990 | map.m_len = 1;
|
| 991 | err = ext4_map_blocks(handle, inode, &map, map_flags);
|
| 992 |
|
| 993 | if (err == 0)
|
| 994 | return create ? ERR_PTR(-ENOSPC) : NULL;
|
| 995 | if (err < 0)
|
| 996 | return ERR_PTR(err);
|
| 997 |
|
| 998 | if (nowait)
|
| 999 | return sb_find_get_block(inode->i_sb, map.m_pblk);
|
| 1000 |
|
| 1001 | |
| 1002 | * Since bh could introduce extra ref count such as referred by
|
| 1003 | * journal_head etc. Try to avoid using __GFP_MOVABLE here
|
| 1004 | * as it may fail the migration when journal_head remains.
|
| 1005 |
|
| 1006 | bh = getblk_unmovable(inode->i_sb->s_bdev, map.m_pblk,
|
| 1007 | inode->i_sb->s_blocksize);
|
| 1008 |
|
| 1009 | if (unlikely(!bh))
|
| 1010 | return ERR_PTR(-ENOMEM);
|
| 1011 | if (map.m_flags & EXT4_MAP_NEW) {
|
| 1012 | ASSERT(create != 0);
|
| 1013 | ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
|
| 1014 | || (handle != NULL));
|
| 1015 |
|
| 1016 | |
| 1017 | * Now that we do not always journal data, we should
|
| 1018 | * keep in mind whether this should always journal the
|
| 1019 | * new buffer as metadata. For now, regular file
|
| 1020 | * writes use ext4_get_block instead, so it's not a
|
| 1021 | * problem.
|
| 1022 |
|
| 1023 | lock_buffer(bh);
|
| 1024 | BUFFER_TRACE(bh, "call get_create_access");
|
| 1025 | err = ext4_journal_get_create_access(handle, inode->i_sb, bh,
|
| 1026 | EXT4_JTR_NONE);
|
| 1027 | if (unlikely(err)) {
|
| 1028 | unlock_buffer(bh);
|
| 1029 | goto errout;
|
| 1030 | }
|
| 1031 | if (!buffer_uptodate(bh)) {
|
| 1032 | memset(bh->b_data, 0, inode->i_sb->s_blocksize);
|
| 1033 | set_buffer_uptodate(bh);
|
| 1034 | }
|
| 1035 | unlock_buffer(bh);
|
| 1036 | BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
|
| 1037 | err = ext4_handle_dirty_metadata(handle, inode, bh);
|
| 1038 | if (unlikely(err))
|
| 1039 | goto errout;
|
| 1040 | } else
|
| 1041 | BUFFER_TRACE(bh, "not a new buffer");
|
| 1042 | return bh;
|
| 1043 | errout:
|
| 1044 | brelse(bh);
|
| 1045 | return ERR_PTR(err);
|
| 1046 | }
|
| 1047 |
|
| 1048 | struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode,
|
| 1049 | ext4_lblk_t block, int map_flags)
|
| 1050 | {
|
| 1051 | struct buffer_head *bh;
|
| 1052 | int ret;
|
| 1053 |
|
| 1054 | bh = ext4_getblk(handle, inode, block, map_flags);
|
| 1055 | if (IS_ERR(bh))
|
| 1056 | return bh;
|
| 1057 | if (!bh || ext4_buffer_uptodate(bh))
|
| 1058 | return bh;
|
| 1059 |
|
| 1060 | ret = ext4_read_bh_lock(bh, REQ_META | REQ_PRIO, true);
|
| 1061 | if (ret) {
|
| 1062 | put_bh(bh);
|
| 1063 | return ERR_PTR(ret);
|
| 1064 | }
|
| 1065 | return bh;
|
| 1066 | }
|
| 1067 |
|
| 1068 |
|
| 1069 | int ext4_bread_batch(struct inode *inode, ext4_lblk_t block, int bh_count,
|
| 1070 | bool wait, struct buffer_head **bhs)
|
| 1071 | {
|
| 1072 | int i, err;
|
| 1073 |
|
| 1074 | for (i = 0; i < bh_count; i++) {
|
| 1075 | bhs[i] = ext4_getblk(NULL, inode, block + i, 0 );
|
| 1076 | if (IS_ERR(bhs[i])) {
|
| 1077 | err = PTR_ERR(bhs[i]);
|
| 1078 | bh_count = i;
|
| 1079 | goto out_brelse;
|
| 1080 | }
|
| 1081 | }
|
| 1082 |
|
| 1083 | for (i = 0; i < bh_count; i++)
|
| 1084 |
|
| 1085 | if (bhs[i] && !ext4_buffer_uptodate(bhs[i]))
|
| 1086 | ext4_read_bh_lock(bhs[i], REQ_META | REQ_PRIO, false);
|
| 1087 |
|
| 1088 | if (!wait)
|
| 1089 | return 0;
|
| 1090 |
|
| 1091 | for (i = 0; i < bh_count; i++)
|
| 1092 | if (bhs[i])
|
| 1093 | wait_on_buffer(bhs[i]);
|
| 1094 |
|
| 1095 | for (i = 0; i < bh_count; i++) {
|
| 1096 | if (bhs[i] && !buffer_uptodate(bhs[i])) {
|
| 1097 | err = -EIO;
|
| 1098 | goto out_brelse;
|
| 1099 | }
|
| 1100 | }
|
| 1101 | return 0;
|
| 1102 |
|
| 1103 | out_brelse:
|
| 1104 | for (i = 0; i < bh_count; i++) {
|
| 1105 | brelse(bhs[i]);
|
| 1106 | bhs[i] = NULL;
|
| 1107 | }
|
| 1108 | return err;
|
| 1109 | }
|
| 1110 |
|
| 1111 | int ext4_walk_page_buffers(handle_t *handle, struct inode *inode,
|
| 1112 | struct buffer_head *head,
|
| 1113 | unsigned from,
|
| 1114 | unsigned to,
|
| 1115 | int *partial,
|
| 1116 | int (*fn)(handle_t *handle, struct inode *inode,
|
| 1117 | struct buffer_head *bh))
|
| 1118 | {
|
| 1119 | struct buffer_head *bh;
|
| 1120 | unsigned block_start, block_end;
|
| 1121 | unsigned blocksize = head->b_size;
|
| 1122 | int err, ret = 0;
|
| 1123 | struct buffer_head *next;
|
| 1124 |
|
| 1125 | for (bh = head, block_start = 0;
|
| 1126 | ret == 0 && (bh != head || !block_start);
|
| 1127 | block_start = block_end, bh = next) {
|
| 1128 | next = bh->b_this_page;
|
| 1129 | block_end = block_start + blocksize;
|
| 1130 | if (block_end <= from || block_start >= to) {
|
| 1131 | if (partial && !buffer_uptodate(bh))
|
| 1132 | *partial = 1;
|
| 1133 | continue;
|
| 1134 | }
|
| 1135 | err = (*fn)(handle, inode, bh);
|
| 1136 | if (!ret)
|
| 1137 | ret = err;
|
| 1138 | }
|
| 1139 | return ret;
|
| 1140 | }
|
| 1141 |
|
| 1142 | |
| 1143 | * Helper for handling dirtying of journalled data. We also mark the folio as
|
| 1144 | * dirty so that writeback code knows about this page (and inode) contains
|
| 1145 | * dirty data. ext4_writepages() then commits appropriate transaction to
|
| 1146 | * make data stable.
|
| 1147 |
|
| 1148 | static int ext4_dirty_journalled_data(handle_t *handle, struct buffer_head *bh)
|
| 1149 | {
|
| 1150 | struct folio *folio = bh->b_folio;
|
| 1151 | struct inode *inode = folio->mapping->host;
|
| 1152 |
|
| 1153 |
|
| 1154 | if (S_ISREG(inode->i_mode))
|
| 1155 | folio_mark_dirty(folio);
|
| 1156 | return ext4_handle_dirty_metadata(handle, NULL, bh);
|
| 1157 | }
|
| 1158 |
|
| 1159 | int do_journal_get_write_access(handle_t *handle, struct inode *inode,
|
| 1160 | struct buffer_head *bh)
|
| 1161 | {
|
| 1162 | if (!buffer_mapped(bh) || buffer_freed(bh))
|
| 1163 | return 0;
|
| 1164 | BUFFER_TRACE(bh, "get write access");
|
| 1165 | return ext4_journal_get_write_access(handle, inode->i_sb, bh,
|
| 1166 | EXT4_JTR_NONE);
|
| 1167 | }
|
| 1168 |
|
| 1169 | int ext4_block_write_begin(handle_t *handle, struct folio *folio,
|
| 1170 | loff_t pos, unsigned len,
|
| 1171 | get_block_t *get_block)
|
| 1172 | {
|
| 1173 | unsigned int from = offset_in_folio(folio, pos);
|
| 1174 | unsigned to = from + len;
|
| 1175 | struct inode *inode = folio->mapping->host;
|
| 1176 | unsigned block_start, block_end;
|
| 1177 | sector_t block;
|
| 1178 | int err = 0;
|
| 1179 | unsigned int blocksize = i_blocksize(inode);
|
| 1180 | struct buffer_head *bh, *head, *wait[2];
|
| 1181 | int nr_wait = 0;
|
| 1182 | int i;
|
| 1183 | bool should_journal_data = ext4_should_journal_data(inode);
|
| 1184 | bool folio_uptodate = folio_test_uptodate(folio);
|
| 1185 |
|
| 1186 | BUG_ON(!folio_test_locked(folio));
|
| 1187 | BUG_ON(to > folio_size(folio));
|
| 1188 | BUG_ON(from > to);
|
| 1189 | WARN_ON_ONCE(blocksize > folio_size(folio));
|
| 1190 |
|
| 1191 | head = folio_buffers(folio);
|
| 1192 | if (!head)
|
| 1193 | head = create_empty_buffers(folio, blocksize, 0);
|
| 1194 | block = EXT4_PG_TO_LBLK(inode, folio->index);
|
| 1195 |
|
| 1196 | for (bh = head, block_start = 0;
|
| 1197 | block_start < to || (!folio_uptodate && bh != head);
|
| 1198 | block++, block_start = block_end, bh = bh->b_this_page) {
|
| 1199 | block_end = block_start + blocksize;
|
| 1200 | if (block_end <= from || block_start >= to) {
|
| 1201 | if (folio_uptodate)
|
| 1202 | set_buffer_uptodate(bh);
|
| 1203 | continue;
|
| 1204 | }
|
| 1205 | if (WARN_ON_ONCE(buffer_new(bh)))
|
| 1206 | clear_buffer_new(bh);
|
| 1207 | if (!buffer_mapped(bh)) {
|
| 1208 | WARN_ON(bh->b_size != blocksize);
|
| 1209 | err = ext4_journal_ensure_extent_credits(handle, inode);
|
| 1210 | if (!err)
|
| 1211 | err = get_block(inode, block, bh, 1);
|
| 1212 | if (err)
|
| 1213 | break;
|
| 1214 | if (buffer_new(bh)) {
|
| 1215 | |
| 1216 | * We may be zeroing partial buffers or all new
|
| 1217 | * buffers in case of failure. Prepare JBD2 for
|
| 1218 | * that.
|
| 1219 |
|
| 1220 | if (should_journal_data)
|
| 1221 | do_journal_get_write_access(handle,
|
| 1222 | inode, bh);
|
| 1223 | if (folio_uptodate) {
|
| 1224 | |
| 1225 | * Unlike __block_write_begin() we leave
|
| 1226 | * dirtying of new uptodate buffers to
|
| 1227 | * ->write_end() time or
|
| 1228 | * folio_zero_new_buffers().
|
| 1229 |
|
| 1230 | set_buffer_uptodate(bh);
|
| 1231 | continue;
|
| 1232 | }
|
| 1233 | if (block_end > to || block_start < from)
|
| 1234 | folio_zero_segments(folio, to,
|
| 1235 | block_end,
|
| 1236 | block_start, from);
|
| 1237 | continue;
|
| 1238 | }
|
| 1239 | }
|
| 1240 | if (folio_uptodate) {
|
| 1241 | set_buffer_uptodate(bh);
|
| 1242 | continue;
|
| 1243 | }
|
| 1244 | if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
|
| 1245 | !buffer_unwritten(bh) &&
|
| 1246 | (block_start < from || block_end > to)) {
|
| 1247 | ext4_read_bh_lock(bh, 0, false);
|
| 1248 | wait[nr_wait++] = bh;
|
| 1249 | }
|
| 1250 | }
|
| 1251 | |
| 1252 | * If we issued read requests, let them complete.
|
| 1253 |
|
| 1254 | for (i = 0; i < nr_wait; i++) {
|
| 1255 | wait_on_buffer(wait[i]);
|
| 1256 | if (!buffer_uptodate(wait[i]))
|
| 1257 | err = -EIO;
|
| 1258 | }
|
| 1259 | if (unlikely(err)) {
|
| 1260 | if (should_journal_data)
|
| 1261 | ext4_journalled_zero_new_buffers(handle, inode, folio,
|
| 1262 | from, to);
|
| 1263 | else
|
| 1264 | folio_zero_new_buffers(folio, from, to);
|
| 1265 | }
|
| 1266 |
|
| 1267 | return err;
|
| 1268 | }
|
| 1269 |
|
| 1270 | |
| 1271 | * To preserve ordering, it is essential that the hole instantiation and
|
| 1272 | * the data write be encapsulated in a single transaction. We cannot
|
| 1273 | * close off a transaction and start a new one between the ext4_get_block()
|
| 1274 | * and the ext4_write_end(). So doing the jbd2_journal_start at the start of
|
| 1275 | * ext4_write_begin() is the right place.
|
| 1276 |
|
| 1277 | static int ext4_write_begin(const struct kiocb *iocb,
|
| 1278 | struct address_space *mapping,
|
| 1279 | loff_t pos, unsigned len,
|
| 1280 | struct folio **foliop, void **fsdata)
|
| 1281 | {
|
| 1282 | struct inode *inode = mapping->host;
|
| 1283 | int ret, needed_blocks;
|
| 1284 | handle_t *handle;
|
| 1285 | int retries = 0;
|
| 1286 | struct folio *folio;
|
| 1287 | pgoff_t index;
|
| 1288 | unsigned from, to;
|
| 1289 |
|
| 1290 | ret = ext4_emergency_state(inode->i_sb);
|
| 1291 | if (unlikely(ret))
|
| 1292 | return ret;
|
| 1293 |
|
| 1294 | *fsdata = (void *)((unsigned long)*fsdata & ~EXT4_WRITE_DATA_INLINE);
|
| 1295 |
|
| 1296 | trace_ext4_write_begin(inode, pos, len);
|
| 1297 | |
| 1298 | * Reserve one block more for addition to orphan list in case
|
| 1299 | * we allocate blocks but write fails for some reason
|
| 1300 |
|
| 1301 | needed_blocks = ext4_chunk_trans_extent(inode,
|
| 1302 | ext4_journal_blocks_per_folio(inode)) + 1;
|
| 1303 | index = pos >> PAGE_SHIFT;
|
| 1304 |
|
| 1305 | if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
|
| 1306 | ret = ext4_try_to_write_inline_data(mapping, inode, pos, len,
|
| 1307 | foliop);
|
| 1308 | if (ret < 0)
|
| 1309 | return ret;
|
| 1310 | if (ret == 1) {
|
| 1311 | *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE);
|
| 1312 | return 0;
|
| 1313 | }
|
| 1314 | }
|
| 1315 |
|
| 1316 | |
| 1317 | * write_begin_get_folio() can take a long time if the
|
| 1318 | * system is thrashing due to memory pressure, or if the folio
|
| 1319 | * is being written back. So grab it first before we start
|
| 1320 | * the transaction handle. This also allows us to allocate
|
| 1321 | * the folio (if needed) without using GFP_NOFS.
|
| 1322 |
|
| 1323 | retry_grab:
|
| 1324 | folio = write_begin_get_folio(iocb, mapping, index, len);
|
| 1325 | if (IS_ERR(folio))
|
| 1326 | return PTR_ERR(folio);
|
| 1327 |
|
| 1328 | if (len > folio_next_pos(folio) - pos)
|
| 1329 | len = folio_next_pos(folio) - pos;
|
| 1330 |
|
| 1331 | from = offset_in_folio(folio, pos);
|
| 1332 | to = from + len;
|
| 1333 |
|
| 1334 | |
| 1335 | * The same as page allocation, we prealloc buffer heads before
|
| 1336 | * starting the handle.
|
| 1337 |
|
| 1338 | if (!folio_buffers(folio))
|
| 1339 | create_empty_buffers(folio, inode->i_sb->s_blocksize, 0);
|
| 1340 |
|
| 1341 | folio_unlock(folio);
|
| 1342 |
|
| 1343 | retry_journal:
|
| 1344 | handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks);
|
| 1345 | if (IS_ERR(handle)) {
|
| 1346 | folio_put(folio);
|
| 1347 | return PTR_ERR(handle);
|
| 1348 | }
|
| 1349 |
|
| 1350 | folio_lock(folio);
|
| 1351 | if (folio->mapping != mapping) {
|
| 1352 |
|
| 1353 | folio_unlock(folio);
|
| 1354 | folio_put(folio);
|
| 1355 | ext4_journal_stop(handle);
|
| 1356 | goto retry_grab;
|
| 1357 | }
|
| 1358 |
|
| 1359 | folio_wait_stable(folio);
|
| 1360 |
|
| 1361 | if (ext4_should_dioread_nolock(inode))
|
| 1362 | ret = ext4_block_write_begin(handle, folio, pos, len,
|
| 1363 | ext4_get_block_unwritten);
|
| 1364 | else
|
| 1365 | ret = ext4_block_write_begin(handle, folio, pos, len,
|
| 1366 | ext4_get_block);
|
| 1367 | if (!ret && ext4_should_journal_data(inode)) {
|
| 1368 | ret = ext4_walk_page_buffers(handle, inode,
|
| 1369 | folio_buffers(folio), from, to,
|
| 1370 | NULL, do_journal_get_write_access);
|
| 1371 | }
|
| 1372 |
|
| 1373 | if (ret) {
|
| 1374 | bool extended = (pos + len > inode->i_size) &&
|
| 1375 | !ext4_verity_in_progress(inode);
|
| 1376 |
|
| 1377 | folio_unlock(folio);
|
| 1378 | |
| 1379 | * ext4_block_write_begin may have instantiated a few blocks
|
| 1380 | * outside i_size. Trim these off again. Don't need
|
| 1381 | * i_size_read because we hold i_rwsem.
|
| 1382 | *
|
| 1383 | * Add inode to orphan list in case we crash before
|
| 1384 | * truncate finishes
|
| 1385 |
|
| 1386 | if (extended && ext4_can_truncate(inode))
|
| 1387 | ext4_orphan_add(handle, inode);
|
| 1388 |
|
| 1389 | ext4_journal_stop(handle);
|
| 1390 | if (extended) {
|
| 1391 | ext4_truncate_failed_write(inode);
|
| 1392 | |
| 1393 | * If truncate failed early the inode might
|
| 1394 | * still be on the orphan list; we need to
|
| 1395 | * make sure the inode is removed from the
|
| 1396 | * orphan list in that case.
|
| 1397 |
|
| 1398 | if (inode->i_nlink)
|
| 1399 | ext4_orphan_del(NULL, inode);
|
| 1400 | }
|
| 1401 |
|
| 1402 | if (ret == -EAGAIN ||
|
| 1403 | (ret == -ENOSPC &&
|
| 1404 | ext4_should_retry_alloc(inode->i_sb, &retries)))
|
| 1405 | goto retry_journal;
|
| 1406 | folio_put(folio);
|
| 1407 | return ret;
|
| 1408 | }
|
| 1409 | *foliop = folio;
|
| 1410 | return ret;
|
| 1411 | }
|
| 1412 |
|
| 1413 |
|
| 1414 | static int write_end_fn(handle_t *handle, struct inode *inode,
|
| 1415 | struct buffer_head *bh)
|
| 1416 | {
|
| 1417 | int ret;
|
| 1418 | if (!buffer_mapped(bh) || buffer_freed(bh))
|
| 1419 | return 0;
|
| 1420 | set_buffer_uptodate(bh);
|
| 1421 | ret = ext4_dirty_journalled_data(handle, bh);
|
| 1422 | clear_buffer_meta(bh);
|
| 1423 | clear_buffer_prio(bh);
|
| 1424 | clear_buffer_new(bh);
|
| 1425 | return ret;
|
| 1426 | }
|
| 1427 |
|
| 1428 | |
| 1429 | * We need to pick up the new inode size which generic_commit_write gave us
|
| 1430 | * `iocb` can be NULL - eg, when called from page_symlink().
|
| 1431 |
|
| 1432 | static int ext4_write_end(const struct kiocb *iocb,
|
| 1433 | struct address_space *mapping,
|
| 1434 | loff_t pos, unsigned len, unsigned copied,
|
| 1435 | struct folio *folio, void *fsdata)
|
| 1436 | {
|
| 1437 | handle_t *handle = ext4_journal_current_handle();
|
| 1438 | struct inode *inode = mapping->host;
|
| 1439 | loff_t old_size = inode->i_size;
|
| 1440 | int ret = 0, ret2;
|
| 1441 | int i_size_changed = 0;
|
| 1442 | bool verity = ext4_verity_in_progress(inode);
|
| 1443 |
|
| 1444 | trace_ext4_write_end(inode, pos, len, copied);
|
| 1445 |
|
| 1446 | if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE)
|
| 1447 | return ext4_write_inline_data_end(inode, pos, len, copied,
|
| 1448 | folio);
|
| 1449 |
|
| 1450 | copied = block_write_end(pos, len, copied, folio);
|
| 1451 | |
| 1452 | * it's important to update i_size while still holding folio lock:
|
| 1453 | * page writeout could otherwise come in and zero beyond i_size.
|
| 1454 | *
|
| 1455 | * If FS_IOC_ENABLE_VERITY is running on this inode, then Merkle tree
|
| 1456 | * blocks are being written past EOF, so skip the i_size update.
|
| 1457 |
|
| 1458 | if (!verity)
|
| 1459 | i_size_changed = ext4_update_inode_size(inode, pos + copied);
|
| 1460 | folio_unlock(folio);
|
| 1461 | folio_put(folio);
|
| 1462 |
|
| 1463 | if (old_size < pos && !verity)
|
| 1464 | pagecache_isize_extended(inode, old_size, pos);
|
| 1465 |
|
| 1466 | |
| 1467 | * Don't mark the inode dirty under folio lock. First, it unnecessarily
|
| 1468 | * makes the holding time of folio lock longer. Second, it forces lock
|
| 1469 | * ordering of folio lock and transaction start for journaling
|
| 1470 | * filesystems.
|
| 1471 |
|
| 1472 | if (i_size_changed)
|
| 1473 | ret = ext4_mark_inode_dirty(handle, inode);
|
| 1474 |
|
| 1475 | if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode))
|
| 1476 | |
| 1477 | * less. We will have blocks allocated outside
|
| 1478 | * inode->i_size. So truncate them
|
| 1479 |
|
| 1480 | ext4_orphan_add(handle, inode);
|
| 1481 |
|
| 1482 | ret2 = ext4_journal_stop(handle);
|
| 1483 | if (!ret)
|
| 1484 | ret = ret2;
|
| 1485 |
|
| 1486 | if (pos + len > inode->i_size && !verity) {
|
| 1487 | ext4_truncate_failed_write(inode);
|
| 1488 | |
| 1489 | * If truncate failed early the inode might still be
|
| 1490 | * on the orphan list; we need to make sure the inode
|
| 1491 | * is removed from the orphan list in that case.
|
| 1492 |
|
| 1493 | if (inode->i_nlink)
|
| 1494 | ext4_orphan_del(NULL, inode);
|
| 1495 | }
|
| 1496 |
|
| 1497 | return ret ? ret : copied;
|
| 1498 | }
|
| 1499 |
|
| 1500 | |
| 1501 | * This is a private version of folio_zero_new_buffers() which doesn't
|
| 1502 | * set the buffer to be dirty, since in data=journalled mode we need
|
| 1503 | * to call ext4_dirty_journalled_data() instead.
|
| 1504 |
|
| 1505 | static void ext4_journalled_zero_new_buffers(handle_t *handle,
|
| 1506 | struct inode *inode,
|
| 1507 | struct folio *folio,
|
| 1508 | unsigned from, unsigned to)
|
| 1509 | {
|
| 1510 | unsigned int block_start = 0, block_end;
|
| 1511 | struct buffer_head *head, *bh;
|
| 1512 |
|
| 1513 | bh = head = folio_buffers(folio);
|
| 1514 | do {
|
| 1515 | block_end = block_start + bh->b_size;
|
| 1516 | if (buffer_new(bh)) {
|
| 1517 | if (block_end > from && block_start < to) {
|
| 1518 | if (!folio_test_uptodate(folio)) {
|
| 1519 | unsigned start, size;
|
| 1520 |
|
| 1521 | start = max(from, block_start);
|
| 1522 | size = min(to, block_end) - start;
|
| 1523 |
|
| 1524 | folio_zero_range(folio, start, size);
|
| 1525 | }
|
| 1526 | clear_buffer_new(bh);
|
| 1527 | write_end_fn(handle, inode, bh);
|
| 1528 | }
|
| 1529 | }
|
| 1530 | block_start = block_end;
|
| 1531 | bh = bh->b_this_page;
|
| 1532 | } while (bh != head);
|
| 1533 | }
|
| 1534 |
|
| 1535 | static int ext4_journalled_write_end(const struct kiocb *iocb,
|
| 1536 | struct address_space *mapping,
|
| 1537 | loff_t pos, unsigned len, unsigned copied,
|
| 1538 | struct folio *folio, void *fsdata)
|
| 1539 | {
|
| 1540 | handle_t *handle = ext4_journal_current_handle();
|
| 1541 | struct inode *inode = mapping->host;
|
| 1542 | loff_t old_size = inode->i_size;
|
| 1543 | int ret = 0, ret2;
|
| 1544 | int partial = 0;
|
| 1545 | unsigned from, to;
|
| 1546 | int size_changed = 0;
|
| 1547 | bool verity = ext4_verity_in_progress(inode);
|
| 1548 |
|
| 1549 | trace_ext4_journalled_write_end(inode, pos, len, copied);
|
| 1550 | from = pos & (PAGE_SIZE - 1);
|
| 1551 | to = from + len;
|
| 1552 |
|
| 1553 | BUG_ON(!ext4_handle_valid(handle));
|
| 1554 |
|
| 1555 | if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE)
|
| 1556 | return ext4_write_inline_data_end(inode, pos, len, copied,
|
| 1557 | folio);
|
| 1558 |
|
| 1559 | if (unlikely(copied < len) && !folio_test_uptodate(folio)) {
|
| 1560 | copied = 0;
|
| 1561 | ext4_journalled_zero_new_buffers(handle, inode, folio,
|
| 1562 | from, to);
|
| 1563 | } else {
|
| 1564 | if (unlikely(copied < len))
|
| 1565 | ext4_journalled_zero_new_buffers(handle, inode, folio,
|
| 1566 | from + copied, to);
|
| 1567 | ret = ext4_walk_page_buffers(handle, inode,
|
| 1568 | folio_buffers(folio),
|
| 1569 | from, from + copied, &partial,
|
| 1570 | write_end_fn);
|
| 1571 | if (!partial)
|
| 1572 | folio_mark_uptodate(folio);
|
| 1573 | }
|
| 1574 | if (!verity)
|
| 1575 | size_changed = ext4_update_inode_size(inode, pos + copied);
|
| 1576 | EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
|
| 1577 | folio_unlock(folio);
|
| 1578 | folio_put(folio);
|
| 1579 |
|
| 1580 | if (old_size < pos && !verity)
|
| 1581 | pagecache_isize_extended(inode, old_size, pos);
|
| 1582 |
|
| 1583 | if (size_changed) {
|
| 1584 | ret2 = ext4_mark_inode_dirty(handle, inode);
|
| 1585 | if (!ret)
|
| 1586 | ret = ret2;
|
| 1587 | }
|
| 1588 |
|
| 1589 | if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode))
|
| 1590 | |
| 1591 | * less. We will have blocks allocated outside
|
| 1592 | * inode->i_size. So truncate them
|
| 1593 |
|
| 1594 | ext4_orphan_add(handle, inode);
|
| 1595 |
|
| 1596 | ret2 = ext4_journal_stop(handle);
|
| 1597 | if (!ret)
|
| 1598 | ret = ret2;
|
| 1599 | if (pos + len > inode->i_size && !verity) {
|
| 1600 | ext4_truncate_failed_write(inode);
|
| 1601 | |
| 1602 | * If truncate failed early the inode might still be
|
| 1603 | * on the orphan list; we need to make sure the inode
|
| 1604 | * is removed from the orphan list in that case.
|
| 1605 |
|
| 1606 | if (inode->i_nlink)
|
| 1607 | ext4_orphan_del(NULL, inode);
|
| 1608 | }
|
| 1609 |
|
| 1610 | return ret ? ret : copied;
|
| 1611 | }
|
| 1612 |
|
| 1613 | |
| 1614 | * Reserve space for 'nr_resv' clusters
|
| 1615 |
|
| 1616 | static int ext4_da_reserve_space(struct inode *inode, int nr_resv)
|
| 1617 | {
|
| 1618 | struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
|
| 1619 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 1620 | int ret;
|
| 1621 |
|
| 1622 | |
| 1623 | * We will charge metadata quota at writeout time; this saves
|
| 1624 | * us from metadata over-estimation, though we may go over by
|
| 1625 | * a small amount in the end. Here we just reserve for data.
|
| 1626 |
|
| 1627 | ret = dquot_reserve_block(inode, EXT4_C2B(sbi, nr_resv));
|
| 1628 | if (ret)
|
| 1629 | return ret;
|
| 1630 |
|
| 1631 | spin_lock(&ei->i_block_reservation_lock);
|
| 1632 | if (ext4_claim_free_clusters(sbi, nr_resv, 0)) {
|
| 1633 | spin_unlock(&ei->i_block_reservation_lock);
|
| 1634 | dquot_release_reservation_block(inode, EXT4_C2B(sbi, nr_resv));
|
| 1635 | return -ENOSPC;
|
| 1636 | }
|
| 1637 | ei->i_reserved_data_blocks += nr_resv;
|
| 1638 | trace_ext4_da_reserve_space(inode, nr_resv);
|
| 1639 | spin_unlock(&ei->i_block_reservation_lock);
|
| 1640 |
|
| 1641 | return 0;
|
| 1642 | }
|
| 1643 |
|
| 1644 | void ext4_da_release_space(struct inode *inode, int to_free)
|
| 1645 | {
|
| 1646 | struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
|
| 1647 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 1648 |
|
| 1649 | if (!to_free)
|
| 1650 | return;
|
| 1651 |
|
| 1652 | spin_lock(&EXT4_I(inode)->i_block_reservation_lock);
|
| 1653 |
|
| 1654 | trace_ext4_da_release_space(inode, to_free);
|
| 1655 | if (unlikely(to_free > ei->i_reserved_data_blocks)) {
|
| 1656 | |
| 1657 | * if there aren't enough reserved blocks, then the
|
| 1658 | * counter is messed up somewhere. Since this
|
| 1659 | * function is called from invalidate page, it's
|
| 1660 | * harmless to return without any action.
|
| 1661 |
|
| 1662 | ext4_warning(inode->i_sb, "ext4_da_release_space: "
|
| 1663 | "ino %llu, to_free %d with only %d reserved "
|
| 1664 | "data blocks", inode->i_ino, to_free,
|
| 1665 | ei->i_reserved_data_blocks);
|
| 1666 | WARN_ON(1);
|
| 1667 | to_free = ei->i_reserved_data_blocks;
|
| 1668 | }
|
| 1669 | ei->i_reserved_data_blocks -= to_free;
|
| 1670 |
|
| 1671 |
|
| 1672 | percpu_counter_sub(&sbi->s_dirtyclusters_counter, to_free);
|
| 1673 |
|
| 1674 | spin_unlock(&EXT4_I(inode)->i_block_reservation_lock);
|
| 1675 |
|
| 1676 | dquot_release_reservation_block(inode, EXT4_C2B(sbi, to_free));
|
| 1677 | }
|
| 1678 |
|
| 1679 | |
| 1680 | * Delayed allocation stuff
|
| 1681 |
|
| 1682 |
|
| 1683 | struct mpage_da_data {
|
| 1684 |
|
| 1685 | struct inode *inode;
|
| 1686 | struct writeback_control *wbc;
|
| 1687 | unsigned int can_map:1;
|
| 1688 |
|
| 1689 |
|
| 1690 | loff_t start_pos;
|
| 1691 | loff_t next_pos;
|
| 1692 | loff_t end_pos;
|
| 1693 |
|
| 1694 | |
| 1695 | * Extent to map - this can be after start_pos because that can be
|
| 1696 | * fully mapped. We somewhat abuse m_flags to store whether the extent
|
| 1697 | * is delalloc or unwritten.
|
| 1698 |
|
| 1699 | struct ext4_map_blocks map;
|
| 1700 | struct ext4_io_submit io_submit;
|
| 1701 | unsigned int do_map:1;
|
| 1702 | unsigned int scanned_until_end:1;
|
| 1703 | unsigned int journalled_more_data:1;
|
| 1704 | };
|
| 1705 |
|
| 1706 | static void mpage_release_unused_pages(struct mpage_da_data *mpd,
|
| 1707 | bool invalidate)
|
| 1708 | {
|
| 1709 | unsigned nr, i;
|
| 1710 | pgoff_t index, end;
|
| 1711 | struct folio_batch fbatch;
|
| 1712 | struct inode *inode = mpd->inode;
|
| 1713 | struct address_space *mapping = inode->i_mapping;
|
| 1714 |
|
| 1715 |
|
| 1716 | if (mpd->start_pos >= mpd->next_pos)
|
| 1717 | return;
|
| 1718 |
|
| 1719 | mpd->scanned_until_end = 0;
|
| 1720 | if (invalidate) {
|
| 1721 | ext4_lblk_t start, last;
|
| 1722 | start = EXT4_B_TO_LBLK(inode, mpd->start_pos);
|
| 1723 | last = mpd->next_pos >> inode->i_blkbits;
|
| 1724 |
|
| 1725 | |
| 1726 | * avoid racing with extent status tree scans made by
|
| 1727 | * ext4_insert_delayed_block()
|
| 1728 |
|
| 1729 | down_write(&EXT4_I(inode)->i_data_sem);
|
| 1730 | ext4_es_remove_extent(inode, start, last - start);
|
| 1731 | up_write(&EXT4_I(inode)->i_data_sem);
|
| 1732 | }
|
| 1733 |
|
| 1734 | folio_batch_init(&fbatch);
|
| 1735 | index = mpd->start_pos >> PAGE_SHIFT;
|
| 1736 | end = mpd->next_pos >> PAGE_SHIFT;
|
| 1737 | while (index < end) {
|
| 1738 | nr = filemap_get_folios(mapping, &index, end - 1, &fbatch);
|
| 1739 | if (nr == 0)
|
| 1740 | break;
|
| 1741 | for (i = 0; i < nr; i++) {
|
| 1742 | struct folio *folio = fbatch.folios[i];
|
| 1743 |
|
| 1744 | if (folio_pos(folio) < mpd->start_pos)
|
| 1745 | continue;
|
| 1746 | if (folio_next_index(folio) > end)
|
| 1747 | continue;
|
| 1748 | BUG_ON(!folio_test_locked(folio));
|
| 1749 | BUG_ON(folio_test_writeback(folio));
|
| 1750 | if (invalidate) {
|
| 1751 | if (folio_mapped(folio)) {
|
| 1752 | folio_clear_dirty_for_io(folio);
|
| 1753 | |
| 1754 | * Unmap folio from page
|
| 1755 | * tables to prevent
|
| 1756 | * subsequent accesses through
|
| 1757 | * stale PTEs. This ensures
|
| 1758 | * future accesses trigger new
|
| 1759 | * page faults rather than
|
| 1760 | * reusing the invalidated
|
| 1761 | * folio.
|
| 1762 |
|
| 1763 | unmap_mapping_pages(folio->mapping,
|
| 1764 | folio->index,
|
| 1765 | folio_nr_pages(folio), false);
|
| 1766 | }
|
| 1767 | block_invalidate_folio(folio, 0,
|
| 1768 | folio_size(folio));
|
| 1769 | folio_clear_uptodate(folio);
|
| 1770 | }
|
| 1771 | folio_unlock(folio);
|
| 1772 | }
|
| 1773 | folio_batch_release(&fbatch);
|
| 1774 | }
|
| 1775 | }
|
| 1776 |
|
| 1777 | static void ext4_print_free_blocks(struct inode *inode)
|
| 1778 | {
|
| 1779 | struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
|
| 1780 | struct super_block *sb = inode->i_sb;
|
| 1781 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 1782 |
|
| 1783 | ext4_msg(sb, KERN_CRIT, "Total free blocks count %lld",
|
| 1784 | EXT4_C2B(EXT4_SB(inode->i_sb),
|
| 1785 | ext4_count_free_clusters(sb)));
|
| 1786 | ext4_msg(sb, KERN_CRIT, "Free/Dirty block details");
|
| 1787 | ext4_msg(sb, KERN_CRIT, "free_blocks=%lld",
|
| 1788 | (long long) EXT4_C2B(EXT4_SB(sb),
|
| 1789 | percpu_counter_sum(&sbi->s_freeclusters_counter)));
|
| 1790 | ext4_msg(sb, KERN_CRIT, "dirty_blocks=%lld",
|
| 1791 | (long long) EXT4_C2B(EXT4_SB(sb),
|
| 1792 | percpu_counter_sum(&sbi->s_dirtyclusters_counter)));
|
| 1793 | ext4_msg(sb, KERN_CRIT, "Block reservation details");
|
| 1794 | ext4_msg(sb, KERN_CRIT, "i_reserved_data_blocks=%u",
|
| 1795 | ei->i_reserved_data_blocks);
|
| 1796 | return;
|
| 1797 | }
|
| 1798 |
|
| 1799 | |
| 1800 | * Check whether the cluster containing lblk has been allocated or has
|
| 1801 | * delalloc reservation.
|
| 1802 | *
|
| 1803 | * Returns 0 if the cluster doesn't have either, 1 if it has delalloc
|
| 1804 | * reservation, 2 if it's already been allocated, negative error code on
|
| 1805 | * failure.
|
| 1806 |
|
| 1807 | static int ext4_clu_alloc_state(struct inode *inode, ext4_lblk_t lblk)
|
| 1808 | {
|
| 1809 | struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
|
| 1810 | int ret;
|
| 1811 |
|
| 1812 |
|
| 1813 | if (ext4_es_scan_clu(inode, &ext4_es_is_delayed, lblk))
|
| 1814 | return 1;
|
| 1815 |
|
| 1816 |
|
| 1817 | if (ext4_es_scan_clu(inode, &ext4_es_is_mapped, lblk))
|
| 1818 | return 2;
|
| 1819 | ret = ext4_clu_mapped(inode, EXT4_B2C(sbi, lblk));
|
| 1820 | if (ret < 0)
|
| 1821 | return ret;
|
| 1822 | if (ret > 0)
|
| 1823 | return 2;
|
| 1824 |
|
| 1825 | return 0;
|
| 1826 | }
|
| 1827 |
|
| 1828 | |
| 1829 | * ext4_insert_delayed_blocks - adds a multiple delayed blocks to the extents
|
| 1830 | * status tree, incrementing the reserved
|
| 1831 | * cluster/block count or making pending
|
| 1832 | * reservations where needed
|
| 1833 | *
|
| 1834 | * @inode - file containing the newly added block
|
| 1835 | * @lblk - start logical block to be added
|
| 1836 | * @len - length of blocks to be added
|
| 1837 | *
|
| 1838 | * Returns 0 on success, negative error code on failure.
|
| 1839 |
|
| 1840 | static int ext4_insert_delayed_blocks(struct inode *inode, ext4_lblk_t lblk,
|
| 1841 | ext4_lblk_t len)
|
| 1842 | {
|
| 1843 | struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
|
| 1844 | int ret;
|
| 1845 | bool lclu_allocated = false;
|
| 1846 | bool end_allocated = false;
|
| 1847 | ext4_lblk_t resv_clu;
|
| 1848 | ext4_lblk_t end = lblk + len - 1;
|
| 1849 |
|
| 1850 | |
| 1851 | * If the cluster containing lblk or end is shared with a delayed,
|
| 1852 | * written, or unwritten extent in a bigalloc file system, it's
|
| 1853 | * already been accounted for and does not need to be reserved.
|
| 1854 | * A pending reservation must be made for the cluster if it's
|
| 1855 | * shared with a written or unwritten extent and doesn't already
|
| 1856 | * have one. Written and unwritten extents can be purged from the
|
| 1857 | * extents status tree if the system is under memory pressure, so
|
| 1858 | * it's necessary to examine the extent tree if a search of the
|
| 1859 | * extents status tree doesn't get a match.
|
| 1860 |
|
| 1861 | if (sbi->s_cluster_ratio == 1) {
|
| 1862 | ret = ext4_da_reserve_space(inode, len);
|
| 1863 | if (ret != 0)
|
| 1864 | return ret;
|
| 1865 | } else {
|
| 1866 | resv_clu = EXT4_B2C(sbi, end) - EXT4_B2C(sbi, lblk) + 1;
|
| 1867 |
|
| 1868 | ret = ext4_clu_alloc_state(inode, lblk);
|
| 1869 | if (ret < 0)
|
| 1870 | return ret;
|
| 1871 | if (ret > 0) {
|
| 1872 | resv_clu--;
|
| 1873 | lclu_allocated = (ret == 2);
|
| 1874 | }
|
| 1875 |
|
| 1876 | if (EXT4_B2C(sbi, lblk) != EXT4_B2C(sbi, end)) {
|
| 1877 | ret = ext4_clu_alloc_state(inode, end);
|
| 1878 | if (ret < 0)
|
| 1879 | return ret;
|
| 1880 | if (ret > 0) {
|
| 1881 | resv_clu--;
|
| 1882 | end_allocated = (ret == 2);
|
| 1883 | }
|
| 1884 | }
|
| 1885 |
|
| 1886 | if (resv_clu) {
|
| 1887 | ret = ext4_da_reserve_space(inode, resv_clu);
|
| 1888 | if (ret != 0)
|
| 1889 | return ret;
|
| 1890 | }
|
| 1891 | }
|
| 1892 |
|
| 1893 | ext4_es_insert_delayed_extent(inode, lblk, len, lclu_allocated,
|
| 1894 | end_allocated);
|
| 1895 | return 0;
|
| 1896 | }
|
| 1897 |
|
| 1898 | |
| 1899 | * Looks up the requested blocks and sets the delalloc extent map.
|
| 1900 | * First try to look up for the extent entry that contains the requested
|
| 1901 | * blocks in the extent status tree without i_data_sem, then try to look
|
| 1902 | * up for the ondisk extent mapping with i_data_sem in read mode,
|
| 1903 | * finally hold i_data_sem in write mode, looks up again and add a
|
| 1904 | * delalloc extent entry if it still couldn't find any extent. Pass out
|
| 1905 | * the mapped extent through @map and return 0 on success.
|
| 1906 |
|
| 1907 | static int ext4_da_map_blocks(struct inode *inode, struct ext4_map_blocks *map)
|
| 1908 | {
|
| 1909 | struct extent_status es;
|
| 1910 | int retval;
|
| 1911 | #ifdef ES_AGGRESSIVE_TEST
|
| 1912 | struct ext4_map_blocks orig_map;
|
| 1913 |
|
| 1914 | memcpy(&orig_map, map, sizeof(*map));
|
| 1915 | #endif
|
| 1916 |
|
| 1917 | map->m_flags = 0;
|
| 1918 | ext_debug(inode, "max_blocks %u, logical block %lu\n", map->m_len,
|
| 1919 | (unsigned long) map->m_lblk);
|
| 1920 |
|
| 1921 | ext4_check_map_extents_env(inode);
|
| 1922 |
|
| 1923 |
|
| 1924 | if (ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es, NULL)) {
|
| 1925 | map->m_len = min_t(unsigned int, map->m_len,
|
| 1926 | es.es_len - (map->m_lblk - es.es_lblk));
|
| 1927 |
|
| 1928 | if (ext4_es_is_hole(&es))
|
| 1929 | goto add_delayed;
|
| 1930 |
|
| 1931 | found:
|
| 1932 | |
| 1933 | * Delayed extent could be allocated by fallocate.
|
| 1934 | * So we need to check it.
|
| 1935 |
|
| 1936 | if (ext4_es_is_delayed(&es)) {
|
| 1937 | map->m_flags |= EXT4_MAP_DELAYED;
|
| 1938 | return 0;
|
| 1939 | }
|
| 1940 |
|
| 1941 | map->m_pblk = ext4_es_pblock(&es) + map->m_lblk - es.es_lblk;
|
| 1942 | if (ext4_es_is_written(&es))
|
| 1943 | map->m_flags |= EXT4_MAP_MAPPED;
|
| 1944 | else if (ext4_es_is_unwritten(&es))
|
| 1945 | map->m_flags |= EXT4_MAP_UNWRITTEN;
|
| 1946 | else
|
| 1947 | BUG();
|
| 1948 |
|
| 1949 | #ifdef ES_AGGRESSIVE_TEST
|
| 1950 | ext4_map_blocks_es_recheck(NULL, inode, map, &orig_map, 0);
|
| 1951 | #endif
|
| 1952 | return 0;
|
| 1953 | }
|
| 1954 |
|
| 1955 | |
| 1956 | * Try to see if we can get the block without requesting a new
|
| 1957 | * file system block.
|
| 1958 |
|
| 1959 | down_read(&EXT4_I(inode)->i_data_sem);
|
| 1960 | if (ext4_has_inline_data(inode))
|
| 1961 | retval = 0;
|
| 1962 | else
|
| 1963 | retval = ext4_map_query_blocks(NULL, inode, map, 0);
|
| 1964 | up_read(&EXT4_I(inode)->i_data_sem);
|
| 1965 | if (retval)
|
| 1966 | return retval < 0 ? retval : 0;
|
| 1967 |
|
| 1968 | add_delayed:
|
| 1969 | down_write(&EXT4_I(inode)->i_data_sem);
|
| 1970 | |
| 1971 | * Page fault path (ext4_page_mkwrite does not take i_rwsem)
|
| 1972 | * and fallocate path (no folio lock) can race. Make sure we
|
| 1973 | * lookup the extent status tree here again while i_data_sem
|
| 1974 | * is held in write mode, before inserting a new da entry in
|
| 1975 | * the extent status tree.
|
| 1976 |
|
| 1977 | if (ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es, NULL)) {
|
| 1978 | map->m_len = min_t(unsigned int, map->m_len,
|
| 1979 | es.es_len - (map->m_lblk - es.es_lblk));
|
| 1980 |
|
| 1981 | if (!ext4_es_is_hole(&es)) {
|
| 1982 | up_write(&EXT4_I(inode)->i_data_sem);
|
| 1983 | goto found;
|
| 1984 | }
|
| 1985 | } else if (!ext4_has_inline_data(inode)) {
|
| 1986 | retval = ext4_map_query_blocks(NULL, inode, map, 0);
|
| 1987 | if (retval) {
|
| 1988 | up_write(&EXT4_I(inode)->i_data_sem);
|
| 1989 | return retval < 0 ? retval : 0;
|
| 1990 | }
|
| 1991 | }
|
| 1992 |
|
| 1993 | map->m_flags |= EXT4_MAP_DELAYED;
|
| 1994 | retval = ext4_insert_delayed_blocks(inode, map->m_lblk, map->m_len);
|
| 1995 | if (!retval)
|
| 1996 | map->m_seq = READ_ONCE(EXT4_I(inode)->i_es_seq);
|
| 1997 | up_write(&EXT4_I(inode)->i_data_sem);
|
| 1998 |
|
| 1999 | return retval;
|
| 2000 | }
|
| 2001 |
|
| 2002 | |
| 2003 | * This is a special get_block_t callback which is used by
|
| 2004 | * ext4_da_write_begin(). It will either return mapped block or
|
| 2005 | * reserve space for a single block.
|
| 2006 | *
|
| 2007 | * For delayed buffer_head we have BH_Mapped, BH_New, BH_Delay set.
|
| 2008 | * We also have b_blocknr = -1 and b_bdev initialized properly
|
| 2009 | *
|
| 2010 | * For unwritten buffer_head we have BH_Mapped, BH_New, BH_Unwritten set.
|
| 2011 | * We also have b_blocknr = physicalblock mapping unwritten extent and b_bdev
|
| 2012 | * initialized properly.
|
| 2013 |
|
| 2014 | int ext4_da_get_block_prep(struct inode *inode, sector_t iblock,
|
| 2015 | struct buffer_head *bh, int create)
|
| 2016 | {
|
| 2017 | struct ext4_map_blocks map;
|
| 2018 | sector_t invalid_block = ~((sector_t) 0xffff);
|
| 2019 | int ret = 0;
|
| 2020 |
|
| 2021 | BUG_ON(create == 0);
|
| 2022 | BUG_ON(bh->b_size != inode->i_sb->s_blocksize);
|
| 2023 |
|
| 2024 | if (invalid_block < ext4_blocks_count(EXT4_SB(inode->i_sb)->s_es))
|
| 2025 | invalid_block = ~0;
|
| 2026 |
|
| 2027 | map.m_lblk = iblock;
|
| 2028 | map.m_len = 1;
|
| 2029 |
|
| 2030 | |
| 2031 | * first, we need to know whether the block is allocated already
|
| 2032 | * preallocated blocks are unmapped but should treated
|
| 2033 | * the same as allocated blocks.
|
| 2034 |
|
| 2035 | ret = ext4_da_map_blocks(inode, &map);
|
| 2036 | if (ret < 0)
|
| 2037 | return ret;
|
| 2038 |
|
| 2039 | if (map.m_flags & EXT4_MAP_DELAYED) {
|
| 2040 | map_bh(bh, inode->i_sb, invalid_block);
|
| 2041 | set_buffer_new(bh);
|
| 2042 | set_buffer_delay(bh);
|
| 2043 | return 0;
|
| 2044 | }
|
| 2045 |
|
| 2046 | map_bh(bh, inode->i_sb, map.m_pblk);
|
| 2047 | ext4_update_bh_state(bh, map.m_flags);
|
| 2048 |
|
| 2049 | if (buffer_unwritten(bh)) {
|
| 2050 | |
| 2051 | * new and mapped. Mapped ensures that we don't do
|
| 2052 | * get_block multiple times when we write to the same
|
| 2053 | * offset and new ensures that we do proper zero out
|
| 2054 | * for partial write.
|
| 2055 |
|
| 2056 | set_buffer_new(bh);
|
| 2057 | set_buffer_mapped(bh);
|
| 2058 | }
|
| 2059 | return 0;
|
| 2060 | }
|
| 2061 |
|
| 2062 | static void mpage_folio_done(struct mpage_da_data *mpd, struct folio *folio)
|
| 2063 | {
|
| 2064 | mpd->start_pos += folio_size(folio);
|
| 2065 | mpd->wbc->nr_to_write -= folio_nr_pages(folio);
|
| 2066 | folio_unlock(folio);
|
| 2067 | }
|
| 2068 |
|
| 2069 | static void mpage_submit_folio(struct mpage_da_data *mpd, struct folio *folio)
|
| 2070 | {
|
| 2071 | size_t len;
|
| 2072 | loff_t size;
|
| 2073 |
|
| 2074 | WARN_ON_ONCE(folio_pos(folio) != mpd->start_pos);
|
| 2075 | folio_clear_dirty_for_io(folio);
|
| 2076 | |
| 2077 | * We have to be very careful here! Nothing protects writeback path
|
| 2078 | * against i_size changes and the page can be writeably mapped into
|
| 2079 | * page tables. So an application can be growing i_size and writing
|
| 2080 | * data through mmap while writeback runs. folio_clear_dirty_for_io()
|
| 2081 | * write-protects our page in page tables and the page cannot get
|
| 2082 | * written to again until we release folio lock. So only after
|
| 2083 | * folio_clear_dirty_for_io() we are safe to sample i_size for
|
| 2084 | * ext4_bio_write_folio() to zero-out tail of the written page. We rely
|
| 2085 | * on the barrier provided by folio_test_clear_dirty() in
|
| 2086 | * folio_clear_dirty_for_io() to make sure i_size is really sampled only
|
| 2087 | * after page tables are updated.
|
| 2088 |
|
| 2089 | size = i_size_read(mpd->inode);
|
| 2090 | len = folio_size(folio);
|
| 2091 | if (folio_pos(folio) + len > size &&
|
| 2092 | !ext4_verity_in_progress(mpd->inode))
|
| 2093 | len = size & (len - 1);
|
| 2094 | ext4_bio_write_folio(&mpd->io_submit, folio, len);
|
| 2095 | }
|
| 2096 |
|
| 2097 | #define BH_FLAGS (BIT(BH_Unwritten) | BIT(BH_Delay))
|
| 2098 |
|
| 2099 | |
| 2100 | * mballoc gives us at most this number of blocks...
|
| 2101 | * XXX: That seems to be only a limitation of ext4_mb_normalize_request().
|
| 2102 | * The rest of mballoc seems to handle chunks up to full group size.
|
| 2103 |
|
| 2104 | #define MAX_WRITEPAGES_EXTENT_LEN 2048
|
| 2105 |
|
| 2106 | |
| 2107 | * mpage_add_bh_to_extent - try to add bh to extent of blocks to map
|
| 2108 | *
|
| 2109 | * @mpd - extent of blocks
|
| 2110 | * @lblk - logical number of the block in the file
|
| 2111 | * @bh - buffer head we want to add to the extent
|
| 2112 | *
|
| 2113 | * The function is used to collect contig. blocks in the same state. If the
|
| 2114 | * buffer doesn't require mapping for writeback and we haven't started the
|
| 2115 | * extent of buffers to map yet, the function returns 'true' immediately - the
|
| 2116 | * caller can write the buffer right away. Otherwise the function returns true
|
| 2117 | * if the block has been added to the extent, false if the block couldn't be
|
| 2118 | * added.
|
| 2119 |
|
| 2120 | static bool mpage_add_bh_to_extent(struct mpage_da_data *mpd, ext4_lblk_t lblk,
|
| 2121 | struct buffer_head *bh)
|
| 2122 | {
|
| 2123 | struct ext4_map_blocks *map = &mpd->map;
|
| 2124 |
|
| 2125 |
|
| 2126 | if (!buffer_dirty(bh) || !buffer_mapped(bh) ||
|
| 2127 | (!buffer_delay(bh) && !buffer_unwritten(bh))) {
|
| 2128 |
|
| 2129 | if (map->m_len == 0)
|
| 2130 | return true;
|
| 2131 | return false;
|
| 2132 | }
|
| 2133 |
|
| 2134 |
|
| 2135 | if (map->m_len == 0) {
|
| 2136 |
|
| 2137 | if (!mpd->do_map)
|
| 2138 | return false;
|
| 2139 | map->m_lblk = lblk;
|
| 2140 | map->m_len = 1;
|
| 2141 | map->m_flags = bh->b_state & BH_FLAGS;
|
| 2142 | return true;
|
| 2143 | }
|
| 2144 |
|
| 2145 |
|
| 2146 | if (map->m_len >= MAX_WRITEPAGES_EXTENT_LEN)
|
| 2147 | return false;
|
| 2148 |
|
| 2149 |
|
| 2150 | if (lblk == map->m_lblk + map->m_len &&
|
| 2151 | (bh->b_state & BH_FLAGS) == map->m_flags) {
|
| 2152 | map->m_len++;
|
| 2153 | return true;
|
| 2154 | }
|
| 2155 | return false;
|
| 2156 | }
|
| 2157 |
|
| 2158 | |
| 2159 | * mpage_process_page_bufs - submit page buffers for IO or add them to extent
|
| 2160 | *
|
| 2161 | * @mpd - extent of blocks for mapping
|
| 2162 | * @head - the first buffer in the page
|
| 2163 | * @bh - buffer we should start processing from
|
| 2164 | * @lblk - logical number of the block in the file corresponding to @bh
|
| 2165 | *
|
| 2166 | * Walk through page buffers from @bh upto @head (exclusive) and either submit
|
| 2167 | * the page for IO if all buffers in this page were mapped and there's no
|
| 2168 | * accumulated extent of buffers to map or add buffers in the page to the
|
| 2169 | * extent of buffers to map. The function returns 1 if the caller can continue
|
| 2170 | * by processing the next page, 0 if it should stop adding buffers to the
|
| 2171 | * extent to map because we cannot extend it anymore.
|
| 2172 |
|
| 2173 | static int mpage_process_page_bufs(struct mpage_da_data *mpd,
|
| 2174 | struct buffer_head *head,
|
| 2175 | struct buffer_head *bh,
|
| 2176 | ext4_lblk_t lblk)
|
| 2177 | {
|
| 2178 | struct inode *inode = mpd->inode;
|
| 2179 | ext4_lblk_t blocks = (i_size_read(inode) + i_blocksize(inode) - 1)
|
| 2180 | >> inode->i_blkbits;
|
| 2181 |
|
| 2182 | if (ext4_verity_in_progress(inode))
|
| 2183 | blocks = EXT_MAX_BLOCKS;
|
| 2184 |
|
| 2185 | do {
|
| 2186 | BUG_ON(buffer_locked(bh));
|
| 2187 |
|
| 2188 | if (lblk >= blocks || !mpage_add_bh_to_extent(mpd, lblk, bh)) {
|
| 2189 |
|
| 2190 | if (mpd->map.m_len)
|
| 2191 | return 0;
|
| 2192 |
|
| 2193 | if (!mpd->do_map)
|
| 2194 | return 0;
|
| 2195 |
|
| 2196 | break;
|
| 2197 | }
|
| 2198 | } while (lblk++, (bh = bh->b_this_page) != head);
|
| 2199 |
|
| 2200 | if (mpd->map.m_len == 0) {
|
| 2201 | mpage_submit_folio(mpd, head->b_folio);
|
| 2202 | mpage_folio_done(mpd, head->b_folio);
|
| 2203 | }
|
| 2204 | if (lblk >= blocks) {
|
| 2205 | mpd->scanned_until_end = 1;
|
| 2206 | return 0;
|
| 2207 | }
|
| 2208 | return 1;
|
| 2209 | }
|
| 2210 |
|
| 2211 | |
| 2212 | * mpage_process_folio - update folio buffers corresponding to changed extent
|
| 2213 | * and may submit fully mapped page for IO
|
| 2214 | * @mpd: description of extent to map, on return next extent to map
|
| 2215 | * @folio: Contains these buffers.
|
| 2216 | * @m_lblk: logical block mapping.
|
| 2217 | * @m_pblk: corresponding physical mapping.
|
| 2218 | * @map_bh: determines on return whether this page requires any further
|
| 2219 | * mapping or not.
|
| 2220 | *
|
| 2221 | * Scan given folio buffers corresponding to changed extent and update buffer
|
| 2222 | * state according to new extent state.
|
| 2223 | * We map delalloc buffers to their physical location, clear unwritten bits.
|
| 2224 | * If the given folio is not fully mapped, we update @mpd to the next extent in
|
| 2225 | * the given folio that needs mapping & return @map_bh as true.
|
| 2226 |
|
| 2227 | static int mpage_process_folio(struct mpage_da_data *mpd, struct folio *folio,
|
| 2228 | ext4_lblk_t *m_lblk, ext4_fsblk_t *m_pblk,
|
| 2229 | bool *map_bh)
|
| 2230 | {
|
| 2231 | struct buffer_head *head, *bh;
|
| 2232 | ext4_io_end_t *io_end = mpd->io_submit.io_end;
|
| 2233 | ext4_lblk_t lblk = *m_lblk;
|
| 2234 | ext4_fsblk_t pblock = *m_pblk;
|
| 2235 | int err = 0;
|
| 2236 | ssize_t io_end_size = 0;
|
| 2237 | struct ext4_io_end_vec *io_end_vec = ext4_last_io_end_vec(io_end);
|
| 2238 |
|
| 2239 | bh = head = folio_buffers(folio);
|
| 2240 | do {
|
| 2241 | if (lblk < mpd->map.m_lblk)
|
| 2242 | continue;
|
| 2243 | if (lblk >= mpd->map.m_lblk + mpd->map.m_len) {
|
| 2244 | |
| 2245 | * Buffer after end of mapped extent.
|
| 2246 | * Find next buffer in the folio to map.
|
| 2247 |
|
| 2248 | mpd->map.m_len = 0;
|
| 2249 | mpd->map.m_flags = 0;
|
| 2250 | io_end_vec->size += io_end_size;
|
| 2251 |
|
| 2252 | err = mpage_process_page_bufs(mpd, head, bh, lblk);
|
| 2253 | if (err > 0)
|
| 2254 | err = 0;
|
| 2255 | if (!err && mpd->map.m_len && mpd->map.m_lblk > lblk) {
|
| 2256 | io_end_vec = ext4_alloc_io_end_vec(io_end);
|
| 2257 | if (IS_ERR(io_end_vec)) {
|
| 2258 | err = PTR_ERR(io_end_vec);
|
| 2259 | goto out;
|
| 2260 | }
|
| 2261 | io_end_vec->offset = EXT4_LBLK_TO_B(mpd->inode,
|
| 2262 | mpd->map.m_lblk);
|
| 2263 | }
|
| 2264 | *map_bh = true;
|
| 2265 | goto out;
|
| 2266 | }
|
| 2267 | if (buffer_delay(bh)) {
|
| 2268 | clear_buffer_delay(bh);
|
| 2269 | bh->b_blocknr = pblock++;
|
| 2270 | }
|
| 2271 | clear_buffer_unwritten(bh);
|
| 2272 | io_end_size += i_blocksize(mpd->inode);
|
| 2273 | } while (lblk++, (bh = bh->b_this_page) != head);
|
| 2274 |
|
| 2275 | io_end_vec->size += io_end_size;
|
| 2276 | *map_bh = false;
|
| 2277 | out:
|
| 2278 | *m_lblk = lblk;
|
| 2279 | *m_pblk = pblock;
|
| 2280 | return err;
|
| 2281 | }
|
| 2282 |
|
| 2283 | |
| 2284 | * mpage_map_buffers - update buffers corresponding to changed extent and
|
| 2285 | * submit fully mapped pages for IO
|
| 2286 | *
|
| 2287 | * @mpd - description of extent to map, on return next extent to map
|
| 2288 | *
|
| 2289 | * Scan buffers corresponding to changed extent (we expect corresponding pages
|
| 2290 | * to be already locked) and update buffer state according to new extent state.
|
| 2291 | * We map delalloc buffers to their physical location, clear unwritten bits,
|
| 2292 | * and mark buffers as uninit when we perform writes to unwritten extents
|
| 2293 | * and do extent conversion after IO is finished. If the last page is not fully
|
| 2294 | * mapped, we update @map to the next extent in the last page that needs
|
| 2295 | * mapping. Otherwise we submit the page for IO.
|
| 2296 |
|
| 2297 | static int mpage_map_and_submit_buffers(struct mpage_da_data *mpd)
|
| 2298 | {
|
| 2299 | struct folio_batch fbatch;
|
| 2300 | unsigned nr, i;
|
| 2301 | struct inode *inode = mpd->inode;
|
| 2302 | pgoff_t start, end;
|
| 2303 | ext4_lblk_t lblk;
|
| 2304 | ext4_fsblk_t pblock;
|
| 2305 | int err;
|
| 2306 | bool map_bh = false;
|
| 2307 |
|
| 2308 | start = EXT4_LBLK_TO_PG(inode, mpd->map.m_lblk);
|
| 2309 | end = EXT4_LBLK_TO_PG(inode, mpd->map.m_lblk + mpd->map.m_len - 1);
|
| 2310 | pblock = mpd->map.m_pblk;
|
| 2311 |
|
| 2312 | folio_batch_init(&fbatch);
|
| 2313 | while (start <= end) {
|
| 2314 | nr = filemap_get_folios(inode->i_mapping, &start, end, &fbatch);
|
| 2315 | if (nr == 0)
|
| 2316 | break;
|
| 2317 | for (i = 0; i < nr; i++) {
|
| 2318 | struct folio *folio = fbatch.folios[i];
|
| 2319 |
|
| 2320 | lblk = EXT4_PG_TO_LBLK(inode, folio->index);
|
| 2321 | err = mpage_process_folio(mpd, folio, &lblk, &pblock,
|
| 2322 | &map_bh);
|
| 2323 | |
| 2324 | * If map_bh is true, means page may require further bh
|
| 2325 | * mapping, or maybe the page was submitted for IO.
|
| 2326 | * So we return to call further extent mapping.
|
| 2327 |
|
| 2328 | if (err < 0 || map_bh)
|
| 2329 | goto out;
|
| 2330 |
|
| 2331 | mpage_submit_folio(mpd, folio);
|
| 2332 | mpage_folio_done(mpd, folio);
|
| 2333 | }
|
| 2334 | folio_batch_release(&fbatch);
|
| 2335 | }
|
| 2336 |
|
| 2337 | mpd->map.m_len = 0;
|
| 2338 | mpd->map.m_flags = 0;
|
| 2339 | return 0;
|
| 2340 | out:
|
| 2341 | folio_batch_release(&fbatch);
|
| 2342 | return err;
|
| 2343 | }
|
| 2344 |
|
| 2345 | static int mpage_map_one_extent(handle_t *handle, struct mpage_da_data *mpd)
|
| 2346 | {
|
| 2347 | struct inode *inode = mpd->inode;
|
| 2348 | struct ext4_map_blocks *map = &mpd->map;
|
| 2349 | int get_blocks_flags;
|
| 2350 | int err, dioread_nolock;
|
| 2351 |
|
| 2352 |
|
| 2353 | err = ext4_journal_ensure_extent_credits(handle, inode);
|
| 2354 | if (err < 0)
|
| 2355 | return err;
|
| 2356 |
|
| 2357 | trace_ext4_da_write_pages_extent(inode, map);
|
| 2358 | |
| 2359 | * Call ext4_map_blocks() to allocate any delayed allocation blocks, or
|
| 2360 | * to convert an unwritten extent to be initialized (in the case
|
| 2361 | * where we have written into one or more preallocated blocks). It is
|
| 2362 | * possible that we're going to need more metadata blocks than
|
| 2363 | * previously reserved. However we must not fail because we're in
|
| 2364 | * writeback and there is nothing we can do about it so it might result
|
| 2365 | * in data loss. So use reserved blocks to allocate metadata if
|
| 2366 | * possible. In addition, do not cache any unrelated extents, as it
|
| 2367 | * only holds the folio lock but does not hold the i_rwsem or
|
| 2368 | * invalidate_lock, which could corrupt the extent status tree.
|
| 2369 |
|
| 2370 | get_blocks_flags = EXT4_GET_BLOCKS_CREATE |
|
| 2371 | EXT4_GET_BLOCKS_METADATA_NOFAIL |
|
| 2372 | EXT4_GET_BLOCKS_IO_SUBMIT |
|
| 2373 | EXT4_EX_NOCACHE;
|
| 2374 |
|
| 2375 | dioread_nolock = ext4_should_dioread_nolock(inode);
|
| 2376 | if (dioread_nolock)
|
| 2377 | get_blocks_flags |= EXT4_GET_BLOCKS_UNWRIT_EXT;
|
| 2378 |
|
| 2379 | err = ext4_map_blocks(handle, inode, map, get_blocks_flags);
|
| 2380 | if (err < 0)
|
| 2381 | return err;
|
| 2382 | if (dioread_nolock && (map->m_flags & EXT4_MAP_UNWRITTEN)) {
|
| 2383 | if (!mpd->io_submit.io_end->handle &&
|
| 2384 | ext4_handle_valid(handle)) {
|
| 2385 | mpd->io_submit.io_end->handle = handle->h_rsv_handle;
|
| 2386 | handle->h_rsv_handle = NULL;
|
| 2387 | }
|
| 2388 | ext4_set_io_unwritten_flag(mpd->io_submit.io_end);
|
| 2389 | }
|
| 2390 |
|
| 2391 | BUG_ON(map->m_len == 0);
|
| 2392 | return 0;
|
| 2393 | }
|
| 2394 |
|
| 2395 | |
| 2396 | * This is used to submit mapped buffers in a single folio that is not fully
|
| 2397 | * mapped for various reasons, such as insufficient space or journal credits.
|
| 2398 |
|
| 2399 | static int mpage_submit_partial_folio(struct mpage_da_data *mpd)
|
| 2400 | {
|
| 2401 | struct inode *inode = mpd->inode;
|
| 2402 | struct folio *folio;
|
| 2403 | loff_t pos;
|
| 2404 |
|
| 2405 | folio = filemap_get_folio(inode->i_mapping,
|
| 2406 | mpd->start_pos >> PAGE_SHIFT);
|
| 2407 | if (IS_ERR(folio))
|
| 2408 | return PTR_ERR(folio);
|
| 2409 | |
| 2410 | * The mapped position should be within the current processing folio
|
| 2411 | * but must not be the folio start position.
|
| 2412 |
|
| 2413 | pos = ((loff_t)mpd->map.m_lblk) << inode->i_blkbits;
|
| 2414 | if (WARN_ON_ONCE((folio_pos(folio) == pos) ||
|
| 2415 | !folio_contains(folio, pos >> PAGE_SHIFT)))
|
| 2416 | return -EINVAL;
|
| 2417 |
|
| 2418 | mpage_submit_folio(mpd, folio);
|
| 2419 | |
| 2420 | * Update start_pos to prevent this folio from being released in
|
| 2421 | * mpage_release_unused_pages(), it will be reset to the aligned folio
|
| 2422 | * pos when this folio is written again in the next round. Additionally,
|
| 2423 | * do not update wbc->nr_to_write here, as it will be updated once the
|
| 2424 | * entire folio has finished processing.
|
| 2425 |
|
| 2426 | mpd->start_pos = pos;
|
| 2427 | folio_unlock(folio);
|
| 2428 | folio_put(folio);
|
| 2429 | return 0;
|
| 2430 | }
|
| 2431 |
|
| 2432 | |
| 2433 | * mpage_map_and_submit_extent - map extent starting at mpd->lblk of length
|
| 2434 | * mpd->len and submit pages underlying it for IO
|
| 2435 | *
|
| 2436 | * @handle - handle for journal operations
|
| 2437 | * @mpd - extent to map
|
| 2438 | * @give_up_on_write - we set this to true iff there is a fatal error and there
|
| 2439 | * is no hope of writing the data. The caller should discard
|
| 2440 | * dirty pages to avoid infinite loops.
|
| 2441 | *
|
| 2442 | * The function maps extent starting at mpd->lblk of length mpd->len. If it is
|
| 2443 | * delayed, blocks are allocated, if it is unwritten, we may need to convert
|
| 2444 | * them to initialized or split the described range from larger unwritten
|
| 2445 | * extent. Note that we need not map all the described range since allocation
|
| 2446 | * can return less blocks or the range is covered by more unwritten extents. We
|
| 2447 | * cannot map more because we are limited by reserved transaction credits. On
|
| 2448 | * the other hand we always make sure that the last touched page is fully
|
| 2449 | * mapped so that it can be written out (and thus forward progress is
|
| 2450 | * guaranteed). After mapping we submit all mapped pages for IO.
|
| 2451 |
|
| 2452 | static int mpage_map_and_submit_extent(handle_t *handle,
|
| 2453 | struct mpage_da_data *mpd,
|
| 2454 | bool *give_up_on_write)
|
| 2455 | {
|
| 2456 | struct inode *inode = mpd->inode;
|
| 2457 | struct ext4_map_blocks *map = &mpd->map;
|
| 2458 | int err;
|
| 2459 | loff_t disksize;
|
| 2460 | int progress = 0;
|
| 2461 | ext4_io_end_t *io_end = mpd->io_submit.io_end;
|
| 2462 | struct ext4_io_end_vec *io_end_vec;
|
| 2463 |
|
| 2464 | io_end_vec = ext4_alloc_io_end_vec(io_end);
|
| 2465 | if (IS_ERR(io_end_vec))
|
| 2466 | return PTR_ERR(io_end_vec);
|
| 2467 | io_end_vec->offset = EXT4_LBLK_TO_B(inode, map->m_lblk);
|
| 2468 | do {
|
| 2469 | err = mpage_map_one_extent(handle, mpd);
|
| 2470 | if (err < 0) {
|
| 2471 | struct super_block *sb = inode->i_sb;
|
| 2472 |
|
| 2473 | if (ext4_emergency_state(sb))
|
| 2474 | goto invalidate_dirty_pages;
|
| 2475 | |
| 2476 | * Let the uper layers retry transient errors.
|
| 2477 | * In the case of ENOSPC, if ext4_count_free_blocks()
|
| 2478 | * is non-zero, a commit should free up blocks.
|
| 2479 |
|
| 2480 | if ((err == -ENOMEM) || (err == -EAGAIN) ||
|
| 2481 | (err == -ENOSPC && ext4_count_free_clusters(sb))) {
|
| 2482 | |
| 2483 | * We may have already allocated extents for
|
| 2484 | * some bhs inside the folio, issue the
|
| 2485 | * corresponding data to prevent stale data.
|
| 2486 |
|
| 2487 | if (progress) {
|
| 2488 | if (mpage_submit_partial_folio(mpd))
|
| 2489 | goto invalidate_dirty_pages;
|
| 2490 | goto update_disksize;
|
| 2491 | }
|
| 2492 | return err;
|
| 2493 | }
|
| 2494 | ext4_msg(sb, KERN_CRIT,
|
| 2495 | "Delayed block allocation failed for "
|
| 2496 | "inode %llu at logical offset %llu with"
|
| 2497 | " max blocks %u with error %d",
|
| 2498 | inode->i_ino,
|
| 2499 | (unsigned long long)map->m_lblk,
|
| 2500 | (unsigned)map->m_len, -err);
|
| 2501 | ext4_msg(sb, KERN_CRIT,
|
| 2502 | "This should not happen!! Data will "
|
| 2503 | "be lost\n");
|
| 2504 | if (err == -ENOSPC)
|
| 2505 | ext4_print_free_blocks(inode);
|
| 2506 | invalidate_dirty_pages:
|
| 2507 | *give_up_on_write = true;
|
| 2508 | return err;
|
| 2509 | }
|
| 2510 | progress = 1;
|
| 2511 | |
| 2512 | * Update buffer state, submit mapped pages, and get us new
|
| 2513 | * extent to map
|
| 2514 |
|
| 2515 | err = mpage_map_and_submit_buffers(mpd);
|
| 2516 | if (err < 0)
|
| 2517 | goto update_disksize;
|
| 2518 | } while (map->m_len);
|
| 2519 |
|
| 2520 | update_disksize:
|
| 2521 | |
| 2522 | * Update on-disk size after IO is submitted. Races with
|
| 2523 | * truncate are avoided by checking i_size under i_data_sem.
|
| 2524 |
|
| 2525 | disksize = mpd->start_pos;
|
| 2526 | if (disksize > READ_ONCE(EXT4_I(inode)->i_disksize)) {
|
| 2527 | int err2;
|
| 2528 | loff_t i_size;
|
| 2529 |
|
| 2530 | down_write(&EXT4_I(inode)->i_data_sem);
|
| 2531 | i_size = i_size_read(inode);
|
| 2532 | if (disksize > i_size)
|
| 2533 | disksize = i_size;
|
| 2534 | if (disksize > EXT4_I(inode)->i_disksize)
|
| 2535 | EXT4_I(inode)->i_disksize = disksize;
|
| 2536 | up_write(&EXT4_I(inode)->i_data_sem);
|
| 2537 | err2 = ext4_mark_inode_dirty(handle, inode);
|
| 2538 | if (err2) {
|
| 2539 | ext4_error_err(inode->i_sb, -err2,
|
| 2540 | "Failed to mark inode %llu dirty",
|
| 2541 | inode->i_ino);
|
| 2542 | }
|
| 2543 | if (!err)
|
| 2544 | err = err2;
|
| 2545 | }
|
| 2546 | return err;
|
| 2547 | }
|
| 2548 |
|
| 2549 | static int ext4_journal_folio_buffers(handle_t *handle, struct folio *folio,
|
| 2550 | size_t len)
|
| 2551 | {
|
| 2552 | struct buffer_head *page_bufs = folio_buffers(folio);
|
| 2553 | struct inode *inode = folio->mapping->host;
|
| 2554 | int ret, err;
|
| 2555 |
|
| 2556 | ret = ext4_walk_page_buffers(handle, inode, page_bufs, 0, len,
|
| 2557 | NULL, do_journal_get_write_access);
|
| 2558 | err = ext4_walk_page_buffers(handle, inode, page_bufs, 0, len,
|
| 2559 | NULL, write_end_fn);
|
| 2560 | if (ret == 0)
|
| 2561 | ret = err;
|
| 2562 | err = ext4_jbd2_inode_add_write(handle, inode, folio_pos(folio), len);
|
| 2563 | if (ret == 0)
|
| 2564 | ret = err;
|
| 2565 | EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
|
| 2566 |
|
| 2567 | return ret;
|
| 2568 | }
|
| 2569 |
|
| 2570 | static int mpage_journal_page_buffers(handle_t *handle,
|
| 2571 | struct mpage_da_data *mpd,
|
| 2572 | struct folio *folio)
|
| 2573 | {
|
| 2574 | struct inode *inode = mpd->inode;
|
| 2575 | loff_t size = i_size_read(inode);
|
| 2576 | size_t len = folio_size(folio);
|
| 2577 |
|
| 2578 | folio_clear_checked(folio);
|
| 2579 | mpd->wbc->nr_to_write -= folio_nr_pages(folio);
|
| 2580 |
|
| 2581 | if (folio_pos(folio) + len > size &&
|
| 2582 | !ext4_verity_in_progress(inode))
|
| 2583 | len = size & (len - 1);
|
| 2584 |
|
| 2585 | return ext4_journal_folio_buffers(handle, folio, len);
|
| 2586 | }
|
| 2587 |
|
| 2588 | |
| 2589 | * mpage_prepare_extent_to_map - find & lock contiguous range of dirty pages
|
| 2590 | * needing mapping, submit mapped pages
|
| 2591 | *
|
| 2592 | * @mpd - where to look for pages
|
| 2593 | *
|
| 2594 | * Walk dirty pages in the mapping. If they are fully mapped, submit them for
|
| 2595 | * IO immediately. If we cannot map blocks, we submit just already mapped
|
| 2596 | * buffers in the page for IO and keep page dirty. When we can map blocks and
|
| 2597 | * we find a page which isn't mapped we start accumulating extent of buffers
|
| 2598 | * underlying these pages that needs mapping (formed by either delayed or
|
| 2599 | * unwritten buffers). We also lock the pages containing these buffers. The
|
| 2600 | * extent found is returned in @mpd structure (starting at mpd->lblk with
|
| 2601 | * length mpd->len blocks).
|
| 2602 | *
|
| 2603 | * Note that this function can attach bios to one io_end structure which are
|
| 2604 | * neither logically nor physically contiguous. Although it may seem as an
|
| 2605 | * unnecessary complication, it is actually inevitable in blocksize < pagesize
|
| 2606 | * case as we need to track IO to all buffers underlying a page in one io_end.
|
| 2607 |
|
| 2608 | static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd)
|
| 2609 | {
|
| 2610 | struct address_space *mapping = mpd->inode->i_mapping;
|
| 2611 | struct folio_batch fbatch;
|
| 2612 | unsigned int nr_folios;
|
| 2613 | pgoff_t index = mpd->start_pos >> PAGE_SHIFT;
|
| 2614 | pgoff_t end = mpd->end_pos >> PAGE_SHIFT;
|
| 2615 | xa_mark_t tag;
|
| 2616 | int i, err = 0;
|
| 2617 | ext4_lblk_t lblk;
|
| 2618 | struct buffer_head *head;
|
| 2619 | handle_t *handle = NULL;
|
| 2620 | int bpp = ext4_journal_blocks_per_folio(mpd->inode);
|
| 2621 |
|
| 2622 | tag = wbc_to_tag(mpd->wbc);
|
| 2623 |
|
| 2624 | mpd->map.m_len = 0;
|
| 2625 | mpd->next_pos = mpd->start_pos;
|
| 2626 | if (ext4_should_journal_data(mpd->inode)) {
|
| 2627 | handle = ext4_journal_start(mpd->inode, EXT4_HT_WRITE_PAGE,
|
| 2628 | bpp);
|
| 2629 | if (IS_ERR(handle))
|
| 2630 | return PTR_ERR(handle);
|
| 2631 | }
|
| 2632 | folio_batch_init(&fbatch);
|
| 2633 | while (index <= end) {
|
| 2634 | nr_folios = filemap_get_folios_tag(mapping, &index, end,
|
| 2635 | tag, &fbatch);
|
| 2636 | if (nr_folios == 0)
|
| 2637 | break;
|
| 2638 |
|
| 2639 | for (i = 0; i < nr_folios; i++) {
|
| 2640 | struct folio *folio = fbatch.folios[i];
|
| 2641 |
|
| 2642 | |
| 2643 | * Accumulated enough dirty pages? This doesn't apply
|
| 2644 | * to WB_SYNC_ALL mode. For integrity sync we have to
|
| 2645 | * keep going because someone may be concurrently
|
| 2646 | * dirtying pages, and we might have synced a lot of
|
| 2647 | * newly appeared dirty pages, but have not synced all
|
| 2648 | * of the old dirty pages.
|
| 2649 |
|
| 2650 | if (mpd->wbc->sync_mode == WB_SYNC_NONE &&
|
| 2651 | mpd->wbc->nr_to_write <=
|
| 2652 | EXT4_LBLK_TO_PG(mpd->inode, mpd->map.m_len))
|
| 2653 | goto out;
|
| 2654 |
|
| 2655 |
|
| 2656 | if (mpd->map.m_len > 0 &&
|
| 2657 | mpd->next_pos != folio_pos(folio))
|
| 2658 | goto out;
|
| 2659 |
|
| 2660 | if (handle) {
|
| 2661 | err = ext4_journal_ensure_credits(handle, bpp,
|
| 2662 | 0);
|
| 2663 | if (err < 0)
|
| 2664 | goto out;
|
| 2665 | }
|
| 2666 |
|
| 2667 | folio_lock(folio);
|
| 2668 | |
| 2669 | * If the page is no longer dirty, or its mapping no
|
| 2670 | * longer corresponds to inode we are writing (which
|
| 2671 | * means it has been truncated or invalidated), or the
|
| 2672 | * page is already under writeback and we are not doing
|
| 2673 | * a data integrity writeback, skip the page
|
| 2674 |
|
| 2675 | if ((folio_test_writeback(folio) &&
|
| 2676 | mpd->wbc->sync_mode == WB_SYNC_NONE) ||
|
| 2677 | unlikely(folio->mapping != mapping)) {
|
| 2678 | folio_unlock(folio);
|
| 2679 | continue;
|
| 2680 | }
|
| 2681 | |
| 2682 | * If the folio is clean, skip writing it back.
|
| 2683 | * Cycle the folio through the writeback state
|
| 2684 | * though, to clear stale xarray tags.
|
| 2685 |
|
| 2686 | if (!folio_test_dirty(folio)) {
|
| 2687 | if (!folio_test_writeback(folio)) {
|
| 2688 | __folio_start_writeback(folio, false);
|
| 2689 | folio_end_writeback(folio);
|
| 2690 | }
|
| 2691 | folio_unlock(folio);
|
| 2692 | continue;
|
| 2693 | }
|
| 2694 |
|
| 2695 | folio_wait_writeback(folio);
|
| 2696 | BUG_ON(folio_test_writeback(folio));
|
| 2697 |
|
| 2698 | |
| 2699 | * Should never happen but for buggy code in
|
| 2700 | * other subsystems that call
|
| 2701 | * set_page_dirty() without properly warning
|
| 2702 | * the file system first. See [1] for more
|
| 2703 | * information.
|
| 2704 | *
|
| 2705 | * [1] https://lore.kernel.org/linux-mm/[email protected]
|
| 2706 |
|
| 2707 | if (!folio_buffers(folio)) {
|
| 2708 | ext4_warning_inode(mpd->inode, "page %lu does not have buffers attached", folio->index);
|
| 2709 | folio_clear_dirty(folio);
|
| 2710 | folio_unlock(folio);
|
| 2711 | continue;
|
| 2712 | }
|
| 2713 |
|
| 2714 | if (mpd->map.m_len == 0)
|
| 2715 | mpd->start_pos = folio_pos(folio);
|
| 2716 | mpd->next_pos = folio_next_pos(folio);
|
| 2717 | |
| 2718 | * Writeout when we cannot modify metadata is simple.
|
| 2719 | * Just submit the page. For data=journal mode we
|
| 2720 | * first handle writeout of the page for checkpoint and
|
| 2721 | * only after that handle delayed page dirtying. This
|
| 2722 | * makes sure current data is checkpointed to the final
|
| 2723 | * location before possibly journalling it again which
|
| 2724 | * is desirable when the page is frequently dirtied
|
| 2725 | * through a pin.
|
| 2726 |
|
| 2727 | if (!mpd->can_map) {
|
| 2728 | mpage_submit_folio(mpd, folio);
|
| 2729 | err = 0;
|
| 2730 |
|
| 2731 | if (folio_test_checked(folio)) {
|
| 2732 | err = mpage_journal_page_buffers(handle,
|
| 2733 | mpd, folio);
|
| 2734 | if (err < 0)
|
| 2735 | goto out;
|
| 2736 | mpd->journalled_more_data = 1;
|
| 2737 | }
|
| 2738 | mpage_folio_done(mpd, folio);
|
| 2739 | } else {
|
| 2740 |
|
| 2741 | lblk = EXT4_PG_TO_LBLK(mpd->inode, folio->index);
|
| 2742 | head = folio_buffers(folio);
|
| 2743 | err = mpage_process_page_bufs(mpd, head, head,
|
| 2744 | lblk);
|
| 2745 | if (err <= 0)
|
| 2746 | goto out;
|
| 2747 | err = 0;
|
| 2748 | }
|
| 2749 | }
|
| 2750 | folio_batch_release(&fbatch);
|
| 2751 | cond_resched();
|
| 2752 | }
|
| 2753 | mpd->scanned_until_end = 1;
|
| 2754 | if (handle)
|
| 2755 | ext4_journal_stop(handle);
|
| 2756 | return 0;
|
| 2757 | out:
|
| 2758 | folio_batch_release(&fbatch);
|
| 2759 | if (handle)
|
| 2760 | ext4_journal_stop(handle);
|
| 2761 | return err;
|
| 2762 | }
|
| 2763 |
|
| 2764 | static int ext4_do_writepages(struct mpage_da_data *mpd)
|
| 2765 | {
|
| 2766 | struct writeback_control *wbc = mpd->wbc;
|
| 2767 | pgoff_t writeback_index = 0;
|
| 2768 | long nr_to_write = wbc->nr_to_write;
|
| 2769 | int range_whole = 0;
|
| 2770 | int cycled = 1;
|
| 2771 | handle_t *handle = NULL;
|
| 2772 | struct inode *inode = mpd->inode;
|
| 2773 | struct address_space *mapping = inode->i_mapping;
|
| 2774 | int needed_blocks, rsv_blocks = 0, ret = 0;
|
| 2775 | struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
|
| 2776 | struct blk_plug plug;
|
| 2777 | bool give_up_on_write = false;
|
| 2778 |
|
| 2779 | trace_ext4_writepages(inode, wbc);
|
| 2780 |
|
| 2781 | |
| 2782 | * No pages to write? This is mainly a kludge to avoid starting
|
| 2783 | * a transaction for special inodes like journal inode on last iput()
|
| 2784 | * because that could violate lock ordering on umount
|
| 2785 |
|
| 2786 | if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
|
| 2787 | goto out_writepages;
|
| 2788 |
|
| 2789 | |
| 2790 | * If the filesystem has aborted, it is read-only, so return
|
| 2791 | * right away instead of dumping stack traces later on that
|
| 2792 | * will obscure the real source of the problem. We test
|
| 2793 | * fs shutdown state instead of sb->s_flag's SB_RDONLY because
|
| 2794 | * the latter could be true if the filesystem is mounted
|
| 2795 | * read-only, and in that case, ext4_writepages should
|
| 2796 | * *never* be called, so if that ever happens, we would want
|
| 2797 | * the stack trace.
|
| 2798 |
|
| 2799 | ret = ext4_emergency_state(mapping->host->i_sb);
|
| 2800 | if (unlikely(ret))
|
| 2801 | goto out_writepages;
|
| 2802 |
|
| 2803 | |
| 2804 | * If we have inline data and arrive here, it means that
|
| 2805 | * we will soon create the block for the 1st page, so
|
| 2806 | * we'd better clear the inline data here.
|
| 2807 |
|
| 2808 | if (ext4_has_inline_data(inode)) {
|
| 2809 |
|
| 2810 | handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
|
| 2811 | if (IS_ERR(handle)) {
|
| 2812 | ret = PTR_ERR(handle);
|
| 2813 | goto out_writepages;
|
| 2814 | }
|
| 2815 | BUG_ON(ext4_test_inode_state(inode,
|
| 2816 | EXT4_STATE_MAY_INLINE_DATA));
|
| 2817 | ext4_destroy_inline_data(handle, inode);
|
| 2818 | ext4_journal_stop(handle);
|
| 2819 | }
|
| 2820 |
|
| 2821 | |
| 2822 | * data=journal mode does not do delalloc so we just need to writeout /
|
| 2823 | * journal already mapped buffers. On the other hand we need to commit
|
| 2824 | * transaction to make data stable. We expect all the data to be
|
| 2825 | * already in the journal (the only exception are DMA pinned pages
|
| 2826 | * dirtied behind our back) so we commit transaction here and run the
|
| 2827 | * writeback loop to checkpoint them. The checkpointing is not actually
|
| 2828 | * necessary to make data persistent *but* quite a few places (extent
|
| 2829 | * shifting operations, fsverity, ...) depend on being able to drop
|
| 2830 | * pagecache pages after calling filemap_write_and_wait() and for that
|
| 2831 | * checkpointing needs to happen.
|
| 2832 |
|
| 2833 | if (ext4_should_journal_data(inode)) {
|
| 2834 | mpd->can_map = 0;
|
| 2835 | if (wbc->sync_mode == WB_SYNC_ALL)
|
| 2836 | ext4_fc_commit(sbi->s_journal,
|
| 2837 | EXT4_I(inode)->i_datasync_tid);
|
| 2838 | }
|
| 2839 | mpd->journalled_more_data = 0;
|
| 2840 |
|
| 2841 | if (ext4_should_dioread_nolock(inode)) {
|
| 2842 | int bpf = ext4_journal_blocks_per_folio(inode);
|
| 2843 | |
| 2844 | * We may need to convert up to one extent per block in the
|
| 2845 | * folio.
|
| 2846 |
|
| 2847 | rsv_blocks = ext4_meta_trans_blocks(inode, bpf, bpf, 0);
|
| 2848 | }
|
| 2849 |
|
| 2850 | if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
|
| 2851 | range_whole = 1;
|
| 2852 |
|
| 2853 | if (wbc->range_cyclic) {
|
| 2854 | writeback_index = mapping->writeback_index;
|
| 2855 | if (writeback_index)
|
| 2856 | cycled = 0;
|
| 2857 | mpd->start_pos = writeback_index << PAGE_SHIFT;
|
| 2858 | mpd->end_pos = LLONG_MAX;
|
| 2859 | } else {
|
| 2860 | mpd->start_pos = wbc->range_start;
|
| 2861 | mpd->end_pos = wbc->range_end;
|
| 2862 | }
|
| 2863 |
|
| 2864 | ext4_io_submit_init(&mpd->io_submit, wbc);
|
| 2865 | retry:
|
| 2866 | if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
|
| 2867 | tag_pages_for_writeback(mapping, mpd->start_pos >> PAGE_SHIFT,
|
| 2868 | mpd->end_pos >> PAGE_SHIFT);
|
| 2869 | blk_start_plug(&plug);
|
| 2870 |
|
| 2871 | |
| 2872 | * First writeback pages that don't need mapping - we can avoid
|
| 2873 | * starting a transaction unnecessarily and also avoid being blocked
|
| 2874 | * in the block layer on device congestion while having transaction
|
| 2875 | * started.
|
| 2876 |
|
| 2877 | mpd->do_map = 0;
|
| 2878 | mpd->scanned_until_end = 0;
|
| 2879 | mpd->io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
|
| 2880 | if (!mpd->io_submit.io_end) {
|
| 2881 | ret = -ENOMEM;
|
| 2882 | goto unplug;
|
| 2883 | }
|
| 2884 | ret = mpage_prepare_extent_to_map(mpd);
|
| 2885 |
|
| 2886 | mpage_release_unused_pages(mpd, false);
|
| 2887 |
|
| 2888 | ext4_io_submit(&mpd->io_submit);
|
| 2889 | ext4_put_io_end_defer(mpd->io_submit.io_end);
|
| 2890 | mpd->io_submit.io_end = NULL;
|
| 2891 | if (ret < 0)
|
| 2892 | goto unplug;
|
| 2893 |
|
| 2894 | while (!mpd->scanned_until_end && wbc->nr_to_write > 0) {
|
| 2895 |
|
| 2896 | mpd->io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
|
| 2897 | if (!mpd->io_submit.io_end) {
|
| 2898 | ret = -ENOMEM;
|
| 2899 | break;
|
| 2900 | }
|
| 2901 |
|
| 2902 | WARN_ON_ONCE(!mpd->can_map);
|
| 2903 | |
| 2904 | * We have two constraints: We find one extent to map and we
|
| 2905 | * must always write out whole page (makes a difference when
|
| 2906 | * blocksize < pagesize) so that we don't block on IO when we
|
| 2907 | * try to write out the rest of the page. Journalled mode is
|
| 2908 | * not supported by delalloc.
|
| 2909 |
|
| 2910 | BUG_ON(ext4_should_journal_data(inode));
|
| 2911 | |
| 2912 | * Calculate the number of credits needed to reserve for one
|
| 2913 | * extent of up to MAX_WRITEPAGES_EXTENT_LEN blocks. It will
|
| 2914 | * attempt to extend the transaction or start a new iteration
|
| 2915 | * if the reserved credits are insufficient.
|
| 2916 |
|
| 2917 | needed_blocks = ext4_chunk_trans_blocks(inode,
|
| 2918 | MAX_WRITEPAGES_EXTENT_LEN);
|
| 2919 |
|
| 2920 | handle = ext4_journal_start_with_reserve(inode,
|
| 2921 | EXT4_HT_WRITE_PAGE, needed_blocks, rsv_blocks);
|
| 2922 | if (IS_ERR(handle)) {
|
| 2923 | ret = PTR_ERR(handle);
|
| 2924 | ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: "
|
| 2925 | "%ld pages, ino %llu; err %d", __func__,
|
| 2926 | wbc->nr_to_write, inode->i_ino, ret);
|
| 2927 |
|
| 2928 | ext4_put_io_end(mpd->io_submit.io_end);
|
| 2929 | mpd->io_submit.io_end = NULL;
|
| 2930 | break;
|
| 2931 | }
|
| 2932 | mpd->do_map = 1;
|
| 2933 |
|
| 2934 | trace_ext4_da_write_folios_start(inode, mpd->start_pos,
|
| 2935 | mpd->next_pos, wbc);
|
| 2936 | ret = mpage_prepare_extent_to_map(mpd);
|
| 2937 | if (!ret && mpd->map.m_len)
|
| 2938 | ret = mpage_map_and_submit_extent(handle, mpd,
|
| 2939 | &give_up_on_write);
|
| 2940 | |
| 2941 | * Caution: If the handle is synchronous,
|
| 2942 | * ext4_journal_stop() can wait for transaction commit
|
| 2943 | * to finish which may depend on writeback of pages to
|
| 2944 | * complete or on page lock to be released. In that
|
| 2945 | * case, we have to wait until after we have
|
| 2946 | * submitted all the IO, released page locks we hold,
|
| 2947 | * and dropped io_end reference (for extent conversion
|
| 2948 | * to be able to complete) before stopping the handle.
|
| 2949 |
|
| 2950 | if (!ext4_handle_valid(handle) || handle->h_sync == 0) {
|
| 2951 | ext4_journal_stop(handle);
|
| 2952 | handle = NULL;
|
| 2953 | mpd->do_map = 0;
|
| 2954 | }
|
| 2955 |
|
| 2956 | mpage_release_unused_pages(mpd, give_up_on_write);
|
| 2957 |
|
| 2958 | ext4_io_submit(&mpd->io_submit);
|
| 2959 |
|
| 2960 | |
| 2961 | * Drop our io_end reference we got from init. We have
|
| 2962 | * to be careful and use deferred io_end finishing if
|
| 2963 | * we are still holding the transaction as we can
|
| 2964 | * release the last reference to io_end which may end
|
| 2965 | * up doing unwritten extent conversion.
|
| 2966 |
|
| 2967 | if (handle) {
|
| 2968 | ext4_put_io_end_defer(mpd->io_submit.io_end);
|
| 2969 | ext4_journal_stop(handle);
|
| 2970 | } else
|
| 2971 | ext4_put_io_end(mpd->io_submit.io_end);
|
| 2972 | mpd->io_submit.io_end = NULL;
|
| 2973 | trace_ext4_da_write_folios_end(inode, mpd->start_pos,
|
| 2974 | mpd->next_pos, wbc, ret);
|
| 2975 |
|
| 2976 | if (ret == -ENOSPC && sbi->s_journal) {
|
| 2977 | |
| 2978 | * Commit the transaction which would
|
| 2979 | * free blocks released in the transaction
|
| 2980 | * and try again
|
| 2981 |
|
| 2982 | jbd2_journal_force_commit_nested(sbi->s_journal);
|
| 2983 | ret = 0;
|
| 2984 | continue;
|
| 2985 | }
|
| 2986 | if (ret == -EAGAIN)
|
| 2987 | ret = 0;
|
| 2988 |
|
| 2989 | if (ret)
|
| 2990 | break;
|
| 2991 | }
|
| 2992 | unplug:
|
| 2993 | blk_finish_plug(&plug);
|
| 2994 | if (!ret && !cycled && wbc->nr_to_write > 0) {
|
| 2995 | cycled = 1;
|
| 2996 | mpd->end_pos = (writeback_index << PAGE_SHIFT) - 1;
|
| 2997 | mpd->start_pos = 0;
|
| 2998 | goto retry;
|
| 2999 | }
|
| 3000 |
|
| 3001 |
|
| 3002 | if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
|
| 3003 | |
| 3004 | * Set the writeback_index so that range_cyclic
|
| 3005 | * mode will write it back later
|
| 3006 |
|
| 3007 | mapping->writeback_index = mpd->start_pos >> PAGE_SHIFT;
|
| 3008 |
|
| 3009 | out_writepages:
|
| 3010 | trace_ext4_writepages_result(inode, wbc, ret,
|
| 3011 | nr_to_write - wbc->nr_to_write);
|
| 3012 | return ret;
|
| 3013 | }
|
| 3014 |
|
| 3015 | static int ext4_writepages(struct address_space *mapping,
|
| 3016 | struct writeback_control *wbc)
|
| 3017 | {
|
| 3018 | struct super_block *sb = mapping->host->i_sb;
|
| 3019 | struct mpage_da_data mpd = {
|
| 3020 | .inode = mapping->host,
|
| 3021 | .wbc = wbc,
|
| 3022 | .can_map = 1,
|
| 3023 | };
|
| 3024 | int ret;
|
| 3025 | int alloc_ctx;
|
| 3026 |
|
| 3027 | ret = ext4_emergency_state(sb);
|
| 3028 | if (unlikely(ret))
|
| 3029 | return ret;
|
| 3030 |
|
| 3031 | alloc_ctx = ext4_writepages_down_read(sb);
|
| 3032 | ret = ext4_do_writepages(&mpd);
|
| 3033 | |
| 3034 | * For data=journal writeback we could have come across pages marked
|
| 3035 | * for delayed dirtying (PageChecked) which were just added to the
|
| 3036 | * running transaction. Try once more to get them to stable storage.
|
| 3037 |
|
| 3038 | if (!ret && mpd.journalled_more_data)
|
| 3039 | ret = ext4_do_writepages(&mpd);
|
| 3040 | ext4_writepages_up_read(sb, alloc_ctx);
|
| 3041 |
|
| 3042 | return ret;
|
| 3043 | }
|
| 3044 |
|
| 3045 | int ext4_normal_submit_inode_data_buffers(struct jbd2_inode *jinode)
|
| 3046 | {
|
| 3047 | loff_t range_start, range_end;
|
| 3048 | struct writeback_control wbc = {
|
| 3049 | .sync_mode = WB_SYNC_ALL,
|
| 3050 | .nr_to_write = LONG_MAX,
|
| 3051 | };
|
| 3052 | struct mpage_da_data mpd = {
|
| 3053 | .inode = jinode->i_vfs_inode,
|
| 3054 | .wbc = &wbc,
|
| 3055 | .can_map = 0,
|
| 3056 | };
|
| 3057 |
|
| 3058 | if (!jbd2_jinode_get_dirty_range(jinode, &range_start, &range_end))
|
| 3059 | return 0;
|
| 3060 |
|
| 3061 | wbc.range_start = range_start;
|
| 3062 | wbc.range_end = range_end;
|
| 3063 |
|
| 3064 | return ext4_do_writepages(&mpd);
|
| 3065 | }
|
| 3066 |
|
| 3067 | static int ext4_dax_writepages(struct address_space *mapping,
|
| 3068 | struct writeback_control *wbc)
|
| 3069 | {
|
| 3070 | int ret;
|
| 3071 | long nr_to_write = wbc->nr_to_write;
|
| 3072 | struct inode *inode = mapping->host;
|
| 3073 | int alloc_ctx;
|
| 3074 |
|
| 3075 | ret = ext4_emergency_state(inode->i_sb);
|
| 3076 | if (unlikely(ret))
|
| 3077 | return ret;
|
| 3078 |
|
| 3079 | alloc_ctx = ext4_writepages_down_read(inode->i_sb);
|
| 3080 | trace_ext4_writepages(inode, wbc);
|
| 3081 |
|
| 3082 | ret = dax_writeback_mapping_range(mapping,
|
| 3083 | EXT4_SB(inode->i_sb)->s_daxdev, wbc);
|
| 3084 | trace_ext4_writepages_result(inode, wbc, ret,
|
| 3085 | nr_to_write - wbc->nr_to_write);
|
| 3086 | ext4_writepages_up_read(inode->i_sb, alloc_ctx);
|
| 3087 | return ret;
|
| 3088 | }
|
| 3089 |
|
| 3090 | static int ext4_nonda_switch(struct super_block *sb)
|
| 3091 | {
|
| 3092 | s64 free_clusters, dirty_clusters;
|
| 3093 | struct ext4_sb_info *sbi = EXT4_SB(sb);
|
| 3094 |
|
| 3095 | |
| 3096 | * switch to non delalloc mode if we are running low
|
| 3097 | * on free block. The free block accounting via percpu
|
| 3098 | * counters can get slightly wrong with percpu_counter_batch getting
|
| 3099 | * accumulated on each CPU without updating global counters
|
| 3100 | * Delalloc need an accurate free block accounting. So switch
|
| 3101 | * to non delalloc when we are near to error range.
|
| 3102 |
|
| 3103 | free_clusters =
|
| 3104 | percpu_counter_read_positive(&sbi->s_freeclusters_counter);
|
| 3105 | dirty_clusters =
|
| 3106 | percpu_counter_read_positive(&sbi->s_dirtyclusters_counter);
|
| 3107 | |
| 3108 | * Start pushing delalloc when 1/2 of free blocks are dirty.
|
| 3109 |
|
| 3110 | if (dirty_clusters && (free_clusters < 2 * dirty_clusters))
|
| 3111 | try_to_writeback_inodes_sb(sb, WB_REASON_FS_FREE_SPACE);
|
| 3112 |
|
| 3113 | if (2 * free_clusters < 3 * dirty_clusters ||
|
| 3114 | free_clusters < (dirty_clusters + EXT4_FREECLUSTERS_WATERMARK)) {
|
| 3115 | |
| 3116 | * free block count is less than 150% of dirty blocks
|
| 3117 | * or free blocks is less than watermark
|
| 3118 |
|
| 3119 | return 1;
|
| 3120 | }
|
| 3121 | return 0;
|
| 3122 | }
|
| 3123 |
|
| 3124 | static int ext4_da_write_begin(const struct kiocb *iocb,
|
| 3125 | struct address_space *mapping,
|
| 3126 | loff_t pos, unsigned len,
|
| 3127 | struct folio **foliop, void **fsdata)
|
| 3128 | {
|
| 3129 | int ret, retries = 0;
|
| 3130 | struct folio *folio;
|
| 3131 | pgoff_t index;
|
| 3132 | struct inode *inode = mapping->host;
|
| 3133 |
|
| 3134 | ret = ext4_emergency_state(inode->i_sb);
|
| 3135 | if (unlikely(ret))
|
| 3136 | return ret;
|
| 3137 |
|
| 3138 | index = pos >> PAGE_SHIFT;
|
| 3139 |
|
| 3140 | if (ext4_nonda_switch(inode->i_sb) || ext4_verity_in_progress(inode)) {
|
| 3141 | *fsdata = (void *)FALL_BACK_TO_NONDELALLOC;
|
| 3142 | return ext4_write_begin(iocb, mapping, pos,
|
| 3143 | len, foliop, fsdata);
|
| 3144 | }
|
| 3145 | *fsdata = (void *)0;
|
| 3146 | trace_ext4_da_write_begin(inode, pos, len);
|
| 3147 |
|
| 3148 | if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
|
| 3149 | ret = ext4_generic_write_inline_data(mapping, inode, pos, len,
|
| 3150 | foliop, true);
|
| 3151 | if (ret < 0)
|
| 3152 | return ret;
|
| 3153 | if (ret == 1) {
|
| 3154 | *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE);
|
| 3155 | return 0;
|
| 3156 | }
|
| 3157 | }
|
| 3158 |
|
| 3159 | retry:
|
| 3160 | folio = write_begin_get_folio(iocb, mapping, index, len);
|
| 3161 | if (IS_ERR(folio))
|
| 3162 | return PTR_ERR(folio);
|
| 3163 |
|
| 3164 | if (len > folio_next_pos(folio) - pos)
|
| 3165 | len = folio_next_pos(folio) - pos;
|
| 3166 |
|
| 3167 | ret = ext4_block_write_begin(NULL, folio, pos, len,
|
| 3168 | ext4_da_get_block_prep);
|
| 3169 | if (ret < 0) {
|
| 3170 | folio_unlock(folio);
|
| 3171 | folio_put(folio);
|
| 3172 | |
| 3173 | * ext4_block_write_begin may have instantiated a few blocks
|
| 3174 | * outside i_size. Trim these off again. Don't need
|
| 3175 | * i_size_read because we hold inode lock.
|
| 3176 |
|
| 3177 | if (pos + len > inode->i_size)
|
| 3178 | ext4_truncate_failed_write(inode);
|
| 3179 |
|
| 3180 | if (ret == -ENOSPC &&
|
| 3181 | ext4_should_retry_alloc(inode->i_sb, &retries))
|
| 3182 | goto retry;
|
| 3183 | return ret;
|
| 3184 | }
|
| 3185 |
|
| 3186 | *foliop = folio;
|
| 3187 | return ret;
|
| 3188 | }
|
| 3189 |
|
| 3190 | |
| 3191 | * Check if we should update i_disksize
|
| 3192 | * when write to the end of file but not require block allocation
|
| 3193 |
|
| 3194 | static int ext4_da_should_update_i_disksize(struct folio *folio,
|
| 3195 | unsigned long offset)
|
| 3196 | {
|
| 3197 | struct buffer_head *bh;
|
| 3198 | struct inode *inode = folio->mapping->host;
|
| 3199 | unsigned int idx;
|
| 3200 | int i;
|
| 3201 |
|
| 3202 | bh = folio_buffers(folio);
|
| 3203 | idx = offset >> inode->i_blkbits;
|
| 3204 |
|
| 3205 | for (i = 0; i < idx; i++)
|
| 3206 | bh = bh->b_this_page;
|
| 3207 |
|
| 3208 | if (!buffer_mapped(bh) || (buffer_delay(bh)) || buffer_unwritten(bh))
|
| 3209 | return 0;
|
| 3210 | return 1;
|
| 3211 | }
|
| 3212 |
|
| 3213 | static int ext4_da_do_write_end(struct address_space *mapping,
|
| 3214 | loff_t pos, unsigned len, unsigned copied,
|
| 3215 | struct folio *folio)
|
| 3216 | {
|
| 3217 | struct inode *inode = mapping->host;
|
| 3218 | loff_t old_size = inode->i_size;
|
| 3219 | bool disksize_changed = false;
|
| 3220 | loff_t new_i_size;
|
| 3221 | handle_t *handle;
|
| 3222 |
|
| 3223 | if (unlikely(!folio_buffers(folio))) {
|
| 3224 | folio_unlock(folio);
|
| 3225 | folio_put(folio);
|
| 3226 | return -EIO;
|
| 3227 | }
|
| 3228 | |
| 3229 | * block_write_end() will mark the inode as dirty with I_DIRTY_PAGES
|
| 3230 | * flag, which all that's needed to trigger page writeback.
|
| 3231 |
|
| 3232 | copied = block_write_end(pos, len, copied, folio);
|
| 3233 | new_i_size = pos + copied;
|
| 3234 |
|
| 3235 | |
| 3236 | * It's important to update i_size while still holding folio lock,
|
| 3237 | * because folio writeout could otherwise come in and zero beyond
|
| 3238 | * i_size.
|
| 3239 | *
|
| 3240 | * Since we are holding inode lock, we are sure i_disksize <=
|
| 3241 | * i_size. We also know that if i_disksize < i_size, there are
|
| 3242 | * delalloc writes pending in the range up to i_size. If the end of
|
| 3243 | * the current write is <= i_size, there's no need to touch
|
| 3244 | * i_disksize since writeback will push i_disksize up to i_size
|
| 3245 | * eventually. If the end of the current write is > i_size and
|
| 3246 | * inside an allocated block which ext4_da_should_update_i_disksize()
|
| 3247 | * checked, we need to update i_disksize here as certain
|
| 3248 | * ext4_writepages() paths not allocating blocks and update i_disksize.
|
| 3249 |
|
| 3250 | if (new_i_size > inode->i_size) {
|
| 3251 | unsigned long end;
|
| 3252 |
|
| 3253 | i_size_write(inode, new_i_size);
|
| 3254 | end = offset_in_folio(folio, new_i_size - 1);
|
| 3255 | if (copied && ext4_da_should_update_i_disksize(folio, end)) {
|
| 3256 | ext4_update_i_disksize(inode, new_i_size);
|
| 3257 | disksize_changed = true;
|
| 3258 | }
|
| 3259 | }
|
| 3260 |
|
| 3261 | folio_unlock(folio);
|
| 3262 | folio_put(folio);
|
| 3263 |
|
| 3264 | if (pos > old_size)
|
| 3265 | pagecache_isize_extended(inode, old_size, pos);
|
| 3266 |
|
| 3267 | if (!disksize_changed)
|
| 3268 | return copied;
|
| 3269 |
|
| 3270 | handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
|
| 3271 | if (IS_ERR(handle))
|
| 3272 | return PTR_ERR(handle);
|
| 3273 | ext4_mark_inode_dirty(handle, inode);
|
| 3274 | ext4_journal_stop(handle);
|
| 3275 |
|
| 3276 | return copied;
|
| 3277 | }
|
| 3278 |
|
| 3279 | static int ext4_da_write_end(const struct kiocb *iocb,
|
| 3280 | struct address_space *mapping,
|
| 3281 | loff_t pos, unsigned len, unsigned copied,
|
| 3282 | struct folio *folio, void *fsdata)
|
| 3283 | {
|
| 3284 | struct inode *inode = mapping->host;
|
| 3285 | unsigned long write_mode = (unsigned long)fsdata;
|
| 3286 |
|
| 3287 | if (write_mode & FALL_BACK_TO_NONDELALLOC)
|
| 3288 | return ext4_write_end(iocb, mapping, pos,
|
| 3289 | len, copied, folio, fsdata);
|
| 3290 |
|
| 3291 | trace_ext4_da_write_end(inode, pos, len, copied);
|
| 3292 |
|
| 3293 | if (write_mode & EXT4_WRITE_DATA_INLINE)
|
| 3294 | return ext4_write_inline_data_end(inode, pos, len, copied,
|
| 3295 | folio);
|
| 3296 |
|
| 3297 | if (unlikely(copied < len) && !folio_test_uptodate(folio))
|
| 3298 | copied = 0;
|
| 3299 |
|
| 3300 | return ext4_da_do_write_end(mapping, pos, len, copied, folio);
|
| 3301 | }
|
| 3302 |
|
| 3303 | |
| 3304 | * Force all delayed allocation blocks to be allocated for a given inode.
|
| 3305 |
|
| 3306 | int ext4_alloc_da_blocks(struct inode *inode)
|
| 3307 | {
|
| 3308 | trace_ext4_alloc_da_blocks(inode);
|
| 3309 |
|
| 3310 | if (!EXT4_I(inode)->i_reserved_data_blocks)
|
| 3311 | return 0;
|
| 3312 |
|
| 3313 | |
| 3314 | * We do something simple for now. The filemap_flush() will
|
| 3315 | * also start triggering a write of the data blocks, which is
|
| 3316 | * not strictly speaking necessary. However, to do otherwise
|
| 3317 | * would require replicating code paths in:
|
| 3318 | *
|
| 3319 | * ext4_writepages() ->
|
| 3320 | * write_cache_pages() ---> (via passed in callback function)
|
| 3321 | * __mpage_da_writepage() -->
|
| 3322 | * mpage_add_bh_to_extent()
|
| 3323 | * mpage_da_map_blocks()
|
| 3324 | *
|
| 3325 | * The problem is that write_cache_pages(), located in
|
| 3326 | * mm/page-writeback.c, marks pages clean in preparation for
|
| 3327 | * doing I/O, which is not desirable if we're not planning on
|
| 3328 | * doing I/O at all.
|
| 3329 | *
|
| 3330 | * We could call write_cache_pages(), and then redirty all of
|
| 3331 | * the pages by calling redirty_page_for_writepage() but that
|
| 3332 | * would be ugly in the extreme. So instead we would need to
|
| 3333 | * replicate parts of the code in the above functions,
|
| 3334 | * simplifying them because we wouldn't actually intend to
|
| 3335 | * write out the pages, but rather only collect contiguous
|
| 3336 | * logical block extents, call the multi-block allocator, and
|
| 3337 | * then update the buffer heads with the block allocations.
|
| 3338 | *
|
| 3339 | * For now, though, we'll cheat by calling filemap_flush(),
|
| 3340 | * which will map the blocks, and start the I/O, but not
|
| 3341 | * actually wait for the I/O to complete.
|
| 3342 |
|
| 3343 | return filemap_flush(inode->i_mapping);
|
| 3344 | }
|
| 3345 |
|
| 3346 | |
| 3347 | * bmap() is special. It gets used by applications such as lilo and by
|
| 3348 | * the swapper to find the on-disk block of a specific piece of data.
|
| 3349 | *
|
| 3350 | * Naturally, this is dangerous if the block concerned is still in the
|
| 3351 | * journal. If somebody makes a swapfile on an ext4 data-journaling
|
| 3352 | * filesystem and enables swap, then they may get a nasty shock when the
|
| 3353 | * data getting swapped to that swapfile suddenly gets overwritten by
|
| 3354 | * the original zero's written out previously to the journal and
|
| 3355 | * awaiting writeback in the kernel's buffer cache.
|
| 3356 | *
|
| 3357 | * So, if we see any bmap calls here on a modified, data-journaled file,
|
| 3358 | * take extra steps to flush any blocks which might be in the cache.
|
| 3359 |
|
| 3360 | static sector_t ext4_bmap(struct address_space *mapping, sector_t block)
|
| 3361 | {
|
| 3362 | struct inode *inode = mapping->host;
|
| 3363 | sector_t ret = 0;
|
| 3364 |
|
| 3365 | inode_lock_shared(inode);
|
| 3366 | |
| 3367 | * We can get here for an inline file via the FIBMAP ioctl
|
| 3368 |
|
| 3369 | if (ext4_has_inline_data(inode))
|
| 3370 | goto out;
|
| 3371 |
|
| 3372 | if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) &&
|
| 3373 | (test_opt(inode->i_sb, DELALLOC) ||
|
| 3374 | ext4_should_journal_data(inode))) {
|
| 3375 | |
| 3376 | * With delalloc or journalled data we want to sync the file so
|
| 3377 | * that we can make sure we allocate blocks for file and data
|
| 3378 | * is in place for the user to see it
|
| 3379 |
|
| 3380 | filemap_write_and_wait(mapping);
|
| 3381 | }
|
| 3382 |
|
| 3383 | ret = iomap_bmap(mapping, block, &ext4_iomap_ops);
|
| 3384 |
|
| 3385 | out:
|
| 3386 | inode_unlock_shared(inode);
|
| 3387 | return ret;
|
| 3388 | }
|
| 3389 |
|
| 3390 | static void ext4_invalidate_folio(struct folio *folio, size_t offset,
|
| 3391 | size_t length)
|
| 3392 | {
|
| 3393 | trace_ext4_invalidate_folio(folio, offset, length);
|
| 3394 |
|
| 3395 |
|
| 3396 | WARN_ON(folio_buffers(folio) && buffer_jbd(folio_buffers(folio)));
|
| 3397 |
|
| 3398 | block_invalidate_folio(folio, offset, length);
|
| 3399 | }
|
| 3400 |
|
| 3401 | static int __ext4_journalled_invalidate_folio(struct folio *folio,
|
| 3402 | size_t offset, size_t length)
|
| 3403 | {
|
| 3404 | journal_t *journal = EXT4_JOURNAL(folio->mapping->host);
|
| 3405 |
|
| 3406 | trace_ext4_journalled_invalidate_folio(folio, offset, length);
|
| 3407 |
|
| 3408 | |
| 3409 | * If it's a full truncate we just forget about the pending dirtying
|
| 3410 |
|
| 3411 | if (offset == 0 && length == folio_size(folio))
|
| 3412 | folio_clear_checked(folio);
|
| 3413 |
|
| 3414 | return jbd2_journal_invalidate_folio(journal, folio, offset, length);
|
| 3415 | }
|
| 3416 |
|
| 3417 |
|
| 3418 | static void ext4_journalled_invalidate_folio(struct folio *folio,
|
| 3419 | size_t offset,
|
| 3420 | size_t length)
|
| 3421 | {
|
| 3422 | WARN_ON(__ext4_journalled_invalidate_folio(folio, offset, length) < 0);
|
| 3423 | }
|
| 3424 |
|
| 3425 | static bool ext4_release_folio(struct folio *folio, gfp_t wait)
|
| 3426 | {
|
| 3427 | struct inode *inode = folio->mapping->host;
|
| 3428 | journal_t *journal = EXT4_JOURNAL(inode);
|
| 3429 |
|
| 3430 | trace_ext4_release_folio(inode, folio);
|
| 3431 |
|
| 3432 |
|
| 3433 | if (folio_test_checked(folio))
|
| 3434 | return false;
|
| 3435 | if (journal)
|
| 3436 | return jbd2_journal_try_to_free_buffers(journal, folio);
|
| 3437 | else
|
| 3438 | return try_to_free_buffers(folio);
|
| 3439 | }
|
| 3440 |
|
| 3441 | static bool ext4_inode_datasync_dirty(struct inode *inode)
|
| 3442 | {
|
| 3443 | journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
|
| 3444 | struct mapping_metadata_bhs *mmb;
|
| 3445 |
|
| 3446 | if (journal) {
|
| 3447 | if (jbd2_transaction_committed(journal,
|
| 3448 | EXT4_I(inode)->i_datasync_tid))
|
| 3449 | return false;
|
| 3450 | if (test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT))
|
| 3451 | return !list_empty(&EXT4_I(inode)->i_fc_list);
|
| 3452 | return true;
|
| 3453 | }
|
| 3454 |
|
| 3455 | mmb = ext4_i_metadata_bhs(inode);
|
| 3456 |
|
| 3457 | if (mmb && mmb_has_buffers(mmb))
|
| 3458 | return true;
|
| 3459 | return inode_state_read_once(inode) & I_DIRTY_DATASYNC;
|
| 3460 | }
|
| 3461 |
|
| 3462 | static void ext4_set_iomap(struct inode *inode, struct iomap *iomap,
|
| 3463 | struct ext4_map_blocks *map, loff_t offset,
|
| 3464 | loff_t length, unsigned int flags)
|
| 3465 | {
|
| 3466 | u8 blkbits = inode->i_blkbits;
|
| 3467 |
|
| 3468 | |
| 3469 | * Writes that span EOF might trigger an I/O size update on completion,
|
| 3470 | * so consider them to be dirty for the purpose of O_DSYNC, even if
|
| 3471 | * there is no other metadata changes being made or are pending.
|
| 3472 |
|
| 3473 | iomap->flags = 0;
|
| 3474 | if (ext4_inode_datasync_dirty(inode) ||
|
| 3475 | offset + length > i_size_read(inode))
|
| 3476 | iomap->flags |= IOMAP_F_DIRTY;
|
| 3477 |
|
| 3478 | if (map->m_flags & EXT4_MAP_NEW)
|
| 3479 | iomap->flags |= IOMAP_F_NEW;
|
| 3480 |
|
| 3481 |
|
| 3482 | if (flags & IOMAP_ATOMIC)
|
| 3483 | iomap->flags |= IOMAP_F_ATOMIC_BIO;
|
| 3484 |
|
| 3485 | if (flags & IOMAP_DAX)
|
| 3486 | iomap->dax_dev = EXT4_SB(inode->i_sb)->s_daxdev;
|
| 3487 | else
|
| 3488 | iomap->bdev = inode->i_sb->s_bdev;
|
| 3489 | iomap->offset = EXT4_LBLK_TO_B(inode, map->m_lblk);
|
| 3490 | iomap->length = EXT4_LBLK_TO_B(inode, map->m_len);
|
| 3491 |
|
| 3492 | if ((map->m_flags & EXT4_MAP_MAPPED) &&
|
| 3493 | !ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
|
| 3494 | iomap->flags |= IOMAP_F_MERGED;
|
| 3495 |
|
| 3496 | |
| 3497 | * Flags passed to ext4_map_blocks() for direct I/O writes can result
|
| 3498 | * in m_flags having both EXT4_MAP_MAPPED and EXT4_MAP_UNWRITTEN bits
|
| 3499 | * set. In order for any allocated unwritten extents to be converted
|
| 3500 | * into written extents correctly within the ->end_io() handler, we
|
| 3501 | * need to ensure that the iomap->type is set appropriately. Hence, the
|
| 3502 | * reason why we need to check whether the EXT4_MAP_UNWRITTEN bit has
|
| 3503 | * been set first.
|
| 3504 |
|
| 3505 | if (map->m_flags & EXT4_MAP_UNWRITTEN) {
|
| 3506 | iomap->type = IOMAP_UNWRITTEN;
|
| 3507 | iomap->addr = (u64) map->m_pblk << blkbits;
|
| 3508 | if (flags & IOMAP_DAX)
|
| 3509 | iomap->addr += EXT4_SB(inode->i_sb)->s_dax_part_off;
|
| 3510 | } else if (map->m_flags & EXT4_MAP_MAPPED) {
|
| 3511 | iomap->type = IOMAP_MAPPED;
|
| 3512 | iomap->addr = (u64) map->m_pblk << blkbits;
|
| 3513 | if (flags & IOMAP_DAX)
|
| 3514 | iomap->addr += EXT4_SB(inode->i_sb)->s_dax_part_off;
|
| 3515 | } else if (map->m_flags & EXT4_MAP_DELAYED) {
|
| 3516 | iomap->type = IOMAP_DELALLOC;
|
| 3517 | iomap->addr = IOMAP_NULL_ADDR;
|
| 3518 | } else {
|
| 3519 | iomap->type = IOMAP_HOLE;
|
| 3520 | iomap->addr = IOMAP_NULL_ADDR;
|
| 3521 | }
|
| 3522 | }
|
| 3523 |
|
| 3524 | static int ext4_map_blocks_atomic_write_slow(handle_t *handle,
|
| 3525 | struct inode *inode, struct ext4_map_blocks *map)
|
| 3526 | {
|
| 3527 | ext4_lblk_t m_lblk = map->m_lblk;
|
| 3528 | unsigned int m_len = map->m_len;
|
| 3529 | unsigned int mapped_len = 0, m_flags = 0;
|
| 3530 | ext4_fsblk_t next_pblk = 0;
|
| 3531 | bool check_next_pblk = false;
|
| 3532 | int ret = 0;
|
| 3533 |
|
| 3534 | WARN_ON_ONCE(!ext4_has_feature_bigalloc(inode->i_sb));
|
| 3535 |
|
| 3536 | |
| 3537 | * This is a slow path in case of mixed mapping. We use
|
| 3538 | * EXT4_GET_BLOCKS_CREATE_ZERO flag here to make sure we get a single
|
| 3539 | * contiguous mapped mapping. This will ensure any unwritten or hole
|
| 3540 | * regions within the requested range is zeroed out and we return
|
| 3541 | * a single contiguous mapped extent.
|
| 3542 |
|
| 3543 | m_flags = EXT4_GET_BLOCKS_CREATE_ZERO;
|
| 3544 |
|
| 3545 | do {
|
| 3546 | ret = ext4_map_blocks(handle, inode, map, m_flags);
|
| 3547 | if (ret < 0 && ret != -ENOSPC)
|
| 3548 | goto out_err;
|
| 3549 | |
| 3550 | * This should never happen, but let's return an error code to
|
| 3551 | * avoid an infinite loop in here.
|
| 3552 |
|
| 3553 | if (ret == 0) {
|
| 3554 | ret = -EFSCORRUPTED;
|
| 3555 | ext4_warning_inode(inode,
|
| 3556 | "ext4_map_blocks() couldn't allocate blocks m_flags: 0x%x, ret:%d",
|
| 3557 | m_flags, ret);
|
| 3558 | goto out_err;
|
| 3559 | }
|
| 3560 | |
| 3561 | * With bigalloc we should never get ENOSPC nor discontiguous
|
| 3562 | * physical extents.
|
| 3563 |
|
| 3564 | if ((check_next_pblk && next_pblk != map->m_pblk) ||
|
| 3565 | ret == -ENOSPC) {
|
| 3566 | ext4_warning_inode(inode,
|
| 3567 | "Non-contiguous allocation detected: expected %llu, got %llu, "
|
| 3568 | "or ext4_map_blocks() returned out of space ret: %d",
|
| 3569 | next_pblk, map->m_pblk, ret);
|
| 3570 | ret = -EFSCORRUPTED;
|
| 3571 | goto out_err;
|
| 3572 | }
|
| 3573 | next_pblk = map->m_pblk + map->m_len;
|
| 3574 | check_next_pblk = true;
|
| 3575 |
|
| 3576 | mapped_len += map->m_len;
|
| 3577 | map->m_lblk += map->m_len;
|
| 3578 | map->m_len = m_len - mapped_len;
|
| 3579 | } while (mapped_len < m_len);
|
| 3580 |
|
| 3581 | |
| 3582 | * We might have done some work in above loop, so we need to query the
|
| 3583 | * start of the physical extent, based on the origin m_lblk and m_len.
|
| 3584 | * Let's also ensure we were able to allocate the required range for
|
| 3585 | * mixed mapping case.
|
| 3586 |
|
| 3587 | map->m_lblk = m_lblk;
|
| 3588 | map->m_len = m_len;
|
| 3589 | map->m_flags = 0;
|
| 3590 |
|
| 3591 | ret = ext4_map_blocks(handle, inode, map,
|
| 3592 | EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF);
|
| 3593 | if (ret != m_len) {
|
| 3594 | ext4_warning_inode(inode,
|
| 3595 | "allocation failed for atomic write request m_lblk:%u, m_len:%u, ret:%d\n",
|
| 3596 | m_lblk, m_len, ret);
|
| 3597 | ret = -EINVAL;
|
| 3598 | }
|
| 3599 | return ret;
|
| 3600 |
|
| 3601 | out_err:
|
| 3602 |
|
| 3603 | map->m_lblk = m_lblk;
|
| 3604 | map->m_len = m_len;
|
| 3605 | map->m_flags = 0;
|
| 3606 | return ret;
|
| 3607 | }
|
| 3608 |
|
| 3609 | |
| 3610 | * ext4_map_blocks_atomic: Helper routine to ensure the entire requested
|
| 3611 | * range in @map [lblk, lblk + len) is one single contiguous extent with no
|
| 3612 | * mixed mappings.
|
| 3613 | *
|
| 3614 | * We first use m_flags passed to us by our caller (ext4_iomap_alloc()).
|
| 3615 | * We only call EXT4_GET_BLOCKS_ZERO in the slow path, when the underlying
|
| 3616 | * physical extent for the requested range does not have a single contiguous
|
| 3617 | * mapping type i.e. (Hole, Mapped, or Unwritten) throughout.
|
| 3618 | * In that case we will loop over the requested range to allocate and zero out
|
| 3619 | * the unwritten / holes in between, to get a single mapped extent from
|
| 3620 | * [m_lblk, m_lblk + m_len). Note that this is only possible because we know
|
| 3621 | * this can be called only with bigalloc enabled filesystem where the underlying
|
| 3622 | * cluster is already allocated. This avoids allocating discontiguous extents
|
| 3623 | * in the slow path due to multiple calls to ext4_map_blocks().
|
| 3624 | * The slow path is mostly non-performance critical path, so it should be ok to
|
| 3625 | * loop using ext4_map_blocks() with appropriate flags to allocate & zero the
|
| 3626 | * underlying short holes/unwritten extents within the requested range.
|
| 3627 |
|
| 3628 | static int ext4_map_blocks_atomic_write(handle_t *handle, struct inode *inode,
|
| 3629 | struct ext4_map_blocks *map, int m_flags,
|
| 3630 | bool *force_commit)
|
| 3631 | {
|
| 3632 | ext4_lblk_t m_lblk = map->m_lblk;
|
| 3633 | unsigned int m_len = map->m_len;
|
| 3634 | int ret = 0;
|
| 3635 |
|
| 3636 | WARN_ON_ONCE(m_len > 1 && !ext4_has_feature_bigalloc(inode->i_sb));
|
| 3637 |
|
| 3638 | ret = ext4_map_blocks(handle, inode, map, m_flags);
|
| 3639 | if (ret < 0 || ret == m_len)
|
| 3640 | goto out;
|
| 3641 | |
| 3642 | * This is a mixed mapping case where we were not able to allocate
|
| 3643 | * a single contiguous extent. In that case let's reset requested
|
| 3644 | * mapping and call the slow path.
|
| 3645 |
|
| 3646 | map->m_lblk = m_lblk;
|
| 3647 | map->m_len = m_len;
|
| 3648 | map->m_flags = 0;
|
| 3649 |
|
| 3650 | |
| 3651 | * slow path means we have mixed mapping, that means we will need
|
| 3652 | * to force txn commit.
|
| 3653 |
|
| 3654 | *force_commit = true;
|
| 3655 | return ext4_map_blocks_atomic_write_slow(handle, inode, map);
|
| 3656 | out:
|
| 3657 | return ret;
|
| 3658 | }
|
| 3659 |
|
| 3660 | static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map,
|
| 3661 | unsigned int flags)
|
| 3662 | {
|
| 3663 | handle_t *handle;
|
| 3664 | int ret, dio_credits, m_flags = 0, retries = 0;
|
| 3665 | bool force_commit = false;
|
| 3666 |
|
| 3667 | if (flags & IOMAP_NOWAIT)
|
| 3668 | return -EAGAIN;
|
| 3669 |
|
| 3670 | |
| 3671 | * Trim the mapping request to the maximum value that we can map at
|
| 3672 | * once for direct I/O.
|
| 3673 |
|
| 3674 | if (map->m_len > DIO_MAX_BLOCKS)
|
| 3675 | map->m_len = DIO_MAX_BLOCKS;
|
| 3676 |
|
| 3677 | |
| 3678 | * journal credits estimation for atomic writes. We call
|
| 3679 | * ext4_map_blocks(), to find if there could be a mixed mapping. If yes,
|
| 3680 | * then let's assume the no. of pextents required can be m_len i.e.
|
| 3681 | * every alternate block can be unwritten and hole.
|
| 3682 |
|
| 3683 | if (flags & IOMAP_ATOMIC) {
|
| 3684 | unsigned int orig_mlen = map->m_len;
|
| 3685 |
|
| 3686 | ret = ext4_map_blocks(NULL, inode, map, 0);
|
| 3687 | if (ret < 0)
|
| 3688 | return ret;
|
| 3689 | if (map->m_len < orig_mlen) {
|
| 3690 | map->m_len = orig_mlen;
|
| 3691 | dio_credits = ext4_meta_trans_blocks(inode, map->m_len,
|
| 3692 | map->m_len, 0);
|
| 3693 | } else {
|
| 3694 | dio_credits = ext4_chunk_trans_blocks(inode,
|
| 3695 | map->m_len);
|
| 3696 | }
|
| 3697 | } else {
|
| 3698 | dio_credits = ext4_chunk_trans_blocks(inode, map->m_len);
|
| 3699 | }
|
| 3700 |
|
| 3701 | retry:
|
| 3702 | |
| 3703 | * Either we allocate blocks and then don't get an unwritten extent, so
|
| 3704 | * in that case we have reserved enough credits. Or, the blocks are
|
| 3705 | * already allocated and unwritten. In that case, the extent conversion
|
| 3706 | * fits into the credits as well.
|
| 3707 |
|
| 3708 | handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, dio_credits);
|
| 3709 | if (IS_ERR(handle))
|
| 3710 | return PTR_ERR(handle);
|
| 3711 |
|
| 3712 | |
| 3713 | * DAX and direct I/O are the only two operations that are currently
|
| 3714 | * supported with IOMAP_WRITE.
|
| 3715 |
|
| 3716 | WARN_ON(!(flags & (IOMAP_DAX | IOMAP_DIRECT)));
|
| 3717 | if (flags & IOMAP_DAX)
|
| 3718 | m_flags = EXT4_GET_BLOCKS_CREATE_ZERO;
|
| 3719 | |
| 3720 | * We use i_size instead of i_disksize here because delalloc writeback
|
| 3721 | * can complete at any point during the I/O and subsequently push the
|
| 3722 | * i_disksize out to i_size. This could be beyond where direct I/O is
|
| 3723 | * happening and thus expose allocated blocks to direct I/O reads.
|
| 3724 |
|
| 3725 | else if (EXT4_LBLK_TO_B(inode, map->m_lblk) >= i_size_read(inode))
|
| 3726 | m_flags = EXT4_GET_BLOCKS_CREATE;
|
| 3727 | else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
|
| 3728 | m_flags = EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT;
|
| 3729 |
|
| 3730 | if (flags & IOMAP_ATOMIC)
|
| 3731 | ret = ext4_map_blocks_atomic_write(handle, inode, map, m_flags,
|
| 3732 | &force_commit);
|
| 3733 | else
|
| 3734 | ret = ext4_map_blocks(handle, inode, map, m_flags);
|
| 3735 |
|
| 3736 | |
| 3737 | * We cannot fill holes in indirect tree based inodes as that could
|
| 3738 | * expose stale data in the case of a crash. Use the magic error code
|
| 3739 | * to fallback to buffered I/O.
|
| 3740 |
|
| 3741 | if (!m_flags && !ret)
|
| 3742 | ret = -ENOTBLK;
|
| 3743 |
|
| 3744 | ext4_journal_stop(handle);
|
| 3745 | if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
|
| 3746 | goto retry;
|
| 3747 |
|
| 3748 | |
| 3749 | * Force commit the current transaction if the allocation spans a mixed
|
| 3750 | * mapping range. This ensures any pending metadata updates (like
|
| 3751 | * unwritten to written extents conversion) in this range are in
|
| 3752 | * consistent state with the file data blocks, before performing the
|
| 3753 | * actual write I/O. If the commit fails, the whole I/O must be aborted
|
| 3754 | * to prevent any possible torn writes.
|
| 3755 |
|
| 3756 | if (ret > 0 && force_commit) {
|
| 3757 | int ret2;
|
| 3758 |
|
| 3759 | ret2 = ext4_force_commit(inode->i_sb);
|
| 3760 | if (ret2)
|
| 3761 | return ret2;
|
| 3762 | }
|
| 3763 |
|
| 3764 | return ret;
|
| 3765 | }
|
| 3766 |
|
| 3767 |
|
| 3768 | int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
|
| 3769 | unsigned flags, struct iomap *iomap, struct iomap *srcmap)
|
| 3770 | {
|
| 3771 | int ret;
|
| 3772 | struct ext4_map_blocks map;
|
| 3773 | u8 blkbits = inode->i_blkbits;
|
| 3774 | unsigned int orig_mlen;
|
| 3775 |
|
| 3776 | if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)
|
| 3777 | return -EINVAL;
|
| 3778 |
|
| 3779 | if (WARN_ON_ONCE(ext4_has_inline_data(inode)))
|
| 3780 | return -ERANGE;
|
| 3781 |
|
| 3782 | |
| 3783 | * Calculate the first and last logical blocks respectively.
|
| 3784 |
|
| 3785 | map.m_lblk = offset >> blkbits;
|
| 3786 | map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits,
|
| 3787 | EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1;
|
| 3788 | orig_mlen = map.m_len;
|
| 3789 |
|
| 3790 | if (flags & IOMAP_WRITE) {
|
| 3791 | |
| 3792 | * We check here if the blocks are already allocated, then we
|
| 3793 | * don't need to start a journal txn and we can directly return
|
| 3794 | * the mapping information. This could boost performance
|
| 3795 | * especially in multi-threaded overwrite requests.
|
| 3796 |
|
| 3797 | if (offset + length <= i_size_read(inode)) {
|
| 3798 | ret = ext4_map_blocks(NULL, inode, &map, 0);
|
| 3799 | |
| 3800 | * For DAX we convert extents to initialized ones before
|
| 3801 | * copying the data, otherwise we do it after I/O so
|
| 3802 | * there's no need to call into ext4_iomap_alloc().
|
| 3803 |
|
| 3804 | if ((map.m_flags & EXT4_MAP_MAPPED) ||
|
| 3805 | (!(flags & IOMAP_DAX) &&
|
| 3806 | (map.m_flags & EXT4_MAP_UNWRITTEN))) {
|
| 3807 | |
| 3808 | * For atomic writes the entire requested
|
| 3809 | * length should be mapped.
|
| 3810 |
|
| 3811 | if (ret == orig_mlen ||
|
| 3812 | (!(flags & IOMAP_ATOMIC) && ret > 0))
|
| 3813 | goto out;
|
| 3814 | }
|
| 3815 | map.m_len = orig_mlen;
|
| 3816 | }
|
| 3817 | ret = ext4_iomap_alloc(inode, &map, flags);
|
| 3818 | } else {
|
| 3819 | ret = ext4_map_blocks(NULL, inode, &map, 0);
|
| 3820 | }
|
| 3821 |
|
| 3822 | if (ret < 0)
|
| 3823 | return ret;
|
| 3824 | out:
|
| 3825 | |
| 3826 | * Sometimes I/O to an encrypted file has to be broken up to guarantee
|
| 3827 | * DUN contiguity. Handle this by limiting the length of the mapping
|
| 3828 | * returned.
|
| 3829 |
|
| 3830 | map.m_len = fscrypt_limit_io_blocks(inode, map.m_lblk, map.m_len);
|
| 3831 |
|
| 3832 | |
| 3833 | * Before returning to iomap, let's ensure the allocated mapping
|
| 3834 | * covers the entire requested length for atomic writes.
|
| 3835 |
|
| 3836 | if (flags & IOMAP_ATOMIC) {
|
| 3837 | if (map.m_len < (length >> blkbits)) {
|
| 3838 | WARN_ON_ONCE(1);
|
| 3839 | return -EINVAL;
|
| 3840 | }
|
| 3841 | }
|
| 3842 | ext4_set_iomap(inode, iomap, &map, offset, length, flags);
|
| 3843 |
|
| 3844 | return 0;
|
| 3845 | }
|
| 3846 |
|
| 3847 | static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next, ext4_iomap_begin);
|
| 3848 |
|
| 3849 | const struct iomap_ops ext4_iomap_ops = {
|
| 3850 | .iomap_next = ext4_iomap_next,
|
| 3851 | };
|
| 3852 |
|
| 3853 | static int ext4_iomap_begin_report(struct inode *inode, loff_t offset,
|
| 3854 | loff_t length, unsigned int flags,
|
| 3855 | struct iomap *iomap, struct iomap *srcmap)
|
| 3856 | {
|
| 3857 | int ret;
|
| 3858 | struct ext4_map_blocks map;
|
| 3859 | u8 blkbits = inode->i_blkbits;
|
| 3860 |
|
| 3861 | if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)
|
| 3862 | return -EINVAL;
|
| 3863 |
|
| 3864 | if (ext4_has_inline_data(inode)) {
|
| 3865 | ret = ext4_inline_data_iomap(inode, iomap);
|
| 3866 | if (ret != -EAGAIN) {
|
| 3867 | if (ret == 0 && offset >= iomap->length)
|
| 3868 | ret = -ENOENT;
|
| 3869 | return ret;
|
| 3870 | }
|
| 3871 | }
|
| 3872 |
|
| 3873 | |
| 3874 | * Calculate the first and last logical block respectively.
|
| 3875 |
|
| 3876 | map.m_lblk = offset >> blkbits;
|
| 3877 | map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits,
|
| 3878 | EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1;
|
| 3879 |
|
| 3880 | |
| 3881 | * Fiemap callers may call for offset beyond s_bitmap_maxbytes.
|
| 3882 | * So handle it here itself instead of querying ext4_map_blocks().
|
| 3883 | * Since ext4_map_blocks() will warn about it and will return
|
| 3884 | * -EIO error.
|
| 3885 |
|
| 3886 | if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
|
| 3887 | struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
|
| 3888 |
|
| 3889 | if (offset >= sbi->s_bitmap_maxbytes) {
|
| 3890 | map.m_flags = 0;
|
| 3891 | goto set_iomap;
|
| 3892 | }
|
| 3893 | }
|
| 3894 |
|
| 3895 | ret = ext4_map_blocks(NULL, inode, &map, 0);
|
| 3896 | if (ret < 0)
|
| 3897 | return ret;
|
| 3898 | set_iomap:
|
| 3899 | ext4_set_iomap(inode, iomap, &map, offset, length, flags);
|
| 3900 |
|
| 3901 | return 0;
|
| 3902 | }
|
| 3903 |
|
| 3904 | static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next_report, ext4_iomap_begin_report);
|
| 3905 |
|
| 3906 | const struct iomap_ops ext4_iomap_report_ops = {
|
| 3907 | .iomap_next = ext4_iomap_next_report,
|
| 3908 | };
|
| 3909 |
|
| 3910 | |
| 3911 | * For data=journal mode, folio should be marked dirty only when it was
|
| 3912 | * writeably mapped. When that happens, it was already attached to the
|
| 3913 | * transaction and marked as jbddirty (we take care of this in
|
| 3914 | * ext4_page_mkwrite()). On transaction commit, we writeprotect page mappings
|
| 3915 | * so we should have nothing to do here, except for the case when someone
|
| 3916 | * had the page pinned and dirtied the page through this pin (e.g. by doing
|
| 3917 | * direct IO to it). In that case we'd need to attach buffers here to the
|
| 3918 | * transaction but we cannot due to lock ordering. We cannot just dirty the
|
| 3919 | * folio and leave attached buffers clean, because the buffers' dirty state is
|
| 3920 | * "definitive". We cannot just set the buffers dirty or jbddirty because all
|
| 3921 | * the journalling code will explode. So what we do is to mark the folio
|
| 3922 | * "pending dirty" and next time ext4_writepages() is called, attach buffers
|
| 3923 | * to the transaction appropriately.
|
| 3924 |
|
| 3925 | static bool ext4_journalled_dirty_folio(struct address_space *mapping,
|
| 3926 | struct folio *folio)
|
| 3927 | {
|
| 3928 | WARN_ON_ONCE(!folio_buffers(folio));
|
| 3929 | if (folio_maybe_dma_pinned(folio))
|
| 3930 | folio_set_checked(folio);
|
| 3931 | return filemap_dirty_folio(mapping, folio);
|
| 3932 | }
|
| 3933 |
|
| 3934 | static bool ext4_dirty_folio(struct address_space *mapping, struct folio *folio)
|
| 3935 | {
|
| 3936 | WARN_ON_ONCE(!folio_test_locked(folio) && !folio_test_dirty(folio));
|
| 3937 | WARN_ON_ONCE(!folio_buffers(folio));
|
| 3938 | return block_dirty_folio(mapping, folio);
|
| 3939 | }
|
| 3940 |
|
| 3941 | static int ext4_iomap_swap_activate(struct swap_info_struct *sis,
|
| 3942 | struct file *file, sector_t *span)
|
| 3943 | {
|
| 3944 | return iomap_swapfile_activate(sis, file, span,
|
| 3945 | &ext4_iomap_report_ops);
|
| 3946 | }
|
| 3947 |
|
| 3948 | static const struct address_space_operations ext4_aops = {
|
| 3949 | .read_folio = ext4_read_folio,
|
| 3950 | .readahead = ext4_readahead,
|
| 3951 | .writepages = ext4_writepages,
|
| 3952 | .write_begin = ext4_write_begin,
|
| 3953 | .write_end = ext4_write_end,
|
| 3954 | .dirty_folio = ext4_dirty_folio,
|
| 3955 | .bmap = ext4_bmap,
|
| 3956 | .invalidate_folio = ext4_invalidate_folio,
|
| 3957 | .release_folio = ext4_release_folio,
|
| 3958 | .migrate_folio = buffer_migrate_folio,
|
| 3959 | .is_partially_uptodate = block_is_partially_uptodate,
|
| 3960 | .error_remove_folio = generic_error_remove_folio,
|
| 3961 | .swap_activate = ext4_iomap_swap_activate,
|
| 3962 | };
|
| 3963 |
|
| 3964 | static const struct address_space_operations ext4_journalled_aops = {
|
| 3965 | .read_folio = ext4_read_folio,
|
| 3966 | .readahead = ext4_readahead,
|
| 3967 | .writepages = ext4_writepages,
|
| 3968 | .write_begin = ext4_write_begin,
|
| 3969 | .write_end = ext4_journalled_write_end,
|
| 3970 | .dirty_folio = ext4_journalled_dirty_folio,
|
| 3971 | .bmap = ext4_bmap,
|
| 3972 | .invalidate_folio = ext4_journalled_invalidate_folio,
|
| 3973 | .release_folio = ext4_release_folio,
|
| 3974 | .migrate_folio = buffer_migrate_folio_norefs,
|
| 3975 | .is_partially_uptodate = block_is_partially_uptodate,
|
| 3976 | .error_remove_folio = generic_error_remove_folio,
|
| 3977 | .swap_activate = ext4_iomap_swap_activate,
|
| 3978 | };
|
| 3979 |
|
| 3980 | static const struct address_space_operations ext4_da_aops = {
|
| 3981 | .read_folio = ext4_read_folio,
|
| 3982 | .readahead = ext4_readahead,
|
| 3983 | .writepages = ext4_writepages,
|
| 3984 | .write_begin = ext4_da_write_begin,
|
| 3985 | .write_end = ext4_da_write_end,
|
| 3986 | .dirty_folio = ext4_dirty_folio,
|
| 3987 | .bmap = ext4_bmap,
|
| 3988 | .invalidate_folio = ext4_invalidate_folio,
|
| 3989 | .release_folio = ext4_release_folio,
|
| 3990 | .migrate_folio = buffer_migrate_folio,
|
| 3991 | .is_partially_uptodate = block_is_partially_uptodate,
|
| 3992 | .error_remove_folio = generic_error_remove_folio,
|
| 3993 | .swap_activate = ext4_iomap_swap_activate,
|
| 3994 | };
|
| 3995 |
|
| 3996 | static const struct address_space_operations ext4_dax_aops = {
|
| 3997 | .writepages = ext4_dax_writepages,
|
| 3998 | .dirty_folio = noop_dirty_folio,
|
| 3999 | .bmap = ext4_bmap,
|
| 4000 | .swap_activate = ext4_iomap_swap_activate,
|
| 4001 | };
|
| 4002 |
|
| 4003 | void ext4_set_aops(struct inode *inode)
|
| 4004 | {
|
| 4005 | switch (ext4_inode_journal_mode(inode)) {
|
| 4006 | case EXT4_INODE_ORDERED_DATA_MODE:
|
| 4007 | case EXT4_INODE_WRITEBACK_DATA_MODE:
|
| 4008 | break;
|
| 4009 | case EXT4_INODE_JOURNAL_DATA_MODE:
|
| 4010 | inode->i_mapping->a_ops = &ext4_journalled_aops;
|
| 4011 | return;
|
| 4012 | default:
|
| 4013 | BUG();
|
| 4014 | }
|
| 4015 | if (IS_DAX(inode))
|
| 4016 | inode->i_mapping->a_ops = &ext4_dax_aops;
|
| 4017 | else if (test_opt(inode->i_sb, DELALLOC))
|
| 4018 | inode->i_mapping->a_ops = &ext4_da_aops;
|
| 4019 | else
|
| 4020 | inode->i_mapping->a_ops = &ext4_aops;
|
| 4021 | }
|
| 4022 |
|
| 4023 | |
| 4024 | * Here we can't skip an unwritten buffer even though it usually reads zero
|
| 4025 | * because it might have data in pagecache (eg, if called from ext4_zero_range,
|
| 4026 | * ext4_punch_hole, etc) which needs to be properly zeroed out. Otherwise a
|
| 4027 | * racing writeback can come later and flush the stale pagecache to disk.
|
| 4028 | *
|
| 4029 | * Return the loaded bh if it actually needs zeroing - in written, dirty
|
| 4030 | * unwritten, or delalloc state. Return NULL if it's clean (i.e., a hole or
|
| 4031 | * a clean unwritten block).
|
| 4032 |
|
| 4033 | static struct buffer_head *ext4_load_tail_bh(struct inode *inode, loff_t from)
|
| 4034 | {
|
| 4035 | unsigned int offset, blocksize, pos;
|
| 4036 | ext4_lblk_t iblock;
|
| 4037 | struct address_space *mapping = inode->i_mapping;
|
| 4038 | struct buffer_head *bh;
|
| 4039 | struct folio *folio;
|
| 4040 | int err = 0;
|
| 4041 |
|
| 4042 | folio = __filemap_get_folio(mapping, from >> PAGE_SHIFT,
|
| 4043 | FGP_WRITEBEGIN | FGP_ACCESSED,
|
| 4044 | mapping_gfp_constraint(mapping, ~__GFP_FS));
|
| 4045 | if (IS_ERR(folio))
|
| 4046 | return ERR_CAST(folio);
|
| 4047 |
|
| 4048 | blocksize = inode->i_sb->s_blocksize;
|
| 4049 |
|
| 4050 | iblock = EXT4_PG_TO_LBLK(inode, folio->index);
|
| 4051 |
|
| 4052 | bh = folio_buffers(folio);
|
| 4053 | if (!bh)
|
| 4054 | bh = create_empty_buffers(folio, blocksize, 0);
|
| 4055 |
|
| 4056 |
|
| 4057 | offset = offset_in_folio(folio, from);
|
| 4058 | pos = blocksize;
|
| 4059 | while (offset >= pos) {
|
| 4060 | bh = bh->b_this_page;
|
| 4061 | iblock++;
|
| 4062 | pos += blocksize;
|
| 4063 | }
|
| 4064 | if (buffer_freed(bh)) {
|
| 4065 | BUFFER_TRACE(bh, "freed: skip");
|
| 4066 | goto unlock;
|
| 4067 | }
|
| 4068 | if (!buffer_mapped(bh)) {
|
| 4069 | BUFFER_TRACE(bh, "unmapped");
|
| 4070 | err = ext4_get_block(inode, iblock, bh, 0);
|
| 4071 | if (err < 0)
|
| 4072 | goto unlock;
|
| 4073 | |
| 4074 | * It's a hole or a clean unwritten block - nothing to do.
|
| 4075 | * Note that a lookup-only get_block (without
|
| 4076 | * EXT4_GET_BLOCKS_CREATE) never sets BH_Mapped for clean
|
| 4077 | * unwritten extents.
|
| 4078 |
|
| 4079 | if (!buffer_mapped(bh)) {
|
| 4080 | BUFFER_TRACE(bh, "still unmapped");
|
| 4081 | goto unlock;
|
| 4082 | }
|
| 4083 | }
|
| 4084 |
|
| 4085 |
|
| 4086 | if (folio_test_uptodate(folio))
|
| 4087 | set_buffer_uptodate(bh);
|
| 4088 |
|
| 4089 | if (!buffer_uptodate(bh)) {
|
| 4090 | err = ext4_read_bh_lock(bh, 0, true);
|
| 4091 | if (err)
|
| 4092 | goto unlock;
|
| 4093 | }
|
| 4094 | return bh;
|
| 4095 |
|
| 4096 | unlock:
|
| 4097 | folio_unlock(folio);
|
| 4098 | folio_put(folio);
|
| 4099 | return err ? ERR_PTR(err) : NULL;
|
| 4100 | }
|
| 4101 |
|
| 4102 | static int ext4_block_do_zero_range(struct inode *inode, loff_t from,
|
| 4103 | loff_t length, bool *did_zero,
|
| 4104 | bool *zero_written)
|
| 4105 | {
|
| 4106 | struct buffer_head *bh;
|
| 4107 | struct folio *folio;
|
| 4108 |
|
| 4109 | bh = ext4_load_tail_bh(inode, from);
|
| 4110 | if (IS_ERR_OR_NULL(bh))
|
| 4111 | return PTR_ERR_OR_ZERO(bh);
|
| 4112 |
|
| 4113 | folio = bh->b_folio;
|
| 4114 | folio_zero_range(folio, offset_in_folio(folio, from), length);
|
| 4115 | BUFFER_TRACE(bh, "zeroed end of block");
|
| 4116 |
|
| 4117 | mark_buffer_dirty(bh);
|
| 4118 | if (did_zero)
|
| 4119 | *did_zero = true;
|
| 4120 | if (zero_written && !buffer_unwritten(bh) && !buffer_delay(bh))
|
| 4121 | *zero_written = true;
|
| 4122 |
|
| 4123 | folio_unlock(folio);
|
| 4124 | folio_put(folio);
|
| 4125 | return 0;
|
| 4126 | }
|
| 4127 |
|
| 4128 | static int ext4_block_journalled_zero_range(struct inode *inode, loff_t from,
|
| 4129 | loff_t length, bool *did_zero)
|
| 4130 | {
|
| 4131 | struct buffer_head *bh;
|
| 4132 | struct folio *folio;
|
| 4133 | handle_t *handle;
|
| 4134 | int err;
|
| 4135 |
|
| 4136 | handle = ext4_journal_start(inode, EXT4_HT_MISC, 1);
|
| 4137 | if (IS_ERR(handle))
|
| 4138 | return PTR_ERR(handle);
|
| 4139 |
|
| 4140 | bh = ext4_load_tail_bh(inode, from);
|
| 4141 | if (IS_ERR_OR_NULL(bh)) {
|
| 4142 | err = PTR_ERR_OR_ZERO(bh);
|
| 4143 | goto out_handle;
|
| 4144 | }
|
| 4145 | folio = bh->b_folio;
|
| 4146 |
|
| 4147 | BUFFER_TRACE(bh, "get write access");
|
| 4148 | err = ext4_journal_get_write_access(handle, inode->i_sb, bh,
|
| 4149 | EXT4_JTR_NONE);
|
| 4150 | if (err)
|
| 4151 | goto out;
|
| 4152 |
|
| 4153 | folio_zero_range(folio, offset_in_folio(folio, from), length);
|
| 4154 | BUFFER_TRACE(bh, "zeroed end of block");
|
| 4155 |
|
| 4156 | err = ext4_dirty_journalled_data(handle, bh);
|
| 4157 | if (err)
|
| 4158 | goto out;
|
| 4159 |
|
| 4160 | if (did_zero)
|
| 4161 | *did_zero = true;
|
| 4162 | out:
|
| 4163 | folio_unlock(folio);
|
| 4164 | folio_put(folio);
|
| 4165 | out_handle:
|
| 4166 | ext4_journal_stop(handle);
|
| 4167 | return err;
|
| 4168 | }
|
| 4169 |
|
| 4170 | |
| 4171 | * Zeros out a mapping of length 'length' starting from file offset
|
| 4172 | * 'from'. The range to be zero'd must be contained with in one block.
|
| 4173 | * If the specified range exceeds the end of the block it will be
|
| 4174 | * shortened to end of the block that corresponds to 'from'.
|
| 4175 |
|
| 4176 | static int ext4_block_zero_range(struct inode *inode,
|
| 4177 | loff_t from, loff_t length, bool *did_zero,
|
| 4178 | bool *zero_written)
|
| 4179 | {
|
| 4180 | unsigned blocksize = inode->i_sb->s_blocksize;
|
| 4181 | unsigned int max = blocksize - (from & (blocksize - 1));
|
| 4182 |
|
| 4183 | |
| 4184 | * correct length if it does not fall between
|
| 4185 | * 'from' and the end of the block
|
| 4186 |
|
| 4187 | if (length > max || length < 0)
|
| 4188 | length = max;
|
| 4189 |
|
| 4190 | if (IS_DAX(inode)) {
|
| 4191 | return dax_zero_range(inode, from, length, did_zero,
|
| 4192 | &ext4_iomap_ops);
|
| 4193 | } else if (ext4_should_journal_data(inode)) {
|
| 4194 | return ext4_block_journalled_zero_range(inode, from, length,
|
| 4195 | did_zero);
|
| 4196 | }
|
| 4197 | return ext4_block_do_zero_range(inode, from, length, did_zero,
|
| 4198 | zero_written);
|
| 4199 | }
|
| 4200 |
|
| 4201 | |
| 4202 | * Zero out a mapping from file offset 'from' up to the end of the block
|
| 4203 | * which corresponds to 'from' or to the given 'end' inside this block.
|
| 4204 | * This required during truncate up and performing append writes. We need
|
| 4205 | * to physically zero the tail end of that block so it doesn't yield old
|
| 4206 | * data if the file is grown.
|
| 4207 |
|
| 4208 | int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end)
|
| 4209 | {
|
| 4210 | unsigned int blocksize = i_blocksize(inode);
|
| 4211 | unsigned int offset;
|
| 4212 | loff_t length = end - from;
|
| 4213 | bool did_zero = false;
|
| 4214 | bool zero_written = false;
|
| 4215 | int err;
|
| 4216 |
|
| 4217 | offset = from & (blocksize - 1);
|
| 4218 | if (!offset || from >= end)
|
| 4219 | return 0;
|
| 4220 | |
| 4221 | * Inline data has no tail block to zero out. Note that a race with
|
| 4222 | * ext4_page_mkwrite() converting inline data to an extent without
|
| 4223 | * holding i_rwsem is safe, as that path zeroes the full block before
|
| 4224 | * copying in the inline data.
|
| 4225 |
|
| 4226 | if (ext4_has_inline_data(inode))
|
| 4227 | return 0;
|
| 4228 |
|
| 4229 | if (IS_ENCRYPTED(inode) && !fscrypt_has_encryption_key(inode))
|
| 4230 | return 0;
|
| 4231 |
|
| 4232 | if (length > blocksize - offset)
|
| 4233 | length = blocksize - offset;
|
| 4234 |
|
| 4235 | err = ext4_block_zero_range(inode, from, length,
|
| 4236 | &did_zero, &zero_written);
|
| 4237 | if (err)
|
| 4238 | return err;
|
| 4239 | |
| 4240 | * It's necessary to order zeroed data before update i_disksize when
|
| 4241 | * truncating up or performing an append write, because there might be
|
| 4242 | * exposing stale on-disk data which may caused by concurrent post-EOF
|
| 4243 | * mmap write during folio writeback.
|
| 4244 |
|
| 4245 | if (ext4_should_order_data(inode) &&
|
| 4246 | did_zero && zero_written && !IS_DAX(inode)) {
|
| 4247 | handle_t *handle;
|
| 4248 |
|
| 4249 | handle = ext4_journal_start(inode, EXT4_HT_MISC, 1);
|
| 4250 | if (IS_ERR(handle))
|
| 4251 | return PTR_ERR(handle);
|
| 4252 |
|
| 4253 | err = ext4_jbd2_inode_add_write(handle, inode, from, length);
|
| 4254 | ext4_journal_stop(handle);
|
| 4255 | if (err)
|
| 4256 | return err;
|
| 4257 | }
|
| 4258 |
|
| 4259 | return 0;
|
| 4260 | }
|
| 4261 |
|
| 4262 | |
| 4263 | * Zero out the unaligned head and tail of the [lstart, lstart+length)
|
| 4264 | * range.
|
| 4265 | *
|
| 4266 | * On return, @partial_zeroed records which edges actually got
|
| 4267 | * partial-zeroed. Set EXT4_PARTIAL_ZERO_START/EXT4_PARTIAL_ZERO_END if
|
| 4268 | * the head/tail block got actually partially zeroed (in written, dirty
|
| 4269 | * unwritten or delalloc state). Cleared if the head/tail block is a
|
| 4270 | * hole or a clean unwritten block, in which case there is nothing that
|
| 4271 | * needs zeroing. When the head and tail land in the same block, both
|
| 4272 | * bits are set together on a successful zeroing.
|
| 4273 |
|
| 4274 | int ext4_zero_partial_blocks(struct inode *inode, loff_t lstart, loff_t length,
|
| 4275 | unsigned int *partial_zeroed)
|
| 4276 | {
|
| 4277 | struct super_block *sb = inode->i_sb;
|
| 4278 | unsigned partial_start, partial_end;
|
| 4279 | ext4_fsblk_t start, end;
|
| 4280 | loff_t byte_end = (lstart + length - 1);
|
| 4281 | bool did_zero = false;
|
| 4282 | int err = 0;
|
| 4283 |
|
| 4284 | partial_start = lstart & (sb->s_blocksize - 1);
|
| 4285 | partial_end = byte_end & (sb->s_blocksize - 1);
|
| 4286 |
|
| 4287 | start = lstart >> sb->s_blocksize_bits;
|
| 4288 | end = byte_end >> sb->s_blocksize_bits;
|
| 4289 |
|
| 4290 |
|
| 4291 | if (start == end &&
|
| 4292 | (partial_start || (partial_end != sb->s_blocksize - 1))) {
|
| 4293 | err = ext4_block_zero_range(inode, lstart, length, &did_zero,
|
| 4294 | NULL);
|
| 4295 | if (did_zero)
|
| 4296 | *partial_zeroed |= (EXT4_PARTIAL_ZERO_START |
|
| 4297 | EXT4_PARTIAL_ZERO_END);
|
| 4298 | return err;
|
| 4299 | }
|
| 4300 |
|
| 4301 | if (partial_start) {
|
| 4302 | err = ext4_block_zero_range(inode, lstart, sb->s_blocksize,
|
| 4303 | &did_zero, NULL);
|
| 4304 | if (err)
|
| 4305 | return err;
|
| 4306 | if (did_zero)
|
| 4307 | *partial_zeroed |= EXT4_PARTIAL_ZERO_START;
|
| 4308 | }
|
| 4309 |
|
| 4310 | if (partial_end != sb->s_blocksize - 1) {
|
| 4311 | did_zero = false;
|
| 4312 | err = ext4_block_zero_range(inode, byte_end - partial_end,
|
| 4313 | partial_end + 1, &did_zero, NULL);
|
| 4314 | if (err)
|
| 4315 | return err;
|
| 4316 | if (did_zero)
|
| 4317 | *partial_zeroed |= EXT4_PARTIAL_ZERO_END;
|
| 4318 | }
|
| 4319 | return err;
|
| 4320 | }
|
| 4321 |
|
| 4322 | int ext4_can_truncate(struct inode *inode)
|
| 4323 | {
|
| 4324 | if (S_ISREG(inode->i_mode))
|
| 4325 | return 1;
|
| 4326 | if (S_ISDIR(inode->i_mode))
|
| 4327 | return 1;
|
| 4328 | if (S_ISLNK(inode->i_mode))
|
| 4329 | return !ext4_inode_is_fast_symlink(inode);
|
| 4330 | return 0;
|
| 4331 | }
|
| 4332 |
|
| 4333 | |
| 4334 | * We have to make sure i_disksize gets properly updated before we truncate
|
| 4335 | * page cache due to hole punching or zero range. Otherwise i_disksize update
|
| 4336 | * can get lost as it may have been postponed to submission of writeback but
|
| 4337 | * that will never happen if we remove the folio containing i_size from the
|
| 4338 | * page cache. Also if we punch hole within i_size but above i_disksize,
|
| 4339 | * following ext4_page_mkwrite() may mistakenly allocate written blocks over
|
| 4340 | * the hole and thus introduce allocated blocks beyond i_disksize which is
|
| 4341 | * not allowed (e2fsck would complain in case of crash).
|
| 4342 |
|
| 4343 | int ext4_update_disksize_before_punch(struct inode *inode, loff_t offset,
|
| 4344 | loff_t len)
|
| 4345 | {
|
| 4346 | handle_t *handle;
|
| 4347 | int ret;
|
| 4348 |
|
| 4349 | loff_t size = i_size_read(inode);
|
| 4350 |
|
| 4351 | WARN_ON(!inode_is_locked(inode));
|
| 4352 | if (offset > size)
|
| 4353 | return 0;
|
| 4354 |
|
| 4355 | if (offset + len < size)
|
| 4356 | size = offset + len;
|
| 4357 | if (EXT4_I(inode)->i_disksize >= size)
|
| 4358 | return 0;
|
| 4359 |
|
| 4360 | handle = ext4_journal_start(inode, EXT4_HT_MISC, 1);
|
| 4361 | if (IS_ERR(handle))
|
| 4362 | return PTR_ERR(handle);
|
| 4363 | ext4_update_i_disksize(inode, size);
|
| 4364 | ret = ext4_mark_inode_dirty(handle, inode);
|
| 4365 | ext4_journal_stop(handle);
|
| 4366 |
|
| 4367 | return ret;
|
| 4368 | }
|
| 4369 |
|
| 4370 | static inline void ext4_truncate_folio(struct inode *inode,
|
| 4371 | loff_t start, loff_t end)
|
| 4372 | {
|
| 4373 | unsigned long blocksize = i_blocksize(inode);
|
| 4374 | struct folio *folio;
|
| 4375 |
|
| 4376 |
|
| 4377 | if (round_up(start, blocksize) >= round_down(end, blocksize))
|
| 4378 | return;
|
| 4379 |
|
| 4380 | folio = filemap_lock_folio(inode->i_mapping, start >> PAGE_SHIFT);
|
| 4381 | if (IS_ERR(folio))
|
| 4382 | return;
|
| 4383 |
|
| 4384 | if (folio_mkclean(folio))
|
| 4385 | folio_mark_dirty(folio);
|
| 4386 | folio_unlock(folio);
|
| 4387 | folio_put(folio);
|
| 4388 | }
|
| 4389 |
|
| 4390 | int ext4_truncate_page_cache_block_range(struct inode *inode,
|
| 4391 | loff_t start, loff_t end)
|
| 4392 | {
|
| 4393 | unsigned long blocksize = i_blocksize(inode);
|
| 4394 | int ret;
|
| 4395 |
|
| 4396 | |
| 4397 | * For journalled data we need to write (and checkpoint) pages
|
| 4398 | * before discarding page cache to avoid inconsitent data on disk
|
| 4399 | * in case of crash before freeing or unwritten converting trans
|
| 4400 | * is committed.
|
| 4401 |
|
| 4402 | if (ext4_should_journal_data(inode)) {
|
| 4403 | ret = filemap_write_and_wait_range(inode->i_mapping, start,
|
| 4404 | end - 1);
|
| 4405 | if (ret)
|
| 4406 | return ret;
|
| 4407 | goto truncate_pagecache;
|
| 4408 | }
|
| 4409 |
|
| 4410 | |
| 4411 | * If the block size is less than the page size, the file's mapped
|
| 4412 | * blocks within one page could be freed or converted to unwritten.
|
| 4413 | * So it's necessary to remove writable userspace mappings, and then
|
| 4414 | * ext4_page_mkwrite() can be called during subsequent write access
|
| 4415 | * to these partial folios.
|
| 4416 |
|
| 4417 | if (!IS_ALIGNED(start | end, PAGE_SIZE) &&
|
| 4418 | blocksize < PAGE_SIZE && start < inode->i_size) {
|
| 4419 | loff_t page_boundary = round_up(start, PAGE_SIZE);
|
| 4420 |
|
| 4421 | ext4_truncate_folio(inode, start, min(page_boundary, end));
|
| 4422 | if (end > page_boundary)
|
| 4423 | ext4_truncate_folio(inode,
|
| 4424 | round_down(end, PAGE_SIZE), end);
|
| 4425 | }
|
| 4426 |
|
| 4427 | truncate_pagecache:
|
| 4428 | truncate_pagecache_range(inode, start, end - 1);
|
| 4429 | return 0;
|
| 4430 | }
|
| 4431 |
|
| 4432 | static void ext4_wait_dax_page(struct inode *inode)
|
| 4433 | {
|
| 4434 | filemap_invalidate_unlock(inode->i_mapping);
|
| 4435 | schedule();
|
| 4436 | filemap_invalidate_lock(inode->i_mapping);
|
| 4437 | }
|
| 4438 |
|
| 4439 | int ext4_break_layouts(struct inode *inode)
|
| 4440 | {
|
| 4441 | if (WARN_ON_ONCE(!rwsem_is_locked(&inode->i_mapping->invalidate_lock)))
|
| 4442 | return -EINVAL;
|
| 4443 |
|
| 4444 | return dax_break_layout_inode(inode, ext4_wait_dax_page);
|
| 4445 | }
|
| 4446 |
|
| 4447 | |
| 4448 | * ext4_punch_hole: punches a hole in a file by releasing the blocks
|
| 4449 | * associated with the given offset and length
|
| 4450 | *
|
| 4451 | * @inode: File inode
|
| 4452 | * @offset: The offset where the hole will begin
|
| 4453 | * @len: The length of the hole
|
| 4454 | *
|
| 4455 | * Returns: 0 on success or negative on failure
|
| 4456 |
|
| 4457 |
|
| 4458 | int ext4_punch_hole(struct file *file, loff_t offset, loff_t length)
|
| 4459 | {
|
| 4460 | struct inode *inode = file_inode(file);
|
| 4461 | struct super_block *sb = inode->i_sb;
|
| 4462 | ext4_lblk_t start_lblk, end_lblk;
|
| 4463 | loff_t max_end = sb->s_maxbytes;
|
| 4464 | loff_t end = offset + length;
|
| 4465 | handle_t *handle;
|
| 4466 | unsigned int credits;
|
| 4467 | unsigned int partial_zeroed = 0;
|
| 4468 | int ret;
|
| 4469 |
|
| 4470 | trace_ext4_punch_hole(inode, offset, length, 0);
|
| 4471 | WARN_ON_ONCE(!inode_is_locked(inode));
|
| 4472 |
|
| 4473 | |
| 4474 | * For indirect-block based inodes, make sure that the hole within
|
| 4475 | * one block before last range.
|
| 4476 |
|
| 4477 | if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
|
| 4478 | max_end = EXT4_SB(sb)->s_bitmap_maxbytes - sb->s_blocksize;
|
| 4479 |
|
| 4480 |
|
| 4481 | if (offset >= inode->i_size || offset >= max_end)
|
| 4482 | return 0;
|
| 4483 |
|
| 4484 | |
| 4485 | * If the hole extends beyond i_size, set the hole to end after
|
| 4486 | * the block that contains i_size to save pointless tail block zeroing.
|
| 4487 |
|
| 4488 | if (end >= inode->i_size)
|
| 4489 | end = round_up(inode->i_size, sb->s_blocksize);
|
| 4490 | if (end > max_end)
|
| 4491 | end = max_end;
|
| 4492 | length = end - offset;
|
| 4493 |
|
| 4494 | ret = ext4_update_disksize_before_punch(inode, offset, length);
|
| 4495 | if (ret)
|
| 4496 | return ret;
|
| 4497 |
|
| 4498 |
|
| 4499 | ret = ext4_truncate_page_cache_block_range(inode, offset, end);
|
| 4500 | if (ret)
|
| 4501 | return ret;
|
| 4502 |
|
| 4503 | ret = ext4_zero_partial_blocks(inode, offset, length, &partial_zeroed);
|
| 4504 | if (ret)
|
| 4505 | return ret;
|
| 4506 | if (((file->f_flags & O_SYNC) || IS_SYNC(inode)) && partial_zeroed) {
|
| 4507 | ret = filemap_write_and_wait_range(inode->i_mapping, offset,
|
| 4508 | end - 1);
|
| 4509 | if (ret)
|
| 4510 | return ret;
|
| 4511 | }
|
| 4512 |
|
| 4513 | if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
|
| 4514 | credits = ext4_chunk_trans_extent(inode, 0);
|
| 4515 | else
|
| 4516 | credits = ext4_blocks_for_truncate(inode);
|
| 4517 | handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
|
| 4518 | if (IS_ERR(handle)) {
|
| 4519 | ret = PTR_ERR(handle);
|
| 4520 | ext4_std_error(sb, ret);
|
| 4521 | return ret;
|
| 4522 | }
|
| 4523 |
|
| 4524 |
|
| 4525 | start_lblk = EXT4_B_TO_LBLK(inode, offset);
|
| 4526 | end_lblk = end >> inode->i_blkbits;
|
| 4527 |
|
| 4528 | if (end_lblk > start_lblk) {
|
| 4529 | ext4_lblk_t hole_len = end_lblk - start_lblk;
|
| 4530 |
|
| 4531 | ext4_fc_track_inode(handle, inode);
|
| 4532 | ext4_check_map_extents_env(inode);
|
| 4533 | down_write(&EXT4_I(inode)->i_data_sem);
|
| 4534 | ext4_discard_preallocations(inode);
|
| 4535 |
|
| 4536 | ext4_es_remove_extent(inode, start_lblk, hole_len);
|
| 4537 |
|
| 4538 | if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
|
| 4539 | ret = ext4_ext_remove_space(inode, start_lblk,
|
| 4540 | end_lblk - 1);
|
| 4541 | else
|
| 4542 | ret = ext4_ind_remove_space(handle, inode, start_lblk,
|
| 4543 | end_lblk);
|
| 4544 | if (ret) {
|
| 4545 | up_write(&EXT4_I(inode)->i_data_sem);
|
| 4546 | goto out_handle;
|
| 4547 | }
|
| 4548 |
|
| 4549 | ext4_es_insert_extent(inode, start_lblk, hole_len, ~0,
|
| 4550 | EXTENT_STATUS_HOLE, 0);
|
| 4551 | up_write(&EXT4_I(inode)->i_data_sem);
|
| 4552 | }
|
| 4553 | ext4_fc_track_range(handle, inode, start_lblk, end_lblk);
|
| 4554 |
|
| 4555 | ret = ext4_mark_inode_dirty(handle, inode);
|
| 4556 | if (unlikely(ret))
|
| 4557 | goto out_handle;
|
| 4558 |
|
| 4559 | ext4_update_inode_fsync_trans(handle, inode, 1);
|
| 4560 | if ((file->f_flags & O_SYNC) || IS_SYNC(inode))
|
| 4561 | ext4_handle_sync(handle);
|
| 4562 | out_handle:
|
| 4563 | ext4_journal_stop(handle);
|
| 4564 | return ret;
|
| 4565 | }
|
| 4566 |
|
| 4567 | int ext4_inode_attach_jinode(struct inode *inode)
|
| 4568 | {
|
| 4569 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 4570 | struct jbd2_inode *jinode;
|
| 4571 |
|
| 4572 | if (ei->jinode || !EXT4_SB(inode->i_sb)->s_journal)
|
| 4573 | return 0;
|
| 4574 |
|
| 4575 | jinode = jbd2_alloc_inode(GFP_KERNEL);
|
| 4576 | spin_lock(&inode->i_lock);
|
| 4577 | if (!ei->jinode) {
|
| 4578 | if (!jinode) {
|
| 4579 | spin_unlock(&inode->i_lock);
|
| 4580 | return -ENOMEM;
|
| 4581 | }
|
| 4582 | jbd2_journal_init_jbd_inode(jinode, inode);
|
| 4583 | |
| 4584 | * Publish ->jinode only after it is fully initialized so that
|
| 4585 | * readers never observe a partially initialized jbd2_inode.
|
| 4586 |
|
| 4587 | smp_wmb();
|
| 4588 | WRITE_ONCE(ei->jinode, jinode);
|
| 4589 | jinode = NULL;
|
| 4590 | }
|
| 4591 | spin_unlock(&inode->i_lock);
|
| 4592 | if (unlikely(jinode != NULL))
|
| 4593 | jbd2_free_inode(jinode);
|
| 4594 | return 0;
|
| 4595 | }
|
| 4596 |
|
| 4597 | |
| 4598 | * ext4_truncate()
|
| 4599 | *
|
| 4600 | * We block out ext4_get_block() block instantiations across the entire
|
| 4601 | * transaction, and VFS/VM ensures that ext4_truncate() cannot run
|
| 4602 | * simultaneously on behalf of the same inode.
|
| 4603 | *
|
| 4604 | * As we work through the truncate and commit bits of it to the journal there
|
| 4605 | * is one core, guiding principle: the file's tree must always be consistent on
|
| 4606 | * disk. We must be able to restart the truncate after a crash.
|
| 4607 | *
|
| 4608 | * The file's tree may be transiently inconsistent in memory (although it
|
| 4609 | * probably isn't), but whenever we close off and commit a journal transaction,
|
| 4610 | * the contents of (the filesystem + the journal) must be consistent and
|
| 4611 | * restartable. It's pretty simple, really: bottom up, right to left (although
|
| 4612 | * left-to-right works OK too).
|
| 4613 | *
|
| 4614 | * Note that at recovery time, journal replay occurs *before* the restart of
|
| 4615 | * truncate against the orphan inode list.
|
| 4616 | *
|
| 4617 | * The committed inode has the new, desired i_size (which is the same as
|
| 4618 | * i_disksize in this case). After a crash, ext4_orphan_cleanup() will see
|
| 4619 | * that this inode's truncate did not complete and it will again call
|
| 4620 | * ext4_truncate() to have another go. So there will be instantiated blocks
|
| 4621 | * to the right of the truncation point in a crashed ext4 filesystem. But
|
| 4622 | * that's fine - as long as they are linked from the inode, the post-crash
|
| 4623 | * ext4_truncate() run will find them and release them.
|
| 4624 |
|
| 4625 | int ext4_truncate(struct inode *inode)
|
| 4626 | {
|
| 4627 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 4628 | unsigned int credits;
|
| 4629 | int err = 0, err2;
|
| 4630 | handle_t *handle;
|
| 4631 |
|
| 4632 | |
| 4633 | * There is a possibility that we're either freeing the inode
|
| 4634 | * or it's a completely new inode. In those cases we might not
|
| 4635 | * have i_rwsem locked because it's not necessary.
|
| 4636 |
|
| 4637 | if (!(inode_state_read_once(inode) & (I_NEW | I_FREEING)))
|
| 4638 | WARN_ON(!inode_is_locked(inode));
|
| 4639 | trace_ext4_truncate_enter(inode);
|
| 4640 |
|
| 4641 | if (!ext4_can_truncate(inode))
|
| 4642 | goto out_trace;
|
| 4643 |
|
| 4644 | if (inode->i_size == 0 && !test_opt(inode->i_sb, NO_AUTO_DA_ALLOC))
|
| 4645 | ext4_set_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE);
|
| 4646 |
|
| 4647 | if (ext4_has_inline_data(inode)) {
|
| 4648 | int has_inline = 1;
|
| 4649 |
|
| 4650 | err = ext4_inline_data_truncate(inode, &has_inline);
|
| 4651 | if (err || has_inline)
|
| 4652 | goto out_trace;
|
| 4653 | }
|
| 4654 |
|
| 4655 |
|
| 4656 | if (inode->i_size & (inode->i_sb->s_blocksize - 1)) {
|
| 4657 | err = ext4_inode_attach_jinode(inode);
|
| 4658 | if (err)
|
| 4659 | goto out_trace;
|
| 4660 |
|
| 4661 |
|
| 4662 | err = ext4_block_zero_eof(inode, inode->i_size, LLONG_MAX);
|
| 4663 | if (err)
|
| 4664 | goto out_trace;
|
| 4665 | }
|
| 4666 |
|
| 4667 | if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
|
| 4668 | credits = ext4_chunk_trans_extent(inode, 1);
|
| 4669 | else
|
| 4670 | credits = ext4_blocks_for_truncate(inode);
|
| 4671 |
|
| 4672 | handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
|
| 4673 | if (IS_ERR(handle)) {
|
| 4674 | err = PTR_ERR(handle);
|
| 4675 | goto out_trace;
|
| 4676 | }
|
| 4677 |
|
| 4678 | |
| 4679 | * We add the inode to the orphan list, so that if this
|
| 4680 | * truncate spans multiple transactions, and we crash, we will
|
| 4681 | * resume the truncate when the filesystem recovers. It also
|
| 4682 | * marks the inode dirty, to catch the new size.
|
| 4683 | *
|
| 4684 | * Implication: the file must always be in a sane, consistent
|
| 4685 | * truncatable state while each transaction commits.
|
| 4686 |
|
| 4687 | err = ext4_orphan_add(handle, inode);
|
| 4688 | if (err)
|
| 4689 | goto out_stop;
|
| 4690 |
|
| 4691 | ext4_fc_track_inode(handle, inode);
|
| 4692 | ext4_check_map_extents_env(inode);
|
| 4693 |
|
| 4694 | down_write(&EXT4_I(inode)->i_data_sem);
|
| 4695 | ext4_discard_preallocations(inode);
|
| 4696 |
|
| 4697 | if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
|
| 4698 | err = ext4_ext_truncate(handle, inode);
|
| 4699 | else
|
| 4700 | ext4_ind_truncate(handle, inode);
|
| 4701 |
|
| 4702 | up_write(&ei->i_data_sem);
|
| 4703 | if (err)
|
| 4704 | goto out_stop;
|
| 4705 |
|
| 4706 | if (IS_SYNC(inode))
|
| 4707 | ext4_handle_sync(handle);
|
| 4708 |
|
| 4709 | out_stop:
|
| 4710 | |
| 4711 | * If this was a simple ftruncate() and the file will remain alive,
|
| 4712 | * then we need to clear up the orphan record which we created above.
|
| 4713 | * However, if this was a real unlink then we were called by
|
| 4714 | * ext4_evict_inode(), and we allow that function to clean up the
|
| 4715 | * orphan info for us.
|
| 4716 |
|
| 4717 | if (inode->i_nlink)
|
| 4718 | ext4_orphan_del(handle, inode);
|
| 4719 |
|
| 4720 | inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode));
|
| 4721 | err2 = ext4_mark_inode_dirty(handle, inode);
|
| 4722 | if (unlikely(err2 && !err))
|
| 4723 | err = err2;
|
| 4724 | ext4_journal_stop(handle);
|
| 4725 |
|
| 4726 | out_trace:
|
| 4727 | trace_ext4_truncate_exit(inode);
|
| 4728 | return err;
|
| 4729 | }
|
| 4730 |
|
| 4731 | static inline u64 ext4_inode_peek_iversion(const struct inode *inode)
|
| 4732 | {
|
| 4733 | if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
|
| 4734 | return inode_peek_iversion_raw(inode);
|
| 4735 | else
|
| 4736 | return inode_peek_iversion(inode);
|
| 4737 | }
|
| 4738 |
|
| 4739 | static int ext4_inode_blocks_set(struct ext4_inode *raw_inode,
|
| 4740 | struct ext4_inode_info *ei)
|
| 4741 | {
|
| 4742 | struct inode *inode = &(ei->vfs_inode);
|
| 4743 | u64 i_blocks = READ_ONCE(inode->i_blocks);
|
| 4744 | struct super_block *sb = inode->i_sb;
|
| 4745 |
|
| 4746 | if (i_blocks <= ~0U) {
|
| 4747 | |
| 4748 | * i_blocks can be represented in a 32 bit variable
|
| 4749 | * as multiple of 512 bytes
|
| 4750 |
|
| 4751 | raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
|
| 4752 | raw_inode->i_blocks_high = 0;
|
| 4753 | ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
|
| 4754 | return 0;
|
| 4755 | }
|
| 4756 |
|
| 4757 | |
| 4758 | * This should never happen since sb->s_maxbytes should not have
|
| 4759 | * allowed this, sb->s_maxbytes was set according to the huge_file
|
| 4760 | * feature in ext4_fill_super().
|
| 4761 |
|
| 4762 | if (!ext4_has_feature_huge_file(sb))
|
| 4763 | return -EFSCORRUPTED;
|
| 4764 |
|
| 4765 | if (i_blocks <= 0xffffffffffffULL) {
|
| 4766 | |
| 4767 | * i_blocks can be represented in a 48 bit variable
|
| 4768 | * as multiple of 512 bytes
|
| 4769 |
|
| 4770 | raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
|
| 4771 | raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
|
| 4772 | ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
|
| 4773 | } else {
|
| 4774 | ext4_set_inode_flag(inode, EXT4_INODE_HUGE_FILE);
|
| 4775 |
|
| 4776 | i_blocks = i_blocks >> (inode->i_blkbits - 9);
|
| 4777 | raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
|
| 4778 | raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
|
| 4779 | }
|
| 4780 | return 0;
|
| 4781 | }
|
| 4782 |
|
| 4783 | static int ext4_fill_raw_inode(struct inode *inode, struct ext4_inode *raw_inode)
|
| 4784 | {
|
| 4785 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 4786 | uid_t i_uid;
|
| 4787 | gid_t i_gid;
|
| 4788 | projid_t i_projid;
|
| 4789 | int block;
|
| 4790 | int err;
|
| 4791 |
|
| 4792 | err = ext4_inode_blocks_set(raw_inode, ei);
|
| 4793 |
|
| 4794 | raw_inode->i_mode = cpu_to_le16(inode->i_mode);
|
| 4795 | i_uid = i_uid_read(inode);
|
| 4796 | i_gid = i_gid_read(inode);
|
| 4797 | i_projid = from_kprojid(&init_user_ns, ei->i_projid);
|
| 4798 | if (!(test_opt(inode->i_sb, NO_UID32))) {
|
| 4799 | raw_inode->i_uid_low = cpu_to_le16(low_16_bits(i_uid));
|
| 4800 | raw_inode->i_gid_low = cpu_to_le16(low_16_bits(i_gid));
|
| 4801 | |
| 4802 | * Fix up interoperability with old kernels. Otherwise,
|
| 4803 | * old inodes get re-used with the upper 16 bits of the
|
| 4804 | * uid/gid intact.
|
| 4805 |
|
| 4806 | if (ei->i_dtime && !ext4_inode_orphan_tracked(inode)) {
|
| 4807 | raw_inode->i_uid_high = 0;
|
| 4808 | raw_inode->i_gid_high = 0;
|
| 4809 | } else {
|
| 4810 | raw_inode->i_uid_high =
|
| 4811 | cpu_to_le16(high_16_bits(i_uid));
|
| 4812 | raw_inode->i_gid_high =
|
| 4813 | cpu_to_le16(high_16_bits(i_gid));
|
| 4814 | }
|
| 4815 | } else {
|
| 4816 | raw_inode->i_uid_low = cpu_to_le16(fs_high2lowuid(i_uid));
|
| 4817 | raw_inode->i_gid_low = cpu_to_le16(fs_high2lowgid(i_gid));
|
| 4818 | raw_inode->i_uid_high = 0;
|
| 4819 | raw_inode->i_gid_high = 0;
|
| 4820 | }
|
| 4821 | raw_inode->i_links_count = cpu_to_le16(inode->i_nlink);
|
| 4822 |
|
| 4823 | EXT4_INODE_SET_CTIME(inode, raw_inode);
|
| 4824 | EXT4_INODE_SET_MTIME(inode, raw_inode);
|
| 4825 | EXT4_INODE_SET_ATIME(inode, raw_inode);
|
| 4826 | EXT4_EINODE_SET_XTIME(i_crtime, ei, raw_inode);
|
| 4827 |
|
| 4828 | raw_inode->i_dtime = cpu_to_le32(ei->i_dtime);
|
| 4829 | raw_inode->i_flags = cpu_to_le32(ei->i_flags & 0xFFFFFFFF);
|
| 4830 | if (likely(!test_opt2(inode->i_sb, HURD_COMPAT)))
|
| 4831 | raw_inode->i_file_acl_high =
|
| 4832 | cpu_to_le16(ei->i_file_acl >> 32);
|
| 4833 | raw_inode->i_file_acl_lo = cpu_to_le32(ei->i_file_acl);
|
| 4834 | ext4_isize_set(raw_inode, ei->i_disksize);
|
| 4835 |
|
| 4836 | raw_inode->i_generation = cpu_to_le32(inode->i_generation);
|
| 4837 | if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) {
|
| 4838 | if (old_valid_dev(inode->i_rdev)) {
|
| 4839 | raw_inode->i_block[0] =
|
| 4840 | cpu_to_le32(old_encode_dev(inode->i_rdev));
|
| 4841 | raw_inode->i_block[1] = 0;
|
| 4842 | } else {
|
| 4843 | raw_inode->i_block[0] = 0;
|
| 4844 | raw_inode->i_block[1] =
|
| 4845 | cpu_to_le32(new_encode_dev(inode->i_rdev));
|
| 4846 | raw_inode->i_block[2] = 0;
|
| 4847 | }
|
| 4848 | } else if (!ext4_has_inline_data(inode)) {
|
| 4849 | for (block = 0; block < EXT4_N_BLOCKS; block++)
|
| 4850 | raw_inode->i_block[block] = ei->i_data[block];
|
| 4851 | }
|
| 4852 |
|
| 4853 | if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
|
| 4854 | u64 ivers = ext4_inode_peek_iversion(inode);
|
| 4855 |
|
| 4856 | raw_inode->i_disk_version = cpu_to_le32(ivers);
|
| 4857 | if (ei->i_extra_isize) {
|
| 4858 | if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
|
| 4859 | raw_inode->i_version_hi =
|
| 4860 | cpu_to_le32(ivers >> 32);
|
| 4861 | raw_inode->i_extra_isize =
|
| 4862 | cpu_to_le16(ei->i_extra_isize);
|
| 4863 | }
|
| 4864 | }
|
| 4865 |
|
| 4866 | if (i_projid != EXT4_DEF_PROJID &&
|
| 4867 | !ext4_has_feature_project(inode->i_sb))
|
| 4868 | err = err ?: -EFSCORRUPTED;
|
| 4869 |
|
| 4870 | if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
|
| 4871 | EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
|
| 4872 | raw_inode->i_projid = cpu_to_le32(i_projid);
|
| 4873 |
|
| 4874 | ext4_inode_csum_set(inode, raw_inode, ei);
|
| 4875 | return err;
|
| 4876 | }
|
| 4877 |
|
| 4878 | |
| 4879 | * ext4_get_inode_loc returns with an extra refcount against the inode's
|
| 4880 | * underlying buffer_head on success. If we pass 'inode' and it does not
|
| 4881 | * have in-inode xattr, we have all inode data in memory that is needed
|
| 4882 | * to recreate the on-disk version of this inode.
|
| 4883 |
|
| 4884 | static int __ext4_get_inode_loc(struct super_block *sb, unsigned long ino,
|
| 4885 | struct inode *inode, struct ext4_iloc *iloc,
|
| 4886 | ext4_fsblk_t *ret_block)
|
| 4887 | {
|
| 4888 | struct ext4_group_desc *gdp;
|
| 4889 | struct buffer_head *bh;
|
| 4890 | ext4_fsblk_t block;
|
| 4891 | struct blk_plug plug;
|
| 4892 | int inodes_per_block, inode_offset;
|
| 4893 |
|
| 4894 | iloc->bh = NULL;
|
| 4895 | if (ino < EXT4_ROOT_INO ||
|
| 4896 | ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count))
|
| 4897 | return -EFSCORRUPTED;
|
| 4898 |
|
| 4899 | iloc->block_group = (ino - 1) / EXT4_INODES_PER_GROUP(sb);
|
| 4900 | gdp = ext4_get_group_desc(sb, iloc->block_group, NULL);
|
| 4901 | if (!gdp)
|
| 4902 | return -EIO;
|
| 4903 |
|
| 4904 | |
| 4905 | * Figure out the offset within the block group inode table
|
| 4906 |
|
| 4907 | inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
|
| 4908 | inode_offset = ((ino - 1) %
|
| 4909 | EXT4_INODES_PER_GROUP(sb));
|
| 4910 | iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb);
|
| 4911 |
|
| 4912 | block = ext4_inode_table(sb, gdp);
|
| 4913 | if ((block <= le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block)) ||
|
| 4914 | (block >= ext4_blocks_count(EXT4_SB(sb)->s_es))) {
|
| 4915 | ext4_error(sb, "Invalid inode table block %llu in "
|
| 4916 | "block_group %u", block, iloc->block_group);
|
| 4917 | return -EFSCORRUPTED;
|
| 4918 | }
|
| 4919 | block += (inode_offset / inodes_per_block);
|
| 4920 |
|
| 4921 | bh = sb_getblk(sb, block);
|
| 4922 | if (unlikely(!bh))
|
| 4923 | return -ENOMEM;
|
| 4924 | if (ext4_buffer_uptodate(bh))
|
| 4925 | goto has_buffer;
|
| 4926 |
|
| 4927 | lock_buffer(bh);
|
| 4928 | if (ext4_buffer_uptodate(bh)) {
|
| 4929 |
|
| 4930 | unlock_buffer(bh);
|
| 4931 | goto has_buffer;
|
| 4932 | }
|
| 4933 |
|
| 4934 | |
| 4935 | * If we have all information of the inode in memory and this
|
| 4936 | * is the only valid inode in the block, we need not read the
|
| 4937 | * block.
|
| 4938 |
|
| 4939 | if (inode && !ext4_test_inode_state(inode, EXT4_STATE_XATTR)) {
|
| 4940 | struct buffer_head *bitmap_bh;
|
| 4941 | int i, start;
|
| 4942 |
|
| 4943 | start = inode_offset & ~(inodes_per_block - 1);
|
| 4944 |
|
| 4945 |
|
| 4946 | bitmap_bh = sb_getblk(sb, ext4_inode_bitmap(sb, gdp));
|
| 4947 | if (unlikely(!bitmap_bh))
|
| 4948 | goto make_io;
|
| 4949 |
|
| 4950 | |
| 4951 | * If the inode bitmap isn't in cache then the
|
| 4952 | * optimisation may end up performing two reads instead
|
| 4953 | * of one, so skip it.
|
| 4954 |
|
| 4955 | if (!buffer_uptodate(bitmap_bh)) {
|
| 4956 | brelse(bitmap_bh);
|
| 4957 | goto make_io;
|
| 4958 | }
|
| 4959 | for (i = start; i < start + inodes_per_block; i++) {
|
| 4960 | if (i == inode_offset)
|
| 4961 | continue;
|
| 4962 | if (ext4_test_bit(i, bitmap_bh->b_data))
|
| 4963 | break;
|
| 4964 | }
|
| 4965 | brelse(bitmap_bh);
|
| 4966 | if (i == start + inodes_per_block) {
|
| 4967 | struct ext4_inode *raw_inode =
|
| 4968 | (struct ext4_inode *) (bh->b_data + iloc->offset);
|
| 4969 |
|
| 4970 |
|
| 4971 | memset(bh->b_data, 0, bh->b_size);
|
| 4972 | if (!ext4_test_inode_state(inode, EXT4_STATE_NEW))
|
| 4973 | ext4_fill_raw_inode(inode, raw_inode);
|
| 4974 | set_buffer_uptodate(bh);
|
| 4975 | unlock_buffer(bh);
|
| 4976 | goto has_buffer;
|
| 4977 | }
|
| 4978 | }
|
| 4979 |
|
| 4980 | make_io:
|
| 4981 | |
| 4982 | * If we need to do any I/O, try to pre-readahead extra
|
| 4983 | * blocks from the inode table.
|
| 4984 |
|
| 4985 | blk_start_plug(&plug);
|
| 4986 | if (EXT4_SB(sb)->s_inode_readahead_blks) {
|
| 4987 | ext4_fsblk_t b, end, table;
|
| 4988 | unsigned num;
|
| 4989 | __u32 ra_blks = EXT4_SB(sb)->s_inode_readahead_blks;
|
| 4990 |
|
| 4991 | table = ext4_inode_table(sb, gdp);
|
| 4992 |
|
| 4993 | b = block & ~((ext4_fsblk_t) ra_blks - 1);
|
| 4994 | if (table > b)
|
| 4995 | b = table;
|
| 4996 | end = b + ra_blks;
|
| 4997 | num = EXT4_INODES_PER_GROUP(sb);
|
| 4998 | if (ext4_has_group_desc_csum(sb))
|
| 4999 | num -= ext4_itable_unused_count(sb, gdp);
|
| 5000 | table += num / inodes_per_block;
|
| 5001 | if (end > table)
|
| 5002 | end = table;
|
| 5003 | while (b <= end)
|
| 5004 | ext4_sb_breadahead_unmovable(sb, b++);
|
| 5005 | }
|
| 5006 |
|
| 5007 | |
| 5008 | * There are other valid inodes in the buffer, this inode
|
| 5009 | * has in-inode xattrs, or we don't have this inode in memory.
|
| 5010 | * Read the block from disk.
|
| 5011 |
|
| 5012 | trace_ext4_load_inode(sb, ino);
|
| 5013 | ext4_read_bh_nowait(bh, REQ_META | REQ_PRIO, NULL,
|
| 5014 | ext4_simulate_fail(sb, EXT4_SIM_INODE_EIO));
|
| 5015 | blk_finish_plug(&plug);
|
| 5016 | wait_on_buffer(bh);
|
| 5017 | if (!buffer_uptodate(bh)) {
|
| 5018 | if (ret_block)
|
| 5019 | *ret_block = block;
|
| 5020 | brelse(bh);
|
| 5021 | return -EIO;
|
| 5022 | }
|
| 5023 | has_buffer:
|
| 5024 | iloc->bh = bh;
|
| 5025 | return 0;
|
| 5026 | }
|
| 5027 |
|
| 5028 | static int __ext4_get_inode_loc_noinmem(struct inode *inode,
|
| 5029 | struct ext4_iloc *iloc)
|
| 5030 | {
|
| 5031 | ext4_fsblk_t err_blk = 0;
|
| 5032 | int ret;
|
| 5033 |
|
| 5034 | ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, NULL, iloc,
|
| 5035 | &err_blk);
|
| 5036 |
|
| 5037 | if (ret == -EIO)
|
| 5038 | ext4_error_inode_block(inode, err_blk, EIO,
|
| 5039 | "unable to read itable block");
|
| 5040 |
|
| 5041 | return ret;
|
| 5042 | }
|
| 5043 |
|
| 5044 | int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc)
|
| 5045 | {
|
| 5046 | ext4_fsblk_t err_blk = 0;
|
| 5047 | int ret;
|
| 5048 |
|
| 5049 | ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, inode, iloc,
|
| 5050 | &err_blk);
|
| 5051 |
|
| 5052 | if (ret == -EIO)
|
| 5053 | ext4_error_inode_block(inode, err_blk, EIO,
|
| 5054 | "unable to read itable block");
|
| 5055 |
|
| 5056 | return ret;
|
| 5057 | }
|
| 5058 |
|
| 5059 | |
| 5060 | * ext4_get_inode_loc_noio() is a best-effort variant of ext4_get_inode_loc().
|
| 5061 | * It looks up the inode table block in the buffer cache and returns -EAGAIN if
|
| 5062 | * the block is not present or not uptodate, without starting any I/O.
|
| 5063 |
|
| 5064 | int ext4_get_inode_loc_noio(struct inode *inode, struct ext4_iloc *iloc)
|
| 5065 | {
|
| 5066 | struct super_block *sb = inode->i_sb;
|
| 5067 | struct ext4_group_desc *gdp;
|
| 5068 | struct buffer_head *bh;
|
| 5069 | ext4_fsblk_t block;
|
| 5070 | int inodes_per_block, inode_offset;
|
| 5071 | unsigned long ino = inode->i_ino;
|
| 5072 |
|
| 5073 | iloc->bh = NULL;
|
| 5074 | if (ino < EXT4_ROOT_INO ||
|
| 5075 | ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count))
|
| 5076 | return -EFSCORRUPTED;
|
| 5077 |
|
| 5078 | iloc->block_group = (ino - 1) / EXT4_INODES_PER_GROUP(sb);
|
| 5079 | gdp = ext4_get_group_desc(sb, iloc->block_group, NULL);
|
| 5080 | if (!gdp)
|
| 5081 | return -EIO;
|
| 5082 |
|
| 5083 |
|
| 5084 | inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
|
| 5085 | inode_offset = ((ino - 1) % EXT4_INODES_PER_GROUP(sb));
|
| 5086 | iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb);
|
| 5087 |
|
| 5088 | block = ext4_inode_table(sb, gdp);
|
| 5089 | if (block <= le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block) ||
|
| 5090 | block >= ext4_blocks_count(EXT4_SB(sb)->s_es)) {
|
| 5091 | ext4_error(sb,
|
| 5092 | "Invalid inode table block %llu in block_group %u",
|
| 5093 | block, iloc->block_group);
|
| 5094 | return -EFSCORRUPTED;
|
| 5095 | }
|
| 5096 | block += inode_offset / inodes_per_block;
|
| 5097 |
|
| 5098 | bh = sb_find_get_block(sb, block);
|
| 5099 | if (!bh)
|
| 5100 | return -EAGAIN;
|
| 5101 | if (!ext4_buffer_uptodate(bh)) {
|
| 5102 | brelse(bh);
|
| 5103 | return -EAGAIN;
|
| 5104 | }
|
| 5105 |
|
| 5106 | iloc->bh = bh;
|
| 5107 | return 0;
|
| 5108 | }
|
| 5109 |
|
| 5110 |
|
| 5111 | int ext4_get_fc_inode_loc(struct super_block *sb, unsigned long ino,
|
| 5112 | struct ext4_iloc *iloc)
|
| 5113 | {
|
| 5114 | return __ext4_get_inode_loc(sb, ino, NULL, iloc, NULL);
|
| 5115 | }
|
| 5116 |
|
| 5117 | static bool ext4_should_enable_dax(struct inode *inode)
|
| 5118 | {
|
| 5119 | struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
|
| 5120 |
|
| 5121 | if (test_opt2(inode->i_sb, DAX_NEVER))
|
| 5122 | return false;
|
| 5123 | if (!S_ISREG(inode->i_mode))
|
| 5124 | return false;
|
| 5125 | if (ext4_should_journal_data(inode))
|
| 5126 | return false;
|
| 5127 | if (ext4_has_inline_data(inode))
|
| 5128 | return false;
|
| 5129 | if (ext4_test_inode_flag(inode, EXT4_INODE_ENCRYPT))
|
| 5130 | return false;
|
| 5131 | if (ext4_test_inode_flag(inode, EXT4_INODE_VERITY))
|
| 5132 | return false;
|
| 5133 | if (!test_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags))
|
| 5134 | return false;
|
| 5135 | if (test_opt(inode->i_sb, DAX_ALWAYS))
|
| 5136 | return true;
|
| 5137 |
|
| 5138 | return ext4_test_inode_flag(inode, EXT4_INODE_DAX);
|
| 5139 | }
|
| 5140 |
|
| 5141 | void ext4_set_inode_flags(struct inode *inode, bool init)
|
| 5142 | {
|
| 5143 | unsigned int flags = EXT4_I(inode)->i_flags;
|
| 5144 | unsigned int new_fl = 0;
|
| 5145 |
|
| 5146 | WARN_ON_ONCE(IS_DAX(inode) && init);
|
| 5147 |
|
| 5148 | if (flags & EXT4_SYNC_FL)
|
| 5149 | new_fl |= S_SYNC;
|
| 5150 | if (flags & EXT4_APPEND_FL)
|
| 5151 | new_fl |= S_APPEND;
|
| 5152 | if (flags & EXT4_IMMUTABLE_FL)
|
| 5153 | new_fl |= S_IMMUTABLE;
|
| 5154 | if (flags & EXT4_NOATIME_FL)
|
| 5155 | new_fl |= S_NOATIME;
|
| 5156 | if (flags & EXT4_DIRSYNC_FL)
|
| 5157 | new_fl |= S_DIRSYNC;
|
| 5158 |
|
| 5159 | |
| 5160 | * here if already set.
|
| 5161 | new_fl |= (inode->i_flags & S_DAX);
|
| 5162 | if (init && ext4_should_enable_dax(inode))
|
| 5163 | new_fl |= S_DAX;
|
| 5164 |
|
| 5165 | if (flags & EXT4_ENCRYPT_FL)
|
| 5166 | new_fl |= S_ENCRYPTED;
|
| 5167 | if (flags & EXT4_CASEFOLD_FL)
|
| 5168 | new_fl |= S_CASEFOLD;
|
| 5169 | if (flags & EXT4_VERITY_FL)
|
| 5170 | new_fl |= S_VERITY;
|
| 5171 | inode_set_flags(inode, new_fl,
|
| 5172 | S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|S_DAX|
|
| 5173 | S_ENCRYPTED|S_CASEFOLD|S_VERITY);
|
| 5174 | }
|
| 5175 |
|
| 5176 | static blkcnt_t ext4_inode_blocks(struct ext4_inode *raw_inode,
|
| 5177 | struct ext4_inode_info *ei)
|
| 5178 | {
|
| 5179 | blkcnt_t i_blocks ;
|
| 5180 | struct inode *inode = &(ei->vfs_inode);
|
| 5181 | struct super_block *sb = inode->i_sb;
|
| 5182 |
|
| 5183 | if (ext4_has_feature_huge_file(sb)) {
|
| 5184 |
|
| 5185 | i_blocks = ((u64)le16_to_cpu(raw_inode->i_blocks_high)) << 32 |
|
| 5186 | le32_to_cpu(raw_inode->i_blocks_lo);
|
| 5187 | if (ext4_test_inode_flag(inode, EXT4_INODE_HUGE_FILE)) {
|
| 5188 |
|
| 5189 | return i_blocks << (inode->i_blkbits - 9);
|
| 5190 | } else {
|
| 5191 | return i_blocks;
|
| 5192 | }
|
| 5193 | } else {
|
| 5194 | return le32_to_cpu(raw_inode->i_blocks_lo);
|
| 5195 | }
|
| 5196 | }
|
| 5197 |
|
| 5198 | static inline int ext4_iget_extra_inode(struct inode *inode,
|
| 5199 | struct ext4_inode *raw_inode,
|
| 5200 | struct ext4_inode_info *ei)
|
| 5201 | {
|
| 5202 | __le32 *magic = (void *)raw_inode +
|
| 5203 | EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize;
|
| 5204 |
|
| 5205 | if (EXT4_INODE_HAS_XATTR_SPACE(inode) &&
|
| 5206 | *magic == cpu_to_le32(EXT4_XATTR_MAGIC)) {
|
| 5207 | int err;
|
| 5208 |
|
| 5209 | err = xattr_check_inode(inode, IHDR(inode, raw_inode),
|
| 5210 | ITAIL(inode, raw_inode));
|
| 5211 | if (err)
|
| 5212 | return err;
|
| 5213 |
|
| 5214 | ext4_set_inode_state(inode, EXT4_STATE_XATTR);
|
| 5215 | err = ext4_find_inline_data_nolock(inode);
|
| 5216 | if (!err && ext4_has_inline_data(inode))
|
| 5217 | ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
|
| 5218 | return err;
|
| 5219 | } else
|
| 5220 | EXT4_I(inode)->i_inline_off = 0;
|
| 5221 | return 0;
|
| 5222 | }
|
| 5223 |
|
| 5224 | int ext4_get_projid(struct inode *inode, kprojid_t *projid)
|
| 5225 | {
|
| 5226 | if (!ext4_has_feature_project(inode->i_sb))
|
| 5227 | return -EOPNOTSUPP;
|
| 5228 | *projid = EXT4_I(inode)->i_projid;
|
| 5229 | return 0;
|
| 5230 | }
|
| 5231 |
|
| 5232 | |
| 5233 | * ext4 has self-managed i_version for ea inodes, it stores the lower 32bit of
|
| 5234 | * refcount in i_version, so use raw values if inode has EXT4_EA_INODE_FL flag
|
| 5235 | * set.
|
| 5236 |
|
| 5237 | static inline void ext4_inode_set_iversion_queried(struct inode *inode, u64 val)
|
| 5238 | {
|
| 5239 | if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
|
| 5240 | inode_set_iversion_raw(inode, val);
|
| 5241 | else
|
| 5242 | inode_set_iversion_queried(inode, val);
|
| 5243 | }
|
| 5244 |
|
| 5245 | static int check_igot_inode(struct inode *inode, ext4_iget_flags flags,
|
| 5246 | const char *function, unsigned int line)
|
| 5247 | {
|
| 5248 | const char *err_str;
|
| 5249 |
|
| 5250 | if (flags & EXT4_IGET_EA_INODE) {
|
| 5251 | if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) {
|
| 5252 | err_str = "missing EA_INODE flag";
|
| 5253 | goto error;
|
| 5254 | }
|
| 5255 | if (ext4_test_inode_state(inode, EXT4_STATE_XATTR) ||
|
| 5256 | EXT4_I(inode)->i_file_acl) {
|
| 5257 | err_str = "ea_inode with extended attributes";
|
| 5258 | goto error;
|
| 5259 | }
|
| 5260 | } else {
|
| 5261 | if ((EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) {
|
| 5262 | |
| 5263 | * open_by_handle_at() could provide an old inode number
|
| 5264 | * that has since been reused for an ea_inode; this does
|
| 5265 | * not indicate filesystem corruption
|
| 5266 |
|
| 5267 | if (flags & EXT4_IGET_HANDLE)
|
| 5268 | return -ESTALE;
|
| 5269 | err_str = "unexpected EA_INODE flag";
|
| 5270 | goto error;
|
| 5271 | }
|
| 5272 | }
|
| 5273 | if (is_bad_inode(inode) && !(flags & EXT4_IGET_BAD)) {
|
| 5274 | err_str = "unexpected bad inode w/o EXT4_IGET_BAD";
|
| 5275 | goto error;
|
| 5276 | }
|
| 5277 | return 0;
|
| 5278 |
|
| 5279 | error:
|
| 5280 | ext4_error_inode(inode, function, line, 0, "%s", err_str);
|
| 5281 | return -EFSCORRUPTED;
|
| 5282 | }
|
| 5283 |
|
| 5284 | void ext4_set_inode_mapping_order(struct inode *inode)
|
| 5285 | {
|
| 5286 | struct super_block *sb = inode->i_sb;
|
| 5287 | u16 min_order, max_order;
|
| 5288 |
|
| 5289 | max_order = EXT4_SB(sb)->s_max_folio_order;
|
| 5290 | if (!max_order)
|
| 5291 | return;
|
| 5292 |
|
| 5293 | min_order = EXT4_SB(sb)->s_min_folio_order;
|
| 5294 | if (!min_order && !S_ISREG(inode->i_mode))
|
| 5295 | return;
|
| 5296 |
|
| 5297 | if (ext4_test_inode_flag(inode, EXT4_INODE_JOURNAL_DATA))
|
| 5298 | max_order = min_order;
|
| 5299 |
|
| 5300 | mapping_set_folio_order_range(inode->i_mapping, min_order, max_order);
|
| 5301 | }
|
| 5302 |
|
| 5303 | static int ext4_iget_match(struct inode *inode, u64 ino, void *data)
|
| 5304 | {
|
| 5305 | if (inode->i_ino != ino)
|
| 5306 | return 0;
|
| 5307 | spin_lock(&inode->i_lock);
|
| 5308 | if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE | I_CREATING)) {
|
| 5309 | spin_unlock(&inode->i_lock);
|
| 5310 | return -1;
|
| 5311 | }
|
| 5312 | __iget(inode);
|
| 5313 | spin_unlock(&inode->i_lock);
|
| 5314 | return 1;
|
| 5315 | }
|
| 5316 |
|
| 5317 | struct inode *__ext4_iget(struct super_block *sb, unsigned long ino,
|
| 5318 | ext4_iget_flags flags, const char *function,
|
| 5319 | unsigned int line)
|
| 5320 | {
|
| 5321 | struct ext4_iloc iloc;
|
| 5322 | struct ext4_inode *raw_inode;
|
| 5323 | struct ext4_inode_info *ei;
|
| 5324 | struct ext4_super_block *es = EXT4_SB(sb)->s_es;
|
| 5325 | struct inode *inode;
|
| 5326 | journal_t *journal = EXT4_SB(sb)->s_journal;
|
| 5327 | long ret;
|
| 5328 | loff_t size;
|
| 5329 | int block;
|
| 5330 | uid_t i_uid;
|
| 5331 | gid_t i_gid;
|
| 5332 | projid_t i_projid;
|
| 5333 |
|
| 5334 | if ((!(flags & EXT4_IGET_SPECIAL) && is_special_ino(sb, ino)) ||
|
| 5335 | (ino < EXT4_ROOT_INO) ||
|
| 5336 | (ino > le32_to_cpu(es->s_inodes_count))) {
|
| 5337 | if (flags & EXT4_IGET_HANDLE)
|
| 5338 | return ERR_PTR(-ESTALE);
|
| 5339 | __ext4_error(sb, function, line, false, EFSCORRUPTED, 0,
|
| 5340 | "inode #%lu: comm %s: iget: illegal inode #",
|
| 5341 | ino, current->comm);
|
| 5342 | return ERR_PTR(-EFSCORRUPTED);
|
| 5343 | }
|
| 5344 |
|
| 5345 | if (flags & EXT4_IGET_NOWAIT) {
|
| 5346 | inode = find_inode_nowait(sb, ino, ext4_iget_match, NULL);
|
| 5347 | if (!inode)
|
| 5348 | return ERR_PTR(-ENOENT);
|
| 5349 |
|
| 5350 | if (inode_state_read_once(inode) & I_NEW)
|
| 5351 | wait_on_new_inode(inode);
|
| 5352 |
|
| 5353 | if (unlikely(inode_unhashed(inode))) {
|
| 5354 | iput(inode);
|
| 5355 | return ERR_PTR(-ENOENT);
|
| 5356 | }
|
| 5357 | } else {
|
| 5358 | inode = iget_locked(sb, ino);
|
| 5359 | if (!inode)
|
| 5360 | return ERR_PTR(-ENOMEM);
|
| 5361 | }
|
| 5362 |
|
| 5363 | if (!(inode_state_read_once(inode) & I_NEW)) {
|
| 5364 | ret = check_igot_inode(inode, flags, function, line);
|
| 5365 | if (ret) {
|
| 5366 | iput(inode);
|
| 5367 | return ERR_PTR(ret);
|
| 5368 | }
|
| 5369 | return inode;
|
| 5370 | }
|
| 5371 |
|
| 5372 | ei = EXT4_I(inode);
|
| 5373 | iloc.bh = NULL;
|
| 5374 |
|
| 5375 | ret = __ext4_get_inode_loc_noinmem(inode, &iloc);
|
| 5376 | if (ret < 0)
|
| 5377 | goto bad_inode;
|
| 5378 | raw_inode = ext4_raw_inode(&iloc);
|
| 5379 |
|
| 5380 | if ((flags & EXT4_IGET_HANDLE) &&
|
| 5381 | (raw_inode->i_links_count == 0) && (raw_inode->i_mode == 0)) {
|
| 5382 | ret = -ESTALE;
|
| 5383 | goto bad_inode;
|
| 5384 | }
|
| 5385 |
|
| 5386 | if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
|
| 5387 | ei->i_extra_isize = le16_to_cpu(raw_inode->i_extra_isize);
|
| 5388 | if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize >
|
| 5389 | EXT4_INODE_SIZE(inode->i_sb) ||
|
| 5390 | (ei->i_extra_isize & 3)) {
|
| 5391 | ext4_error_inode(inode, function, line, 0,
|
| 5392 | "iget: bad extra_isize %u "
|
| 5393 | "(inode size %u)",
|
| 5394 | ei->i_extra_isize,
|
| 5395 | EXT4_INODE_SIZE(inode->i_sb));
|
| 5396 | ret = -EFSCORRUPTED;
|
| 5397 | goto bad_inode;
|
| 5398 | }
|
| 5399 | } else
|
| 5400 | ei->i_extra_isize = 0;
|
| 5401 |
|
| 5402 |
|
| 5403 | if (ext4_has_feature_metadata_csum(sb)) {
|
| 5404 | struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
|
| 5405 | __u32 csum;
|
| 5406 | __le32 inum = cpu_to_le32(inode->i_ino);
|
| 5407 | __le32 gen = raw_inode->i_generation;
|
| 5408 | csum = ext4_chksum(sbi->s_csum_seed, (__u8 *)&inum,
|
| 5409 | sizeof(inum));
|
| 5410 | ei->i_csum_seed = ext4_chksum(csum, (__u8 *)&gen, sizeof(gen));
|
| 5411 | }
|
| 5412 |
|
| 5413 | if ((!ext4_inode_csum_verify(inode, raw_inode, ei) ||
|
| 5414 | ext4_simulate_fail(sb, EXT4_SIM_INODE_CRC)) &&
|
| 5415 | (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))) {
|
| 5416 | ext4_error_inode_err(inode, function, line, 0,
|
| 5417 | EFSBADCRC, "iget: checksum invalid");
|
| 5418 | ret = -EFSBADCRC;
|
| 5419 | goto bad_inode;
|
| 5420 | }
|
| 5421 |
|
| 5422 | inode->i_mode = le16_to_cpu(raw_inode->i_mode);
|
| 5423 | i_uid = (uid_t)le16_to_cpu(raw_inode->i_uid_low);
|
| 5424 | i_gid = (gid_t)le16_to_cpu(raw_inode->i_gid_low);
|
| 5425 | if (ext4_has_feature_project(sb) &&
|
| 5426 | EXT4_INODE_SIZE(sb) > EXT4_GOOD_OLD_INODE_SIZE &&
|
| 5427 | EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
|
| 5428 | i_projid = (projid_t)le32_to_cpu(raw_inode->i_projid);
|
| 5429 | else
|
| 5430 | i_projid = EXT4_DEF_PROJID;
|
| 5431 |
|
| 5432 | if (!(test_opt(inode->i_sb, NO_UID32))) {
|
| 5433 | i_uid |= le16_to_cpu(raw_inode->i_uid_high) << 16;
|
| 5434 | i_gid |= le16_to_cpu(raw_inode->i_gid_high) << 16;
|
| 5435 | }
|
| 5436 | i_uid_write(inode, i_uid);
|
| 5437 | i_gid_write(inode, i_gid);
|
| 5438 | ei->i_projid = make_kprojid(&init_user_ns, i_projid);
|
| 5439 | set_nlink(inode, le16_to_cpu(raw_inode->i_links_count));
|
| 5440 |
|
| 5441 | ei->i_inline_off = 0;
|
| 5442 | ei->i_dir_start_lookup = 0;
|
| 5443 | ei->i_dtime = le32_to_cpu(raw_inode->i_dtime);
|
| 5444 | |
| 5445 | * This is needed because nfsd might try to access dead inodes
|
| 5446 | * the test is that same one that e2fsck uses
|
| 5447 | * NeilBrown 1999oct15
|
| 5448 |
|
| 5449 | if (inode->i_nlink == 0) {
|
| 5450 | if ((inode->i_mode == 0 || flags & EXT4_IGET_SPECIAL ||
|
| 5451 | !(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS)) &&
|
| 5452 | ino != EXT4_BOOT_LOADER_INO) {
|
| 5453 |
|
| 5454 | if (flags & EXT4_IGET_SPECIAL) {
|
| 5455 | ext4_error_inode(inode, function, line, 0,
|
| 5456 | "iget: special inode unallocated");
|
| 5457 | ret = -EFSCORRUPTED;
|
| 5458 | } else
|
| 5459 | ret = -ESTALE;
|
| 5460 | goto bad_inode;
|
| 5461 | }
|
| 5462 | |
| 5463 | * valid i_mode and are being read by the orphan
|
| 5464 | * recovery code: that's fine, we're about to complete
|
| 5465 | * the process of deleting those.
|
| 5466 | * OR it is the EXT4_BOOT_LOADER_INO which is
|
| 5467 | * not initialized on a new filesystem.
|
| 5468 | }
|
| 5469 | ei->i_flags = le32_to_cpu(raw_inode->i_flags);
|
| 5470 | ext4_set_inode_flags(inode, true);
|
| 5471 |
|
| 5472 | if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA) &&
|
| 5473 | ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
|
| 5474 | ext4_error_inode(inode, function, line, 0,
|
| 5475 | "inode has both inline data and extents flags");
|
| 5476 | ret = -EFSCORRUPTED;
|
| 5477 | goto bad_inode;
|
| 5478 | }
|
| 5479 | inode->i_blocks = ext4_inode_blocks(raw_inode, ei);
|
| 5480 | ei->i_file_acl = le32_to_cpu(raw_inode->i_file_acl_lo);
|
| 5481 | if (ext4_has_feature_64bit(sb))
|
| 5482 | ei->i_file_acl |=
|
| 5483 | ((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32;
|
| 5484 | inode->i_size = ext4_isize(sb, raw_inode);
|
| 5485 | size = i_size_read(inode);
|
| 5486 | if (size < 0 || size > ext4_get_maxbytes(inode)) {
|
| 5487 | ext4_error_inode(inode, function, line, 0,
|
| 5488 | "iget: bad i_size value: %lld", size);
|
| 5489 | ret = -EFSCORRUPTED;
|
| 5490 | goto bad_inode;
|
| 5491 | }
|
| 5492 | |
| 5493 | * If dir_index is not enabled but there's dir with INDEX flag set,
|
| 5494 | * we'd normally treat htree data as empty space. But with metadata
|
| 5495 | * checksumming that corrupts checksums so forbid that.
|
| 5496 |
|
| 5497 | if (!ext4_has_feature_dir_index(sb) &&
|
| 5498 | ext4_has_feature_metadata_csum(sb) &&
|
| 5499 | ext4_test_inode_flag(inode, EXT4_INODE_INDEX)) {
|
| 5500 | ext4_error_inode(inode, function, line, 0,
|
| 5501 | "iget: Dir with htree data on filesystem without dir_index feature.");
|
| 5502 | ret = -EFSCORRUPTED;
|
| 5503 | goto bad_inode;
|
| 5504 | }
|
| 5505 | ei->i_disksize = inode->i_size;
|
| 5506 | #ifdef CONFIG_QUOTA
|
| 5507 | ei->i_reserved_quota = 0;
|
| 5508 | #endif
|
| 5509 | inode->i_generation = le32_to_cpu(raw_inode->i_generation);
|
| 5510 | ei->i_block_group = iloc.block_group;
|
| 5511 | ei->i_last_alloc_group = ~0;
|
| 5512 | |
| 5513 | * NOTE! The in-memory inode i_data array is in little-endian order
|
| 5514 | * even on big-endian machines: we do NOT byteswap the block numbers!
|
| 5515 |
|
| 5516 | for (block = 0; block < EXT4_N_BLOCKS; block++)
|
| 5517 | ei->i_data[block] = raw_inode->i_block[block];
|
| 5518 | INIT_LIST_HEAD(&ei->i_orphan);
|
| 5519 | ext4_fc_init_inode(&ei->vfs_inode);
|
| 5520 |
|
| 5521 | |
| 5522 | * Set transaction id's of transactions that have to be committed
|
| 5523 | * to finish f[data]sync. We set them to currently running transaction
|
| 5524 | * as we cannot be sure that the inode or some of its metadata isn't
|
| 5525 | * part of the transaction - the inode could have been reclaimed and
|
| 5526 | * now it is reread from disk.
|
| 5527 |
|
| 5528 | if (journal) {
|
| 5529 | transaction_t *transaction;
|
| 5530 | tid_t tid;
|
| 5531 |
|
| 5532 | read_lock(&journal->j_state_lock);
|
| 5533 | if (journal->j_running_transaction)
|
| 5534 | transaction = journal->j_running_transaction;
|
| 5535 | else
|
| 5536 | transaction = journal->j_committing_transaction;
|
| 5537 | if (transaction)
|
| 5538 | tid = transaction->t_tid;
|
| 5539 | else
|
| 5540 | tid = journal->j_commit_sequence;
|
| 5541 | read_unlock(&journal->j_state_lock);
|
| 5542 | ei->i_sync_tid = tid;
|
| 5543 | ei->i_datasync_tid = tid;
|
| 5544 | }
|
| 5545 |
|
| 5546 | if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
|
| 5547 | if (ei->i_extra_isize == 0) {
|
| 5548 |
|
| 5549 | BUILD_BUG_ON(sizeof(struct ext4_inode) & 3);
|
| 5550 | ei->i_extra_isize = sizeof(struct ext4_inode) -
|
| 5551 | EXT4_GOOD_OLD_INODE_SIZE;
|
| 5552 | } else {
|
| 5553 | ret = ext4_iget_extra_inode(inode, raw_inode, ei);
|
| 5554 | if (ret)
|
| 5555 | goto bad_inode;
|
| 5556 | }
|
| 5557 | }
|
| 5558 |
|
| 5559 | EXT4_INODE_GET_CTIME(inode, raw_inode);
|
| 5560 | EXT4_INODE_GET_ATIME(inode, raw_inode);
|
| 5561 | EXT4_INODE_GET_MTIME(inode, raw_inode);
|
| 5562 | EXT4_EINODE_GET_XTIME(i_crtime, ei, raw_inode);
|
| 5563 |
|
| 5564 | if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
|
| 5565 | u64 ivers = le32_to_cpu(raw_inode->i_disk_version);
|
| 5566 |
|
| 5567 | if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
|
| 5568 | if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
|
| 5569 | ivers |=
|
| 5570 | (__u64)(le32_to_cpu(raw_inode->i_version_hi)) << 32;
|
| 5571 | }
|
| 5572 | ext4_inode_set_iversion_queried(inode, ivers);
|
| 5573 | }
|
| 5574 |
|
| 5575 | ret = 0;
|
| 5576 | if (ei->i_file_acl &&
|
| 5577 | !ext4_inode_block_valid(inode, ei->i_file_acl, 1)) {
|
| 5578 | ext4_error_inode(inode, function, line, 0,
|
| 5579 | "iget: bad extended attribute block %llu",
|
| 5580 | ei->i_file_acl);
|
| 5581 | ret = -EFSCORRUPTED;
|
| 5582 | goto bad_inode;
|
| 5583 | } else if (!ext4_has_inline_data(inode)) {
|
| 5584 |
|
| 5585 | if (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) &&
|
| 5586 | (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
|
| 5587 | (S_ISLNK(inode->i_mode) &&
|
| 5588 | !ext4_inode_is_fast_symlink(inode)))) {
|
| 5589 | if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
|
| 5590 | ret = ext4_ext_check_inode(inode);
|
| 5591 | else
|
| 5592 | ret = ext4_ind_check_inode(inode);
|
| 5593 | }
|
| 5594 | }
|
| 5595 | if (ret)
|
| 5596 | goto bad_inode;
|
| 5597 |
|
| 5598 | if (S_ISREG(inode->i_mode)) {
|
| 5599 | inode->i_op = &ext4_file_inode_operations;
|
| 5600 | inode->i_fop = &ext4_file_operations;
|
| 5601 | ext4_set_aops(inode);
|
| 5602 | } else if (S_ISDIR(inode->i_mode)) {
|
| 5603 | inode->i_op = &ext4_dir_inode_operations;
|
| 5604 | inode->i_fop = &ext4_dir_operations;
|
| 5605 | } else if (S_ISLNK(inode->i_mode)) {
|
| 5606 |
|
| 5607 | if (IS_APPEND(inode) || IS_IMMUTABLE(inode)) {
|
| 5608 | ext4_error_inode(inode, function, line, 0,
|
| 5609 | "iget: immutable or append flags "
|
| 5610 | "not allowed on symlinks");
|
| 5611 | ret = -EFSCORRUPTED;
|
| 5612 | goto bad_inode;
|
| 5613 | }
|
| 5614 | if (IS_ENCRYPTED(inode)) {
|
| 5615 | inode->i_op = &ext4_encrypted_symlink_inode_operations;
|
| 5616 | } else if (ext4_inode_is_fast_symlink(inode)) {
|
| 5617 | inode->i_op = &ext4_fast_symlink_inode_operations;
|
| 5618 |
|
| 5619 | |
| 5620 | * Orphan cleanup can see inodes with i_size == 0
|
| 5621 | * and i_data uninitialized. Skip size checks in
|
| 5622 | * that case. This is safe because the first thing
|
| 5623 | * ext4_evict_inode() does for fast symlinks is
|
| 5624 | * clearing of i_data and i_size.
|
| 5625 |
|
| 5626 | if ((EXT4_SB(sb)->s_mount_state & EXT4_ORPHAN_FS)) {
|
| 5627 | if (inode->i_nlink != 0) {
|
| 5628 | ext4_error_inode(inode, function, line, 0,
|
| 5629 | "invalid orphan symlink nlink %d",
|
| 5630 | inode->i_nlink);
|
| 5631 | ret = -EFSCORRUPTED;
|
| 5632 | goto bad_inode;
|
| 5633 | }
|
| 5634 | } else {
|
| 5635 | if (inode->i_size == 0 ||
|
| 5636 | inode->i_size >= sizeof(ei->i_data) ||
|
| 5637 | strnlen((char *)ei->i_data, inode->i_size + 1) !=
|
| 5638 | inode->i_size) {
|
| 5639 | ext4_error_inode(inode, function, line, 0,
|
| 5640 | "invalid fast symlink length %llu",
|
| 5641 | (unsigned long long)inode->i_size);
|
| 5642 | ret = -EFSCORRUPTED;
|
| 5643 | goto bad_inode;
|
| 5644 | }
|
| 5645 | inode_set_cached_link(inode, (char *)ei->i_data,
|
| 5646 | inode->i_size);
|
| 5647 | }
|
| 5648 | } else {
|
| 5649 | inode->i_op = &ext4_symlink_inode_operations;
|
| 5650 | }
|
| 5651 | } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) ||
|
| 5652 | S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) {
|
| 5653 | inode->i_op = &ext4_special_inode_operations;
|
| 5654 | if (raw_inode->i_block[0])
|
| 5655 | init_special_inode(inode, inode->i_mode,
|
| 5656 | old_decode_dev(le32_to_cpu(raw_inode->i_block[0])));
|
| 5657 | else
|
| 5658 | init_special_inode(inode, inode->i_mode,
|
| 5659 | new_decode_dev(le32_to_cpu(raw_inode->i_block[1])));
|
| 5660 | } else if (ino == EXT4_BOOT_LOADER_INO) {
|
| 5661 | make_bad_inode(inode);
|
| 5662 | } else {
|
| 5663 | ret = -EFSCORRUPTED;
|
| 5664 | ext4_error_inode(inode, function, line, 0,
|
| 5665 | "iget: bogus i_mode (%o)", inode->i_mode);
|
| 5666 | goto bad_inode;
|
| 5667 | }
|
| 5668 | if (IS_CASEFOLDED(inode) && !ext4_has_feature_casefold(inode->i_sb)) {
|
| 5669 | ext4_error_inode(inode, function, line, 0,
|
| 5670 | "casefold flag without casefold feature");
|
| 5671 | ret = -EFSCORRUPTED;
|
| 5672 | goto bad_inode;
|
| 5673 | }
|
| 5674 |
|
| 5675 | ext4_set_inode_mapping_order(inode);
|
| 5676 |
|
| 5677 | ret = check_igot_inode(inode, flags, function, line);
|
| 5678 | |
| 5679 | * -ESTALE here means there is nothing inherently wrong with the inode,
|
| 5680 | * it's just not an inode we can return for an fhandle lookup.
|
| 5681 |
|
| 5682 | if (ret == -ESTALE) {
|
| 5683 | brelse(iloc.bh);
|
| 5684 | unlock_new_inode(inode);
|
| 5685 | iput(inode);
|
| 5686 | return ERR_PTR(-ESTALE);
|
| 5687 | }
|
| 5688 | if (ret)
|
| 5689 | goto bad_inode;
|
| 5690 | brelse(iloc.bh);
|
| 5691 |
|
| 5692 | if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) && !ei->i_file_acl)
|
| 5693 | cache_no_acl(inode);
|
| 5694 | unlock_new_inode(inode);
|
| 5695 | return inode;
|
| 5696 |
|
| 5697 | bad_inode:
|
| 5698 | brelse(iloc.bh);
|
| 5699 | iget_failed(inode);
|
| 5700 | return ERR_PTR(ret);
|
| 5701 | }
|
| 5702 |
|
| 5703 | static void __ext4_update_other_inode_time(struct super_block *sb,
|
| 5704 | unsigned long orig_ino,
|
| 5705 | unsigned long ino,
|
| 5706 | struct ext4_inode *raw_inode)
|
| 5707 | {
|
| 5708 | struct inode *inode;
|
| 5709 |
|
| 5710 | inode = find_inode_by_ino_rcu(sb, ino);
|
| 5711 | if (!inode)
|
| 5712 | return;
|
| 5713 |
|
| 5714 | if (!inode_is_dirtytime_only(inode))
|
| 5715 | return;
|
| 5716 |
|
| 5717 | spin_lock(&inode->i_lock);
|
| 5718 | if (inode_is_dirtytime_only(inode)) {
|
| 5719 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 5720 |
|
| 5721 | inode_state_clear(inode, I_DIRTY_TIME);
|
| 5722 | spin_unlock(&inode->i_lock);
|
| 5723 |
|
| 5724 | spin_lock(&ei->i_raw_lock);
|
| 5725 | EXT4_INODE_SET_CTIME(inode, raw_inode);
|
| 5726 | EXT4_INODE_SET_MTIME(inode, raw_inode);
|
| 5727 | EXT4_INODE_SET_ATIME(inode, raw_inode);
|
| 5728 | ext4_inode_csum_set(inode, raw_inode, ei);
|
| 5729 | spin_unlock(&ei->i_raw_lock);
|
| 5730 | trace_ext4_other_inode_update_time(inode, orig_ino);
|
| 5731 | return;
|
| 5732 | }
|
| 5733 | spin_unlock(&inode->i_lock);
|
| 5734 | }
|
| 5735 |
|
| 5736 | |
| 5737 | * Opportunistically update the other time fields for other inodes in
|
| 5738 | * the same inode table block.
|
| 5739 |
|
| 5740 | static void ext4_update_other_inodes_time(struct super_block *sb,
|
| 5741 | unsigned long orig_ino, char *buf)
|
| 5742 | {
|
| 5743 | unsigned long ino;
|
| 5744 | int i, inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
|
| 5745 | int inode_size = EXT4_INODE_SIZE(sb);
|
| 5746 |
|
| 5747 | |
| 5748 | * Calculate the first inode in the inode table block. Inode
|
| 5749 | * numbers are one-based. That is, the first inode in a block
|
| 5750 | * (assuming 4k blocks and 256 byte inodes) is (n*16 + 1).
|
| 5751 |
|
| 5752 | ino = ((orig_ino - 1) & ~(inodes_per_block - 1)) + 1;
|
| 5753 | rcu_read_lock();
|
| 5754 | for (i = 0; i < inodes_per_block; i++, ino++, buf += inode_size) {
|
| 5755 | if (ino == orig_ino)
|
| 5756 | continue;
|
| 5757 | __ext4_update_other_inode_time(sb, orig_ino, ino,
|
| 5758 | (struct ext4_inode *)buf);
|
| 5759 | }
|
| 5760 | rcu_read_unlock();
|
| 5761 | }
|
| 5762 |
|
| 5763 | |
| 5764 | * Post the struct inode info into an on-disk inode location in the
|
| 5765 | * buffer-cache. This gobbles the caller's reference to the
|
| 5766 | * buffer_head in the inode location struct.
|
| 5767 | *
|
| 5768 | * The caller must have write access to iloc->bh.
|
| 5769 |
|
| 5770 | static int ext4_do_update_inode(handle_t *handle,
|
| 5771 | struct inode *inode,
|
| 5772 | struct ext4_iloc *iloc)
|
| 5773 | {
|
| 5774 | struct ext4_inode *raw_inode = ext4_raw_inode(iloc);
|
| 5775 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 5776 | struct buffer_head *bh = iloc->bh;
|
| 5777 | struct super_block *sb = inode->i_sb;
|
| 5778 | int err;
|
| 5779 | int need_datasync = 0, set_large_file = 0;
|
| 5780 |
|
| 5781 | spin_lock(&ei->i_raw_lock);
|
| 5782 |
|
| 5783 | |
| 5784 | * For fields not tracked in the in-memory inode, initialise them
|
| 5785 | * to zero for new inodes.
|
| 5786 |
|
| 5787 | if (ext4_test_inode_state(inode, EXT4_STATE_NEW))
|
| 5788 | memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size);
|
| 5789 |
|
| 5790 | if (READ_ONCE(ei->i_disksize) != ext4_isize(inode->i_sb, raw_inode))
|
| 5791 | need_datasync = 1;
|
| 5792 | if (ei->i_disksize > 0x7fffffffULL) {
|
| 5793 | if (!ext4_has_feature_large_file(sb) ||
|
| 5794 | EXT4_SB(sb)->s_es->s_rev_level == cpu_to_le32(EXT4_GOOD_OLD_REV))
|
| 5795 | set_large_file = 1;
|
| 5796 | }
|
| 5797 |
|
| 5798 | err = ext4_fill_raw_inode(inode, raw_inode);
|
| 5799 | spin_unlock(&ei->i_raw_lock);
|
| 5800 | if (err) {
|
| 5801 | EXT4_ERROR_INODE(inode, "corrupted inode contents");
|
| 5802 | goto out_brelse;
|
| 5803 | }
|
| 5804 |
|
| 5805 | if (inode->i_sb->s_flags & SB_LAZYTIME)
|
| 5806 | ext4_update_other_inodes_time(inode->i_sb, inode->i_ino,
|
| 5807 | bh->b_data);
|
| 5808 |
|
| 5809 | BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
|
| 5810 | err = ext4_handle_dirty_metadata(handle, NULL, bh);
|
| 5811 | if (err)
|
| 5812 | goto out_error;
|
| 5813 | ext4_clear_inode_state(inode, EXT4_STATE_NEW);
|
| 5814 | if (set_large_file) {
|
| 5815 | BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get write access");
|
| 5816 | err = ext4_journal_get_write_access(handle, sb,
|
| 5817 | EXT4_SB(sb)->s_sbh,
|
| 5818 | EXT4_JTR_NONE);
|
| 5819 | if (err)
|
| 5820 | goto out_error;
|
| 5821 | lock_buffer(EXT4_SB(sb)->s_sbh);
|
| 5822 | ext4_set_feature_large_file(sb);
|
| 5823 | ext4_superblock_csum_set(sb);
|
| 5824 | unlock_buffer(EXT4_SB(sb)->s_sbh);
|
| 5825 | ext4_handle_sync(handle);
|
| 5826 | err = ext4_handle_dirty_metadata(handle, NULL,
|
| 5827 | EXT4_SB(sb)->s_sbh);
|
| 5828 | }
|
| 5829 | ext4_update_inode_fsync_trans(handle, inode, need_datasync);
|
| 5830 | out_error:
|
| 5831 | ext4_std_error(inode->i_sb, err);
|
| 5832 | out_brelse:
|
| 5833 | brelse(bh);
|
| 5834 | return err;
|
| 5835 | }
|
| 5836 |
|
| 5837 | |
| 5838 | * ext4_write_inode()
|
| 5839 | *
|
| 5840 | * We are called from a few places:
|
| 5841 | *
|
| 5842 | * - Within generic_file_aio_write() -> generic_write_sync() for O_SYNC files.
|
| 5843 | * Here, there will be no transaction running. We wait for any running
|
| 5844 | * transaction to commit.
|
| 5845 | *
|
| 5846 | * - Within flush work (sys_sync(), kupdate and such).
|
| 5847 | * We wait on commit, if told to.
|
| 5848 | *
|
| 5849 | * - Within iput_final() -> write_inode_now()
|
| 5850 | * We wait on commit, if told to.
|
| 5851 | *
|
| 5852 | * In all cases it is actually safe for us to return without doing anything,
|
| 5853 | * because the inode has been copied into a raw inode buffer in
|
| 5854 | * ext4_mark_inode_dirty(). This is a correctness thing for WB_SYNC_ALL
|
| 5855 | * writeback.
|
| 5856 | *
|
| 5857 | * For nojournal mode all the work is done in ext4_sync_inode_metadata()
|
| 5858 | * because inode content is already copied into raw inode buffer and inode
|
| 5859 | * is marked with I_METADATA_WRITEBACK.
|
| 5860 | *
|
| 5861 | * Note that we are absolutely dependent upon all inode dirtiers doing the
|
| 5862 | * right thing: they *must* call mark_inode_dirty() after dirtying info in
|
| 5863 | * which we are interested.
|
| 5864 | *
|
| 5865 | * It would be a bug for them to not do this. The code:
|
| 5866 | *
|
| 5867 | * mark_inode_dirty(inode)
|
| 5868 | * stuff();
|
| 5869 | * inode->i_size = expr;
|
| 5870 | *
|
| 5871 | * is in error because write_inode() could occur while `stuff()' is running,
|
| 5872 | * and the new i_size will be lost. Plus the inode will no longer be on the
|
| 5873 | * superblock's dirty inode list.
|
| 5874 |
|
| 5875 | int ext4_write_inode(struct inode *inode, struct writeback_control *wbc)
|
| 5876 | {
|
| 5877 | int err;
|
| 5878 |
|
| 5879 | if (WARN_ON_ONCE(current->flags & PF_MEMALLOC))
|
| 5880 | return 0;
|
| 5881 |
|
| 5882 | err = ext4_emergency_state(inode->i_sb);
|
| 5883 | if (unlikely(err))
|
| 5884 | return err;
|
| 5885 |
|
| 5886 | if (!EXT4_SB(inode->i_sb)->s_journal)
|
| 5887 | return 0;
|
| 5888 |
|
| 5889 | if (ext4_journal_current_handle()) {
|
| 5890 | ext4_debug("called recursively, non-PF_MEMALLOC!\n");
|
| 5891 | dump_stack();
|
| 5892 | return -EIO;
|
| 5893 | }
|
| 5894 |
|
| 5895 | |
| 5896 | * No need to force transaction in WB_SYNC_NONE mode. Also
|
| 5897 | * ext4_sync_fs() will force the commit after everything is
|
| 5898 | * written.
|
| 5899 |
|
| 5900 | if (wbc->sync_mode != WB_SYNC_ALL || wbc->for_sync)
|
| 5901 | return 0;
|
| 5902 |
|
| 5903 | return ext4_fc_commit(EXT4_SB(inode->i_sb)->s_journal,
|
| 5904 | EXT4_I(inode)->i_sync_tid);
|
| 5905 | }
|
| 5906 |
|
| 5907 | int ext4_sync_inode_metadata(struct inode *inode, struct writeback_control *wbc)
|
| 5908 | {
|
| 5909 | struct ext4_iloc iloc;
|
| 5910 | struct mapping_metadata_bhs *mmb;
|
| 5911 | int err;
|
| 5912 |
|
| 5913 |
|
| 5914 | if (WARN_ON_ONCE(EXT4_SB(inode->i_sb)->s_journal))
|
| 5915 | return -EFSCORRUPTED;
|
| 5916 |
|
| 5917 | err = __ext4_get_inode_loc_noinmem(inode, &iloc);
|
| 5918 | if (err)
|
| 5919 | return err;
|
| 5920 | mmb = READ_ONCE(EXT4_I(inode)->i_metadata_bhs);
|
| 5921 | if (mmb) {
|
| 5922 | err = mmb_sync(mmb);
|
| 5923 | if (err)
|
| 5924 | goto out;
|
| 5925 | }
|
| 5926 | sync_dirty_buffer(iloc.bh);
|
| 5927 | if (buffer_write_io_error(iloc.bh)) {
|
| 5928 | ext4_error_inode_block(inode, iloc.bh->b_blocknr, EIO,
|
| 5929 | "IO error syncing inode");
|
| 5930 | err = -EIO;
|
| 5931 | }
|
| 5932 | out:
|
| 5933 | brelse(iloc.bh);
|
| 5934 | return err;
|
| 5935 | }
|
| 5936 |
|
| 5937 | |
| 5938 | * In data=journal mode ext4_journalled_invalidate_folio() may fail to invalidate
|
| 5939 | * buffers that are attached to a folio straddling i_size and are undergoing
|
| 5940 | * commit. In that case we have to wait for commit to finish and try again.
|
| 5941 |
|
| 5942 | static void ext4_wait_for_tail_page_commit(struct inode *inode)
|
| 5943 | {
|
| 5944 | unsigned offset;
|
| 5945 | journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
|
| 5946 | tid_t commit_tid;
|
| 5947 | int ret;
|
| 5948 | bool has_transaction;
|
| 5949 |
|
| 5950 | offset = inode->i_size & (PAGE_SIZE - 1);
|
| 5951 | |
| 5952 | * If the folio is fully truncated, we don't need to wait for any commit
|
| 5953 | * (and we even should not as __ext4_journalled_invalidate_folio() may
|
| 5954 | * strip all buffers from the folio but keep the folio dirty which can then
|
| 5955 | * confuse e.g. concurrent ext4_writepages() seeing dirty folio without
|
| 5956 | * buffers). Also we don't need to wait for any commit if all buffers in
|
| 5957 | * the folio remain valid. This is most beneficial for the common case of
|
| 5958 | * blocksize == PAGESIZE.
|
| 5959 |
|
| 5960 | if (!offset || offset > (PAGE_SIZE - i_blocksize(inode)))
|
| 5961 | return;
|
| 5962 | while (1) {
|
| 5963 | struct folio *folio = filemap_lock_folio(inode->i_mapping,
|
| 5964 | inode->i_size >> PAGE_SHIFT);
|
| 5965 | if (IS_ERR(folio))
|
| 5966 | return;
|
| 5967 | ret = __ext4_journalled_invalidate_folio(folio, offset,
|
| 5968 | folio_size(folio) - offset);
|
| 5969 | folio_unlock(folio);
|
| 5970 | folio_put(folio);
|
| 5971 | if (ret != -EBUSY)
|
| 5972 | return;
|
| 5973 | has_transaction = false;
|
| 5974 | read_lock(&journal->j_state_lock);
|
| 5975 | if (journal->j_committing_transaction) {
|
| 5976 | commit_tid = journal->j_committing_transaction->t_tid;
|
| 5977 | has_transaction = true;
|
| 5978 | }
|
| 5979 | read_unlock(&journal->j_state_lock);
|
| 5980 | if (has_transaction)
|
| 5981 | jbd2_log_wait_commit(journal, commit_tid);
|
| 5982 | }
|
| 5983 | }
|
| 5984 |
|
| 5985 | |
| 5986 | * ext4_setattr()
|
| 5987 | *
|
| 5988 | * Called from notify_change.
|
| 5989 | *
|
| 5990 | * We want to trap VFS attempts to truncate the file as soon as
|
| 5991 | * possible. In particular, we want to make sure that when the VFS
|
| 5992 | * shrinks i_size, we put the inode on the orphan list and modify
|
| 5993 | * i_disksize immediately, so that during the subsequent flushing of
|
| 5994 | * dirty pages and freeing of disk blocks, we can guarantee that any
|
| 5995 | * commit will leave the blocks being flushed in an unused state on
|
| 5996 | * disk. (On recovery, the inode will get truncated and the blocks will
|
| 5997 | * be freed, so we have a strong guarantee that no future commit will
|
| 5998 | * leave these blocks visible to the user.)
|
| 5999 | *
|
| 6000 | * Another thing we have to assure is that if we are in ordered mode
|
| 6001 | * and inode is still attached to the committing transaction, we must
|
| 6002 | * we start writeout of all the dirty pages which are being truncated.
|
| 6003 | * This way we are sure that all the data written in the previous
|
| 6004 | * transaction are already on disk (truncate waits for pages under
|
| 6005 | * writeback).
|
| 6006 | *
|
| 6007 | * Called with inode->i_rwsem down.
|
| 6008 |
|
| 6009 | int ext4_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
|
| 6010 | struct iattr *attr)
|
| 6011 | {
|
| 6012 | struct inode *inode = d_inode(dentry);
|
| 6013 | int error, rc = 0;
|
| 6014 | int orphan = 0;
|
| 6015 | const unsigned int ia_valid = attr->ia_valid;
|
| 6016 | bool inc_ivers = true;
|
| 6017 |
|
| 6018 | error = ext4_emergency_state(inode->i_sb);
|
| 6019 | if (unlikely(error))
|
| 6020 | return error;
|
| 6021 |
|
| 6022 | if (unlikely(IS_IMMUTABLE(inode)))
|
| 6023 | return -EPERM;
|
| 6024 |
|
| 6025 | if (unlikely(IS_APPEND(inode) &&
|
| 6026 | (ia_valid & (ATTR_MODE | ATTR_UID |
|
| 6027 | ATTR_GID | ATTR_TIMES_SET))))
|
| 6028 | return -EPERM;
|
| 6029 |
|
| 6030 | error = setattr_prepare(idmap, dentry, attr);
|
| 6031 | if (error)
|
| 6032 | return error;
|
| 6033 |
|
| 6034 | error = fscrypt_prepare_setattr(dentry, attr);
|
| 6035 | if (error)
|
| 6036 | return error;
|
| 6037 |
|
| 6038 | if (is_quota_modification(idmap, inode, attr)) {
|
| 6039 | error = dquot_initialize(inode);
|
| 6040 | if (error)
|
| 6041 | return error;
|
| 6042 | }
|
| 6043 |
|
| 6044 | if (i_uid_needs_update(idmap, attr, inode) ||
|
| 6045 | i_gid_needs_update(idmap, attr, inode)) {
|
| 6046 | handle_t *handle;
|
| 6047 |
|
| 6048 | |
| 6049 | * inode block, ? - but truncate inode update has it)
|
| 6050 | handle = ext4_journal_start(inode, EXT4_HT_QUOTA,
|
| 6051 | (EXT4_MAXQUOTAS_INIT_BLOCKS(inode->i_sb) +
|
| 6052 | EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb)) + 3);
|
| 6053 | if (IS_ERR(handle)) {
|
| 6054 | error = PTR_ERR(handle);
|
| 6055 | goto err_out;
|
| 6056 | }
|
| 6057 |
|
| 6058 | |
| 6059 | * counts xattr inode references.
|
| 6060 |
|
| 6061 | down_read(&EXT4_I(inode)->xattr_sem);
|
| 6062 | error = dquot_transfer(idmap, inode, attr);
|
| 6063 | up_read(&EXT4_I(inode)->xattr_sem);
|
| 6064 |
|
| 6065 | if (error) {
|
| 6066 | ext4_journal_stop(handle);
|
| 6067 | return error;
|
| 6068 | }
|
| 6069 | |
| 6070 | * one transaction
|
| 6071 | i_uid_update(idmap, attr, inode);
|
| 6072 | i_gid_update(idmap, attr, inode);
|
| 6073 | error = ext4_mark_inode_dirty(handle, inode);
|
| 6074 | ext4_journal_stop(handle);
|
| 6075 | if (unlikely(error)) {
|
| 6076 | return error;
|
| 6077 | }
|
| 6078 | }
|
| 6079 |
|
| 6080 | if (attr->ia_valid & ATTR_SIZE) {
|
| 6081 | handle_t *handle;
|
| 6082 | loff_t oldsize = inode->i_size;
|
| 6083 | loff_t old_disksize;
|
| 6084 | int shrink = (attr->ia_size < inode->i_size);
|
| 6085 |
|
| 6086 | if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
|
| 6087 | struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
|
| 6088 |
|
| 6089 | if (attr->ia_size > sbi->s_bitmap_maxbytes) {
|
| 6090 | return -EFBIG;
|
| 6091 | }
|
| 6092 | }
|
| 6093 | if (!S_ISREG(inode->i_mode)) {
|
| 6094 | return -EINVAL;
|
| 6095 | }
|
| 6096 |
|
| 6097 | if (attr->ia_size == inode->i_size)
|
| 6098 | inc_ivers = false;
|
| 6099 |
|
| 6100 | |
| 6101 | * If file has inline data but new size exceeds inline capacity,
|
| 6102 | * convert to extent-based storage first to prevent inconsistent
|
| 6103 | * state (inline flag set but size exceeds inline capacity).
|
| 6104 |
|
| 6105 | if (ext4_has_inline_data(inode) &&
|
| 6106 | attr->ia_size > EXT4_I(inode)->i_inline_size) {
|
| 6107 | error = ext4_convert_inline_data(inode);
|
| 6108 | if (error)
|
| 6109 | goto err_out;
|
| 6110 | }
|
| 6111 |
|
| 6112 | if (shrink) {
|
| 6113 | if (ext4_should_order_data(inode)) {
|
| 6114 | error = ext4_begin_ordered_truncate(inode,
|
| 6115 | attr->ia_size);
|
| 6116 | if (error)
|
| 6117 | goto err_out;
|
| 6118 | }
|
| 6119 | |
| 6120 | * Blocks are going to be removed from the inode. Wait
|
| 6121 | * for dio in flight.
|
| 6122 |
|
| 6123 | inode_dio_wait(inode);
|
| 6124 | }
|
| 6125 |
|
| 6126 | filemap_invalidate_lock(inode->i_mapping);
|
| 6127 |
|
| 6128 | rc = ext4_break_layouts(inode);
|
| 6129 | if (rc) {
|
| 6130 | filemap_invalidate_unlock(inode->i_mapping);
|
| 6131 | goto err_out;
|
| 6132 | }
|
| 6133 |
|
| 6134 | if (attr->ia_size != inode->i_size) {
|
| 6135 |
|
| 6136 | if (attr->ia_size & (inode->i_sb->s_blocksize - 1) ||
|
| 6137 | oldsize & (inode->i_sb->s_blocksize - 1)) {
|
| 6138 | error = ext4_inode_attach_jinode(inode);
|
| 6139 | if (error)
|
| 6140 | goto out_mmap_sem;
|
| 6141 | }
|
| 6142 |
|
| 6143 | |
| 6144 | * Update c/mtime and tail zero the EOF folio on
|
| 6145 | * truncate up. ext4_truncate() handles the shrink case
|
| 6146 | * below.
|
| 6147 |
|
| 6148 | if (!shrink) {
|
| 6149 | inode_set_mtime_to_ts(inode,
|
| 6150 | inode_set_ctime_current(inode));
|
| 6151 | if (oldsize & (inode->i_sb->s_blocksize - 1)) {
|
| 6152 | error = ext4_block_zero_eof(inode,
|
| 6153 | oldsize, LLONG_MAX);
|
| 6154 | if (error)
|
| 6155 | goto out_mmap_sem;
|
| 6156 | }
|
| 6157 | }
|
| 6158 |
|
| 6159 | handle = ext4_journal_start(inode, EXT4_HT_INODE, 3);
|
| 6160 | if (IS_ERR(handle)) {
|
| 6161 | error = PTR_ERR(handle);
|
| 6162 | goto out_mmap_sem;
|
| 6163 | }
|
| 6164 | if (ext4_handle_valid(handle) && shrink) {
|
| 6165 | error = ext4_orphan_add(handle, inode);
|
| 6166 | orphan = 1;
|
| 6167 | }
|
| 6168 |
|
| 6169 | if (shrink)
|
| 6170 | ext4_fc_track_range(handle, inode,
|
| 6171 | (attr->ia_size > 0 ? attr->ia_size - 1 : 0) >>
|
| 6172 | inode->i_sb->s_blocksize_bits,
|
| 6173 | EXT_MAX_BLOCKS - 1);
|
| 6174 | else
|
| 6175 | ext4_fc_track_range(
|
| 6176 | handle, inode,
|
| 6177 | (oldsize > 0 ? oldsize - 1 : oldsize) >>
|
| 6178 | inode->i_sb->s_blocksize_bits,
|
| 6179 | (attr->ia_size > 0 ? attr->ia_size - 1 : 0) >>
|
| 6180 | inode->i_sb->s_blocksize_bits);
|
| 6181 |
|
| 6182 | down_write(&EXT4_I(inode)->i_data_sem);
|
| 6183 | old_disksize = EXT4_I(inode)->i_disksize;
|
| 6184 | EXT4_I(inode)->i_disksize = attr->ia_size;
|
| 6185 |
|
| 6186 | |
| 6187 | * We have to update i_size under i_data_sem together
|
| 6188 | * with i_disksize to avoid races with writeback code
|
| 6189 | * running ext4_wb_update_i_disksize().
|
| 6190 |
|
| 6191 | if (!error)
|
| 6192 | i_size_write(inode, attr->ia_size);
|
| 6193 | else
|
| 6194 | EXT4_I(inode)->i_disksize = old_disksize;
|
| 6195 | up_write(&EXT4_I(inode)->i_data_sem);
|
| 6196 | rc = ext4_mark_inode_dirty(handle, inode);
|
| 6197 | if (!error)
|
| 6198 | error = rc;
|
| 6199 | ext4_journal_stop(handle);
|
| 6200 | if (error)
|
| 6201 | goto out_mmap_sem;
|
| 6202 | if (!shrink) {
|
| 6203 | pagecache_isize_extended(inode, oldsize,
|
| 6204 | inode->i_size);
|
| 6205 | } else if (ext4_should_journal_data(inode)) {
|
| 6206 | ext4_wait_for_tail_page_commit(inode);
|
| 6207 | }
|
| 6208 | }
|
| 6209 |
|
| 6210 | |
| 6211 | * Truncate pagecache after we've waited for commit
|
| 6212 | * in data=journal mode to make pages freeable.
|
| 6213 |
|
| 6214 | truncate_pagecache(inode, inode->i_size);
|
| 6215 | |
| 6216 | * Call ext4_truncate() even if i_size didn't change to
|
| 6217 | * truncate possible preallocated blocks.
|
| 6218 |
|
| 6219 | if (attr->ia_size <= oldsize) {
|
| 6220 | rc = ext4_truncate(inode);
|
| 6221 | if (rc)
|
| 6222 | error = rc;
|
| 6223 | }
|
| 6224 | out_mmap_sem:
|
| 6225 | filemap_invalidate_unlock(inode->i_mapping);
|
| 6226 | }
|
| 6227 |
|
| 6228 | if (!error) {
|
| 6229 | if (inc_ivers)
|
| 6230 | inode_inc_iversion(inode);
|
| 6231 | setattr_copy(idmap, inode, attr);
|
| 6232 | mark_inode_dirty(inode);
|
| 6233 | }
|
| 6234 |
|
| 6235 | |
| 6236 | * If the call to ext4_truncate failed to get a transaction handle at
|
| 6237 | * all, we need to clean up the in-core orphan list manually.
|
| 6238 |
|
| 6239 | if (orphan && inode->i_nlink)
|
| 6240 | ext4_orphan_del(NULL, inode);
|
| 6241 |
|
| 6242 | if (!error && (ia_valid & ATTR_MODE))
|
| 6243 | rc = posix_acl_chmod(idmap, dentry, inode->i_mode);
|
| 6244 |
|
| 6245 | err_out:
|
| 6246 | if (error)
|
| 6247 | ext4_std_error(inode->i_sb, error);
|
| 6248 | if (!error)
|
| 6249 | error = rc;
|
| 6250 | return error;
|
| 6251 | }
|
| 6252 |
|
| 6253 | u32 ext4_dio_alignment(struct inode *inode)
|
| 6254 | {
|
| 6255 | if (fsverity_active(inode))
|
| 6256 | return 0;
|
| 6257 | if (ext4_should_journal_data(inode))
|
| 6258 | return 0;
|
| 6259 | if (ext4_has_inline_data(inode))
|
| 6260 | return 0;
|
| 6261 | if (IS_ENCRYPTED(inode))
|
| 6262 | return i_blocksize(inode);
|
| 6263 | return 1;
|
| 6264 | }
|
| 6265 |
|
| 6266 | int ext4_getattr(const struct mnt_idmap *idmap, const struct path *path,
|
| 6267 | struct kstat *stat, u32 request_mask, unsigned int query_flags)
|
| 6268 | {
|
| 6269 | struct inode *inode = d_inode(path->dentry);
|
| 6270 | struct ext4_inode *raw_inode;
|
| 6271 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 6272 | unsigned int flags;
|
| 6273 |
|
| 6274 | if ((request_mask & STATX_BTIME) &&
|
| 6275 | EXT4_FITS_IN_INODE(raw_inode, ei, i_crtime)) {
|
| 6276 | stat->result_mask |= STATX_BTIME;
|
| 6277 | stat->btime.tv_sec = ei->i_crtime.tv_sec;
|
| 6278 | stat->btime.tv_nsec = ei->i_crtime.tv_nsec;
|
| 6279 | }
|
| 6280 |
|
| 6281 |
|
| 6282 | if ((request_mask & STATX_DIOALIGN) && S_ISREG(inode->i_mode)) {
|
| 6283 | u32 dio_align = ext4_dio_alignment(inode);
|
| 6284 |
|
| 6285 | stat->result_mask |= STATX_DIOALIGN;
|
| 6286 | if (dio_align == 1) {
|
| 6287 | struct block_device *bdev = inode->i_sb->s_bdev;
|
| 6288 |
|
| 6289 |
|
| 6290 | stat->dio_mem_align = bdev_dma_alignment(bdev) + 1;
|
| 6291 | stat->dio_offset_align = bdev_logical_block_size(bdev);
|
| 6292 | } else {
|
| 6293 | stat->dio_mem_align = dio_align;
|
| 6294 | stat->dio_offset_align = dio_align;
|
| 6295 | }
|
| 6296 | }
|
| 6297 |
|
| 6298 | if ((request_mask & STATX_WRITE_ATOMIC) && S_ISREG(inode->i_mode)) {
|
| 6299 | struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
|
| 6300 | unsigned int awu_min = 0, awu_max = 0;
|
| 6301 |
|
| 6302 | if (ext4_inode_can_atomic_write(inode)) {
|
| 6303 | awu_min = sbi->s_awu_min;
|
| 6304 | awu_max = sbi->s_awu_max;
|
| 6305 | }
|
| 6306 |
|
| 6307 | generic_fill_statx_atomic_writes(stat, awu_min, awu_max, 0);
|
| 6308 | }
|
| 6309 |
|
| 6310 | flags = ei->i_flags & EXT4_FL_USER_VISIBLE;
|
| 6311 | if (flags & EXT4_APPEND_FL)
|
| 6312 | stat->attributes |= STATX_ATTR_APPEND;
|
| 6313 | if (flags & EXT4_COMPR_FL)
|
| 6314 | stat->attributes |= STATX_ATTR_COMPRESSED;
|
| 6315 | if (flags & EXT4_ENCRYPT_FL)
|
| 6316 | stat->attributes |= STATX_ATTR_ENCRYPTED;
|
| 6317 | if (flags & EXT4_IMMUTABLE_FL)
|
| 6318 | stat->attributes |= STATX_ATTR_IMMUTABLE;
|
| 6319 | if (flags & EXT4_NODUMP_FL)
|
| 6320 | stat->attributes |= STATX_ATTR_NODUMP;
|
| 6321 |
|
| 6322 | stat->attributes_mask |= (STATX_ATTR_APPEND |
|
| 6323 | STATX_ATTR_COMPRESSED |
|
| 6324 | STATX_ATTR_ENCRYPTED |
|
| 6325 | STATX_ATTR_IMMUTABLE |
|
| 6326 | STATX_ATTR_NODUMP);
|
| 6327 |
|
| 6328 | generic_fillattr(idmap, request_mask, inode, stat);
|
| 6329 | return 0;
|
| 6330 | }
|
| 6331 |
|
| 6332 | int ext4_file_getattr(const struct mnt_idmap *idmap,
|
| 6333 | const struct path *path, struct kstat *stat,
|
| 6334 | u32 request_mask, unsigned int query_flags)
|
| 6335 | {
|
| 6336 | struct inode *inode = d_inode(path->dentry);
|
| 6337 | u64 delalloc_blocks;
|
| 6338 |
|
| 6339 | ext4_getattr(idmap, path, stat, request_mask, query_flags);
|
| 6340 |
|
| 6341 | |
| 6342 | * If there is inline data in the inode, the inode will normally not
|
| 6343 | * have data blocks allocated (it may have an external xattr block).
|
| 6344 | * Report at least one sector for such files, so tools like tar, rsync,
|
| 6345 | * others don't incorrectly think the file is completely sparse.
|
| 6346 |
|
| 6347 | if (unlikely(ext4_has_inline_data(inode)))
|
| 6348 | stat->blocks += (stat->size + 511) >> 9;
|
| 6349 |
|
| 6350 | |
| 6351 | * We can't update i_blocks if the block allocation is delayed
|
| 6352 | * otherwise in the case of system crash before the real block
|
| 6353 | * allocation is done, we will have i_blocks inconsistent with
|
| 6354 | * on-disk file blocks.
|
| 6355 | * We always keep i_blocks updated together with real
|
| 6356 | * allocation. But to not confuse with user, stat
|
| 6357 | * will return the blocks that include the delayed allocation
|
| 6358 | * blocks for this file.
|
| 6359 |
|
| 6360 | delalloc_blocks = EXT4_C2B(EXT4_SB(inode->i_sb),
|
| 6361 | EXT4_I(inode)->i_reserved_data_blocks);
|
| 6362 | stat->blocks += delalloc_blocks << (inode->i_sb->s_blocksize_bits - 9);
|
| 6363 | return 0;
|
| 6364 | }
|
| 6365 |
|
| 6366 | static int ext4_index_trans_blocks(struct inode *inode, int lblocks,
|
| 6367 | int pextents)
|
| 6368 | {
|
| 6369 | if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)))
|
| 6370 | return ext4_ind_trans_blocks(inode, lblocks);
|
| 6371 | return ext4_ext_index_trans_blocks(inode, pextents);
|
| 6372 | }
|
| 6373 |
|
| 6374 | |
| 6375 | * Calculate number of credits needed in a transaction to:
|
| 6376 | * * Allocate data blocks from @alloc_extents different groups - note that
|
| 6377 | * with flexbg a single physical extent can span multiple groups but
|
| 6378 | * single mballoc request only returns extent within one group.
|
| 6379 | * * Allocate metatadata (extent tree blocks, indirect blocks) to store
|
| 6380 | * pointers to @pextents data extents having @lblocks in total.
|
| 6381 | * * Modify extent tree / indirect block tree, inode, superblock, quota
|
| 6382 | * tracking, xattr blocks
|
| 6383 |
|
| 6384 | int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents,
|
| 6385 | int alloc_extents)
|
| 6386 | {
|
| 6387 | ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb);
|
| 6388 | int gdpblocks;
|
| 6389 | int idxblocks;
|
| 6390 | int ret;
|
| 6391 |
|
| 6392 | |
| 6393 | * How many index and leaf blocks need to touch to map @lblocks
|
| 6394 | * logical blocks to @pextents physical extents?
|
| 6395 |
|
| 6396 | idxblocks = ext4_index_trans_blocks(inode, lblocks, pextents);
|
| 6397 |
|
| 6398 | |
| 6399 | * Now let's see how many group bitmaps and group descriptors need
|
| 6400 | * to account
|
| 6401 |
|
| 6402 | groups = idxblocks + alloc_extents;
|
| 6403 | gdpblocks = groups;
|
| 6404 | if (groups > ngroups)
|
| 6405 | groups = ngroups;
|
| 6406 | if (groups > EXT4_SB(inode->i_sb)->s_gdb_count)
|
| 6407 | gdpblocks = EXT4_SB(inode->i_sb)->s_gdb_count;
|
| 6408 |
|
| 6409 |
|
| 6410 | ret = idxblocks + groups + gdpblocks;
|
| 6411 |
|
| 6412 |
|
| 6413 | ret += EXT4_META_TRANS_BLOCKS(inode->i_sb);
|
| 6414 |
|
| 6415 | return ret;
|
| 6416 | }
|
| 6417 |
|
| 6418 | |
| 6419 | * Calculate the journal credits for modifying the number of blocks
|
| 6420 | * in a single extent within one transaction. 'nrblocks' is used only
|
| 6421 | * for non-extent inodes. For extent type inodes, 'nrblocks' can be
|
| 6422 | * zero if the exact number of blocks is unknown.
|
| 6423 |
|
| 6424 | int ext4_chunk_trans_extent(struct inode *inode, int nrblocks)
|
| 6425 | {
|
| 6426 | int ret;
|
| 6427 |
|
| 6428 | ret = ext4_meta_trans_blocks(inode, nrblocks, 1, 1);
|
| 6429 |
|
| 6430 | if (ext4_should_journal_data(inode))
|
| 6431 | ret += nrblocks;
|
| 6432 | return ret;
|
| 6433 | }
|
| 6434 |
|
| 6435 | |
| 6436 | * Calculate the journal credits for a chunk of data modification.
|
| 6437 | *
|
| 6438 | * This is called from DIO, fallocate or whoever calling
|
| 6439 | * ext4_map_blocks() to map/allocate a chunk of contiguous disk blocks.
|
| 6440 | *
|
| 6441 | * journal buffers for data blocks are not included here, as DIO
|
| 6442 | * and fallocate do no need to journal data buffers.
|
| 6443 |
|
| 6444 | int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks)
|
| 6445 | {
|
| 6446 | return ext4_meta_trans_blocks(inode, nrblocks, 1, 1);
|
| 6447 | }
|
| 6448 |
|
| 6449 | |
| 6450 | * The caller must have previously called ext4_reserve_inode_write().
|
| 6451 | * Give this, we know that the caller already has write access to iloc->bh.
|
| 6452 |
|
| 6453 | int ext4_mark_iloc_dirty(handle_t *handle,
|
| 6454 | struct inode *inode, struct ext4_iloc *iloc)
|
| 6455 | {
|
| 6456 | struct super_block *sb = inode->i_sb;
|
| 6457 | int err = 0;
|
| 6458 |
|
| 6459 | err = ext4_emergency_state(sb);
|
| 6460 | if (unlikely(err)) {
|
| 6461 | put_bh(iloc->bh);
|
| 6462 | return err;
|
| 6463 | }
|
| 6464 | ext4_fc_track_inode(handle, inode);
|
| 6465 |
|
| 6466 |
|
| 6467 | get_bh(iloc->bh);
|
| 6468 |
|
| 6469 |
|
| 6470 | err = ext4_do_update_inode(handle, inode, iloc);
|
| 6471 | put_bh(iloc->bh);
|
| 6472 | |
| 6473 | * Mark that there's metadata writeout pending for the inode so that it
|
| 6474 | * gets properly flushed on fsync(2) and similar. We don't bother for
|
| 6475 | * fastcommit replay as that flushes the whole bdev afterwards anyway.
|
| 6476 | * It is faster this way and we avoid entering fs writeback paths which
|
| 6477 | * aren't fully initialized yet.
|
| 6478 |
|
| 6479 | if (!ext4_handle_valid(handle) &&
|
| 6480 | !(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)) {
|
| 6481 | |
| 6482 | * Inode didn't need to go through dirtying, make sure it is
|
| 6483 | * attached to wb so that writeback can handle it.
|
| 6484 |
|
| 6485 | spin_lock(&inode->i_lock);
|
| 6486 | inode_attach_wb(inode, NULL);
|
| 6487 | spin_unlock(&inode->i_lock);
|
| 6488 | set_inode_metadata_writeback(inode);
|
| 6489 | }
|
| 6490 | return err;
|
| 6491 | }
|
| 6492 |
|
| 6493 | |
| 6494 | * On success, We end up with an outstanding reference count against
|
| 6495 | * iloc->bh. This _must_ be cleaned up later.
|
| 6496 |
|
| 6497 |
|
| 6498 | int
|
| 6499 | ext4_reserve_inode_write(handle_t *handle, struct inode *inode,
|
| 6500 | struct ext4_iloc *iloc)
|
| 6501 | {
|
| 6502 | int err;
|
| 6503 |
|
| 6504 | err = ext4_emergency_state(inode->i_sb);
|
| 6505 | if (unlikely(err))
|
| 6506 | return err;
|
| 6507 |
|
| 6508 | err = ext4_get_inode_loc(inode, iloc);
|
| 6509 | if (!err) {
|
| 6510 | BUFFER_TRACE(iloc->bh, "get_write_access");
|
| 6511 | err = ext4_journal_get_write_access(handle, inode->i_sb,
|
| 6512 | iloc->bh, EXT4_JTR_NONE);
|
| 6513 | if (err) {
|
| 6514 | brelse(iloc->bh);
|
| 6515 | iloc->bh = NULL;
|
| 6516 | }
|
| 6517 | ext4_fc_track_inode(handle, inode);
|
| 6518 | }
|
| 6519 | ext4_std_error(inode->i_sb, err);
|
| 6520 | return err;
|
| 6521 | }
|
| 6522 |
|
| 6523 | static int __ext4_expand_extra_isize(struct inode *inode,
|
| 6524 | unsigned int new_extra_isize,
|
| 6525 | struct ext4_iloc *iloc,
|
| 6526 | handle_t *handle, int *no_expand)
|
| 6527 | {
|
| 6528 | struct ext4_inode *raw_inode;
|
| 6529 | struct ext4_xattr_ibody_header *header;
|
| 6530 | unsigned int inode_size = EXT4_INODE_SIZE(inode->i_sb);
|
| 6531 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 6532 | int error;
|
| 6533 |
|
| 6534 |
|
| 6535 | if ((EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize > inode_size) ||
|
| 6536 | (ei->i_extra_isize & 3)) {
|
| 6537 | EXT4_ERROR_INODE(inode, "bad extra_isize %u (inode size %u)",
|
| 6538 | ei->i_extra_isize,
|
| 6539 | EXT4_INODE_SIZE(inode->i_sb));
|
| 6540 | return -EFSCORRUPTED;
|
| 6541 | }
|
| 6542 | if ((new_extra_isize < ei->i_extra_isize) ||
|
| 6543 | (new_extra_isize < 4) ||
|
| 6544 | (new_extra_isize > inode_size - EXT4_GOOD_OLD_INODE_SIZE))
|
| 6545 | return -EINVAL;
|
| 6546 |
|
| 6547 | raw_inode = ext4_raw_inode(iloc);
|
| 6548 |
|
| 6549 | header = IHDR(inode, raw_inode);
|
| 6550 |
|
| 6551 |
|
| 6552 | if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) ||
|
| 6553 | header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) {
|
| 6554 | memset((void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE +
|
| 6555 | EXT4_I(inode)->i_extra_isize, 0,
|
| 6556 | new_extra_isize - EXT4_I(inode)->i_extra_isize);
|
| 6557 | EXT4_I(inode)->i_extra_isize = new_extra_isize;
|
| 6558 | return 0;
|
| 6559 | }
|
| 6560 |
|
| 6561 | |
| 6562 | * We may need to allocate external xattr block so we need quotas
|
| 6563 | * initialized. Here we can be called with various locks held so we
|
| 6564 | * cannot affort to initialize quotas ourselves. So just bail.
|
| 6565 |
|
| 6566 | if (dquot_initialize_needed(inode))
|
| 6567 | return -EAGAIN;
|
| 6568 |
|
| 6569 |
|
| 6570 | error = ext4_expand_extra_isize_ea(inode, new_extra_isize,
|
| 6571 | raw_inode, handle);
|
| 6572 | if (error) {
|
| 6573 | |
| 6574 | * Inode size expansion failed; don't try again
|
| 6575 |
|
| 6576 | *no_expand = 1;
|
| 6577 | }
|
| 6578 |
|
| 6579 | return error;
|
| 6580 | }
|
| 6581 |
|
| 6582 | |
| 6583 | * Expand an inode by new_extra_isize bytes.
|
| 6584 | * Returns 0 on success or negative error number on failure.
|
| 6585 |
|
| 6586 | static int ext4_try_to_expand_extra_isize(struct inode *inode,
|
| 6587 | unsigned int new_extra_isize,
|
| 6588 | struct ext4_iloc iloc,
|
| 6589 | handle_t *handle)
|
| 6590 | {
|
| 6591 | int no_expand;
|
| 6592 | int error;
|
| 6593 |
|
| 6594 | if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND))
|
| 6595 | return -EOVERFLOW;
|
| 6596 |
|
| 6597 | |
| 6598 | * Skip expansion during mount (!SB_ACTIVE). Expanding extra isize
|
| 6599 | * may move xattrs to external blocks and release ea_inodes via iput.
|
| 6600 | * When !SB_ACTIVE, iput triggers write_inode_now() which acquires
|
| 6601 | * s_writepages_rwsem, causing a deadlock with the caller's active
|
| 6602 | * jbd2 handle (lock order: s_writepages_rwsem -> jbd2_handle).
|
| 6603 |
|
| 6604 | if (unlikely(!(inode->i_sb->s_flags & SB_ACTIVE)))
|
| 6605 | return -EBUSY;
|
| 6606 |
|
| 6607 | |
| 6608 | * In nojournal mode, we can immediately attempt to expand
|
| 6609 | * the inode. When journaled, we first need to obtain extra
|
| 6610 | * buffer credits since we may write into the EA block
|
| 6611 | * with this same handle. If journal_extend fails, then it will
|
| 6612 | * only result in a minor loss of functionality for that inode.
|
| 6613 | * If this is felt to be critical, then e2fsck should be run to
|
| 6614 | * force a large enough s_min_extra_isize.
|
| 6615 |
|
| 6616 | if (ext4_journal_extend(handle,
|
| 6617 | EXT4_DATA_TRANS_BLOCKS(inode->i_sb), 0) != 0)
|
| 6618 | return -ENOSPC;
|
| 6619 |
|
| 6620 | if (ext4_write_trylock_xattr(inode, &no_expand) == 0)
|
| 6621 | return -EBUSY;
|
| 6622 |
|
| 6623 | error = __ext4_expand_extra_isize(inode, new_extra_isize, &iloc,
|
| 6624 | handle, &no_expand);
|
| 6625 | ext4_write_unlock_xattr(inode, &no_expand);
|
| 6626 |
|
| 6627 | return error;
|
| 6628 | }
|
| 6629 |
|
| 6630 | int ext4_expand_extra_isize(struct inode *inode,
|
| 6631 | unsigned int new_extra_isize,
|
| 6632 | struct ext4_iloc *iloc)
|
| 6633 | {
|
| 6634 | handle_t *handle;
|
| 6635 | int no_expand;
|
| 6636 | int error, rc;
|
| 6637 |
|
| 6638 | if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) {
|
| 6639 | brelse(iloc->bh);
|
| 6640 | return -EOVERFLOW;
|
| 6641 | }
|
| 6642 |
|
| 6643 | handle = ext4_journal_start(inode, EXT4_HT_INODE,
|
| 6644 | EXT4_DATA_TRANS_BLOCKS(inode->i_sb));
|
| 6645 | if (IS_ERR(handle)) {
|
| 6646 | error = PTR_ERR(handle);
|
| 6647 | brelse(iloc->bh);
|
| 6648 | return error;
|
| 6649 | }
|
| 6650 |
|
| 6651 | ext4_write_lock_xattr(inode, &no_expand);
|
| 6652 |
|
| 6653 | BUFFER_TRACE(iloc->bh, "get_write_access");
|
| 6654 | error = ext4_journal_get_write_access(handle, inode->i_sb, iloc->bh,
|
| 6655 | EXT4_JTR_NONE);
|
| 6656 | if (error) {
|
| 6657 | brelse(iloc->bh);
|
| 6658 | goto out_unlock;
|
| 6659 | }
|
| 6660 |
|
| 6661 | error = __ext4_expand_extra_isize(inode, new_extra_isize, iloc,
|
| 6662 | handle, &no_expand);
|
| 6663 |
|
| 6664 | rc = ext4_mark_iloc_dirty(handle, inode, iloc);
|
| 6665 | if (!error)
|
| 6666 | error = rc;
|
| 6667 |
|
| 6668 | out_unlock:
|
| 6669 | ext4_write_unlock_xattr(inode, &no_expand);
|
| 6670 | ext4_journal_stop(handle);
|
| 6671 | return error;
|
| 6672 | }
|
| 6673 |
|
| 6674 | |
| 6675 | * What we do here is to mark the in-core inode as clean with respect to inode
|
| 6676 | * dirtiness (it may still be data-dirty).
|
| 6677 | * This means that the in-core inode may be reaped by prune_icache
|
| 6678 | * without having to perform any I/O. This is a very good thing,
|
| 6679 | * because *any* task may call prune_icache - even ones which
|
| 6680 | * have a transaction open against a different journal.
|
| 6681 | *
|
| 6682 | * Is this cheating? Not really. Sure, we haven't written the
|
| 6683 | * inode out, but prune_icache isn't a user-visible syncing function.
|
| 6684 | * Whenever the user wants stuff synced (sys_sync, sys_msync, sys_fsync)
|
| 6685 | * we start and wait on commits.
|
| 6686 |
|
| 6687 | int __ext4_mark_inode_dirty(handle_t *handle, struct inode *inode,
|
| 6688 | const char *func, unsigned int line)
|
| 6689 | {
|
| 6690 | struct ext4_iloc iloc;
|
| 6691 | struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
|
| 6692 | int err;
|
| 6693 |
|
| 6694 | might_sleep();
|
| 6695 | trace_ext4_mark_inode_dirty(inode, _RET_IP_);
|
| 6696 | err = ext4_reserve_inode_write(handle, inode, &iloc);
|
| 6697 | if (err)
|
| 6698 | goto out;
|
| 6699 |
|
| 6700 | if (EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize)
|
| 6701 | ext4_try_to_expand_extra_isize(inode, sbi->s_want_extra_isize,
|
| 6702 | iloc, handle);
|
| 6703 |
|
| 6704 | err = ext4_mark_iloc_dirty(handle, inode, &iloc);
|
| 6705 | out:
|
| 6706 | if (unlikely(err))
|
| 6707 | ext4_error_inode_err(inode, func, line, 0, err,
|
| 6708 | "mark_inode_dirty error");
|
| 6709 | return err;
|
| 6710 | }
|
| 6711 |
|
| 6712 | |
| 6713 | * ext4_dirty_inode() is called from __mark_inode_dirty()
|
| 6714 | *
|
| 6715 | * We're really interested in the case where a file is being extended.
|
| 6716 | * i_size has been changed by generic_commit_write() and we thus need
|
| 6717 | * to include the updated inode in the current transaction.
|
| 6718 | *
|
| 6719 | * Also, dquot_alloc_block() will always dirty the inode when blocks
|
| 6720 | * are allocated to the file.
|
| 6721 | *
|
| 6722 | * If the inode is marked synchronous, we don't honour that here - doing
|
| 6723 | * so would cause a commit on atime updates, which we don't bother doing.
|
| 6724 | * We handle synchronous inodes at the highest possible level.
|
| 6725 |
|
| 6726 | void ext4_dirty_inode(struct inode *inode, int flags)
|
| 6727 | {
|
| 6728 | handle_t *handle;
|
| 6729 |
|
| 6730 | handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
|
| 6731 | if (IS_ERR(handle))
|
| 6732 | return;
|
| 6733 | ext4_mark_inode_dirty(handle, inode);
|
| 6734 | ext4_journal_stop(handle);
|
| 6735 | }
|
| 6736 |
|
| 6737 | int ext4_change_inode_journal_flag(struct inode *inode, int val)
|
| 6738 | {
|
| 6739 | journal_t *journal;
|
| 6740 | handle_t *handle;
|
| 6741 | int err;
|
| 6742 | int alloc_ctx;
|
| 6743 |
|
| 6744 | |
| 6745 | * We have to be very careful here: changing a data block's
|
| 6746 | * journaling status dynamically is dangerous. If we write a
|
| 6747 | * data block to the journal, change the status and then delete
|
| 6748 | * that block, we risk forgetting to revoke the old log record
|
| 6749 | * from the journal and so a subsequent replay can corrupt data.
|
| 6750 | * So, first we make sure that the journal is empty and that
|
| 6751 | * nobody is changing anything.
|
| 6752 |
|
| 6753 |
|
| 6754 | journal = EXT4_JOURNAL(inode);
|
| 6755 | if (!journal)
|
| 6756 | return 0;
|
| 6757 | if (is_journal_aborted(journal))
|
| 6758 | return -EROFS;
|
| 6759 |
|
| 6760 |
|
| 6761 | inode_dio_wait(inode);
|
| 6762 |
|
| 6763 | |
| 6764 | * Before flushing the journal and switching inode's aops, we have
|
| 6765 | * to flush all dirty data the inode has. There can be outstanding
|
| 6766 | * delayed allocations, there can be unwritten extents created by
|
| 6767 | * fallocate or buffered writes in dioread_nolock mode covered by
|
| 6768 | * dirty data which can be converted only after flushing the dirty
|
| 6769 | * data (and journalled aops don't know how to handle these cases).
|
| 6770 |
|
| 6771 | filemap_invalidate_lock(inode->i_mapping);
|
| 6772 | err = filemap_write_and_wait(inode->i_mapping);
|
| 6773 | if (err < 0) {
|
| 6774 | filemap_invalidate_unlock(inode->i_mapping);
|
| 6775 | return err;
|
| 6776 | }
|
| 6777 |
|
| 6778 | truncate_pagecache(inode, 0);
|
| 6779 |
|
| 6780 | alloc_ctx = ext4_writepages_down_write(inode->i_sb);
|
| 6781 | jbd2_journal_lock_updates(journal);
|
| 6782 |
|
| 6783 | |
| 6784 | * OK, there are no updates running now, and all cached data is
|
| 6785 | * synced to disk. We are now in a completely consistent state
|
| 6786 | * which doesn't have anything in the journal, and we know that
|
| 6787 | * no filesystem updates are running, so it is safe to modify
|
| 6788 | * the inode's in-core data-journaling state flag now.
|
| 6789 |
|
| 6790 |
|
| 6791 | if (val)
|
| 6792 | ext4_set_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
|
| 6793 | else {
|
| 6794 | err = jbd2_journal_flush(journal, 0);
|
| 6795 | if (err < 0) {
|
| 6796 | jbd2_journal_unlock_updates(journal);
|
| 6797 | ext4_writepages_up_write(inode->i_sb, alloc_ctx);
|
| 6798 | filemap_invalidate_unlock(inode->i_mapping);
|
| 6799 | return err;
|
| 6800 | }
|
| 6801 | ext4_clear_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
|
| 6802 | }
|
| 6803 | ext4_set_aops(inode);
|
| 6804 | ext4_set_inode_mapping_order(inode);
|
| 6805 |
|
| 6806 | jbd2_journal_unlock_updates(journal);
|
| 6807 | ext4_writepages_up_write(inode->i_sb, alloc_ctx);
|
| 6808 | filemap_invalidate_unlock(inode->i_mapping);
|
| 6809 |
|
| 6810 |
|
| 6811 |
|
| 6812 | handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
|
| 6813 | if (IS_ERR(handle))
|
| 6814 | return PTR_ERR(handle);
|
| 6815 |
|
| 6816 | ext4_fc_mark_ineligible(inode->i_sb,
|
| 6817 | EXT4_FC_REASON_JOURNAL_FLAG_CHANGE, handle);
|
| 6818 | err = ext4_mark_inode_dirty(handle, inode);
|
| 6819 | ext4_handle_sync(handle);
|
| 6820 | ext4_journal_stop(handle);
|
| 6821 | ext4_std_error(inode->i_sb, err);
|
| 6822 |
|
| 6823 | return err;
|
| 6824 | }
|
| 6825 |
|
| 6826 | static int ext4_bh_unmapped(handle_t *handle, struct inode *inode,
|
| 6827 | struct buffer_head *bh)
|
| 6828 | {
|
| 6829 | return !buffer_mapped(bh);
|
| 6830 | }
|
| 6831 |
|
| 6832 | static int ext4_block_page_mkwrite(struct inode *inode, struct folio *folio,
|
| 6833 | get_block_t get_block)
|
| 6834 | {
|
| 6835 | handle_t *handle;
|
| 6836 | loff_t size;
|
| 6837 | unsigned long len;
|
| 6838 | int credits;
|
| 6839 | int ret;
|
| 6840 |
|
| 6841 | credits = ext4_chunk_trans_extent(inode,
|
| 6842 | ext4_journal_blocks_per_folio(inode));
|
| 6843 | handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, credits);
|
| 6844 | if (IS_ERR(handle))
|
| 6845 | return PTR_ERR(handle);
|
| 6846 |
|
| 6847 | folio_lock(folio);
|
| 6848 | size = i_size_read(inode);
|
| 6849 |
|
| 6850 | if (folio->mapping != inode->i_mapping || folio_pos(folio) > size) {
|
| 6851 | ret = -EFAULT;
|
| 6852 | goto out_error;
|
| 6853 | }
|
| 6854 |
|
| 6855 | len = folio_size(folio);
|
| 6856 | if (folio_pos(folio) + len > size)
|
| 6857 | len = size - folio_pos(folio);
|
| 6858 |
|
| 6859 | ret = ext4_block_write_begin(handle, folio, 0, len, get_block);
|
| 6860 | if (ret)
|
| 6861 | goto out_error;
|
| 6862 |
|
| 6863 | if (!ext4_should_journal_data(inode)) {
|
| 6864 | block_commit_write(folio, 0, len);
|
| 6865 | folio_mark_dirty(folio);
|
| 6866 | } else {
|
| 6867 | ret = ext4_journal_folio_buffers(handle, folio, len);
|
| 6868 | if (ret)
|
| 6869 | goto out_error;
|
| 6870 | }
|
| 6871 | ext4_journal_stop(handle);
|
| 6872 | folio_wait_stable(folio);
|
| 6873 | return ret;
|
| 6874 |
|
| 6875 | out_error:
|
| 6876 | folio_unlock(folio);
|
| 6877 | ext4_journal_stop(handle);
|
| 6878 | return ret;
|
| 6879 | }
|
| 6880 |
|
| 6881 | vm_fault_t ext4_page_mkwrite(struct vm_fault *vmf)
|
| 6882 | {
|
| 6883 | struct vm_area_struct *vma = vmf->vma;
|
| 6884 | struct folio *folio = page_folio(vmf->page);
|
| 6885 | loff_t size;
|
| 6886 | unsigned long len;
|
| 6887 | int err;
|
| 6888 | vm_fault_t ret;
|
| 6889 | struct file *file = vma->vm_file;
|
| 6890 | struct inode *inode = file_inode(file);
|
| 6891 | struct address_space *mapping = inode->i_mapping;
|
| 6892 | get_block_t *get_block = ext4_get_block;
|
| 6893 | int retries = 0;
|
| 6894 |
|
| 6895 | if (unlikely(IS_IMMUTABLE(inode)))
|
| 6896 | return VM_FAULT_SIGBUS;
|
| 6897 |
|
| 6898 | sb_start_pagefault(inode->i_sb);
|
| 6899 | file_update_time(vma->vm_file);
|
| 6900 |
|
| 6901 | filemap_invalidate_lock_shared(mapping);
|
| 6902 |
|
| 6903 | err = ext4_convert_inline_data(inode);
|
| 6904 | if (err)
|
| 6905 | goto out_ret;
|
| 6906 |
|
| 6907 | |
| 6908 | * On data journalling we skip straight to the transaction handle:
|
| 6909 | * there's no delalloc; page truncated will be checked later; the
|
| 6910 | * early return w/ all buffers mapped (calculates size/len) can't
|
| 6911 | * be used; and there's no dioread_nolock, so only ext4_get_block.
|
| 6912 |
|
| 6913 | if (ext4_should_journal_data(inode))
|
| 6914 | goto retry_alloc;
|
| 6915 |
|
| 6916 |
|
| 6917 | if (test_opt(inode->i_sb, DELALLOC) &&
|
| 6918 | !ext4_nonda_switch(inode->i_sb)) {
|
| 6919 | do {
|
| 6920 | err = block_page_mkwrite(vma, vmf,
|
| 6921 | ext4_da_get_block_prep);
|
| 6922 | } while (err == -ENOSPC &&
|
| 6923 | ext4_should_retry_alloc(inode->i_sb, &retries));
|
| 6924 | goto out_ret;
|
| 6925 | }
|
| 6926 |
|
| 6927 | folio_lock(folio);
|
| 6928 | size = i_size_read(inode);
|
| 6929 |
|
| 6930 | if (folio->mapping != mapping || folio_pos(folio) > size) {
|
| 6931 | folio_unlock(folio);
|
| 6932 | ret = VM_FAULT_NOPAGE;
|
| 6933 | goto out;
|
| 6934 | }
|
| 6935 |
|
| 6936 | len = folio_size(folio);
|
| 6937 | if (folio_pos(folio) + len > size)
|
| 6938 | len = size - folio_pos(folio);
|
| 6939 | |
| 6940 | * Return if we have all the buffers mapped. This avoids the need to do
|
| 6941 | * journal_start/journal_stop which can block and take a long time
|
| 6942 | *
|
| 6943 | * This cannot be done for data journalling, as we have to add the
|
| 6944 | * inode to the transaction's list to writeprotect pages on commit.
|
| 6945 |
|
| 6946 | if (folio_buffers(folio)) {
|
| 6947 | if (!ext4_walk_page_buffers(NULL, inode, folio_buffers(folio),
|
| 6948 | 0, len, NULL,
|
| 6949 | ext4_bh_unmapped)) {
|
| 6950 |
|
| 6951 | folio_wait_stable(folio);
|
| 6952 | ret = VM_FAULT_LOCKED;
|
| 6953 | goto out;
|
| 6954 | }
|
| 6955 | }
|
| 6956 | folio_unlock(folio);
|
| 6957 |
|
| 6958 | if (ext4_should_dioread_nolock(inode))
|
| 6959 | get_block = ext4_get_block_unwritten;
|
| 6960 | retry_alloc:
|
| 6961 |
|
| 6962 | err = ext4_block_page_mkwrite(inode, folio, get_block);
|
| 6963 | if (err == -EAGAIN ||
|
| 6964 | (err == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)))
|
| 6965 | goto retry_alloc;
|
| 6966 | out_ret:
|
| 6967 | ret = vmf_fs_error(err);
|
| 6968 | out:
|
| 6969 | filemap_invalidate_unlock_shared(mapping);
|
| 6970 | sb_end_pagefault(inode->i_sb);
|
| 6971 | return ret;
|
| 6972 | }
|