diff --git a/Documentation/ABI/testing/sysfs-fs-f2fs b/Documentation/ABI/testing/sysfs-fs-f2fs index 1b58c02..0cebc89 100644 --- a/Documentation/ABI/testing/sysfs-fs-f2fs +++ b/Documentation/ABI/testing/sysfs-fs-f2fs @@ -986,6 +986,7 @@ Description: This sysfs entry can be used to enable/disable to adjust priority f 0x00000008 gc_lock 0x00000010 cp_global 0x00000020 io_rwsem + 0x00000040 nat_tree_lock ========== ================== What: /sys/fs/f2fs//lock_duration_priority @@ -1002,3 +1003,20 @@ Description: It can be used to tune priority of f2fs critical task, e.g. f2fs_ck threads, limitation as below: - it requires user has CAP_SYS_NICE capability. - the range is [100, 139], by default the value is 120. + +What: /sys/fs/f2fs//max_atc_write_bio_size +Date: June 2026 +Contact: Bart Van Assche +Description: Every time a write operation completes f2fs_write_end_io() is + called. This function may be called from an atomic context, + e.g. from inside an interrupt handler. This attribute controls + the maximum size of a write bio that is completed in atomic + (atc) context. The default value for this attribute is UINT_MAX + which means that this functionality is disabled by default. + +What: /sys/fs/f2fs//pinned_area_max_secno +Date: August 2026 +Contact: "Daeho Jeong" +Description: This is a read-only entry to show the upper bound section number + for pinned files. Pinned files will only be allocated within + sections 0 to pinned_area_max_secno - 1. diff --git a/Documentation/filesystems/f2fs.rst b/Documentation/filesystems/f2fs.rst index b45d7a6..771216f 100644 --- a/Documentation/filesystems/f2fs.rst +++ b/Documentation/filesystems/f2fs.rst @@ -415,7 +415,13 @@ lookup_mode=%s Control the directory lookup behavior for casefolded auto F2FS determines the mode based on the on-disk `SB_ENC_NO_COMPAT_FALLBACK_FL` flag. - ================== ======================================== +resizable_tail_secno=%u Control the number of sections at the tail of the + filesystem reserved for online resizing. Pinned files + will only be allocated within sections 0 to + (MAIN_SECS - resizable_tail_secno) - 1. If set to 0 + (default), there is no tail restriction unless running + on a zoned block device where conventional zones are + used. ======================== ============================================================ Debugfs Entries @@ -1043,6 +1049,41 @@ So, the key idea is, user can do any file operations on /dev/vdc, and reclaim the space after the use, while the space is counted as /data. That doesn't require modifying partition size and filesystem format. +Dynamic Device Aliasing Management +---------------------------------- + +In addition to static device aliasing by deleting the aliasing file, F2FS +supports dynamic management of device aliasing. This mechanism allows the system +to dynamically transition partition ownership between F2FS userdata and external +entities (e.g., zRAM, raw partition) based on system requirements without +deleting the master aliasing file or requiring unmount/remount. + +The master aliasing file is created during the initial format of the file system +and remains as a persistent control entity (ioctl gateway) in the root directory. + +- Partition Reservation (In-service to Aliased) + When a specific partition needs to be dedicated to external services (e.g., zRAM), + a user can reserve the device alias range via ioctl. The kernel resets GC victim + information for the target range, marks segments as in-use to prevent new + allocations, and triggers forced GC to migrate existing valid data out of the + range. Finally, it reserves these blocks in the SIT to effectively exclude the + device from the usable capacity. + +- Partition Release (Aliased to In-service) + When external usage concludes, the space is reclaimed not by deleting the file, + but through the release ioctl. The kernel truncates blocks associated with + the file, releasing them back to general filesystem allocation. + +.. code-block:: + + # f2fs_io dev_alias release /mnt/f2fs/vdc.file + # df -h + /dev/vdb 64G 753M 64G 2% /mnt/f2fs + + # f2fs_io dev_alias reserve /mnt/f2fs/vdc.file + # df -h + /dev/vdb 64G 33G 32G 52% /mnt/f2fs + Per-file Read-Only Large Folio Support -------------------------------------- diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index 064f5b5..4b59f30 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -107,6 +107,7 @@ static bool need_uplift_priority(struct f2fs_rwsem *sem, bool is_write) case LOCK_NAME_GC_LOCK: case LOCK_NAME_CP_GLOBAL: case LOCK_NAME_IO_RWSEM: + case LOCK_NAME_NAT_TREE_LOCK: return true; default: f2fs_bug_on(sem->sbi, 1); @@ -766,28 +767,116 @@ static void __remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type) spin_unlock(&im->ino_lock); } +static void __set_ino_bitmap(struct f2fs_sb_info *sbi, nid_t ino, int type) +{ + struct inode_management *im = &sbi->im[type]; + unsigned long index = INO_SLOT_INDEX(ino); + unsigned int ofs = INO_BIT_OFFSET(ino); + void **slot, *entry; + unsigned long bitmap = 0; + int ret; + + ret = radix_tree_preload(GFP_NOFS | __GFP_NOFAIL); + f2fs_bug_on(sbi, ret); + + spin_lock(&im->ino_lock); + slot = radix_tree_lookup_slot(&im->ino_root, index); + if (slot) { + entry = radix_tree_deref_slot_protected(slot, &im->ino_lock); + bitmap = xa_to_value(entry); + if (!(bitmap & (1UL << ofs))) { + bitmap |= (1UL << ofs); + entry = xa_mk_value(bitmap); + radix_tree_replace_slot(&im->ino_root, slot, entry); + } + } else { + bitmap |= (1UL << ofs); + entry = xa_mk_value(bitmap); + if (unlikely(radix_tree_insert(&im->ino_root, index, entry))) + f2fs_bug_on(sbi, 1); + } + spin_unlock(&im->ino_lock); + radix_tree_preload_end(); +} + +static void __clear_ino_bitmap(struct f2fs_sb_info *sbi, nid_t ino, int type) +{ + struct inode_management *im = &sbi->im[type]; + unsigned long index = INO_SLOT_INDEX(ino); + unsigned int ofs = INO_BIT_OFFSET(ino); + void **slot, *entry; + unsigned long bitmap; + + spin_lock(&im->ino_lock); + slot = radix_tree_lookup_slot(&im->ino_root, index); + if (slot) { + entry = radix_tree_deref_slot_protected(slot, &im->ino_lock); + bitmap = xa_to_value(entry); + if (bitmap & (1UL << ofs)) + bitmap &= ~(1UL << ofs); + + if (bitmap) { + entry = xa_mk_value(bitmap); + radix_tree_replace_slot(&im->ino_root, slot, entry); + } else { + radix_tree_delete(&im->ino_root, index); + } + } + spin_unlock(&im->ino_lock); +} + +static void f2fs_wait_for_inode_record(struct f2fs_sb_info *sbi, int mode) +{ + if (mode != APPEND_INO && mode != UPDATE_INO) + return; + + /* Let's wait for some pending updates for APPEND_INO and UPDATE_INO. */ + flush_workqueue(sbi->evict_wq); +} + +static void __f2fs_add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, + unsigned int devidx, int type) +{ + if (type <= FLUSH_INO) + __add_ino_entry(sbi, ino, devidx, type); + else + __set_ino_bitmap(sbi, ino, type); +} + void f2fs_add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type) { - /* add new dirty ino entry into list */ - __add_ino_entry(sbi, ino, 0, type); + __f2fs_add_ino_entry(sbi, ino, 0, type); } void f2fs_remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type) { - /* remove dirty ino entry from list */ - __remove_ino_entry(sbi, ino, type); + if (type <= FLUSH_INO) + __remove_ino_entry(sbi, ino, type); + else + __clear_ino_bitmap(sbi, ino, type); } -/* mode should be APPEND_INO, UPDATE_INO or TRANS_DIR_INO */ +/* mode should be APPEND_INO, UPDATE_INO, TRANS_DIR_INO and XATTR_DIR_INO */ bool f2fs_exist_written_data(struct f2fs_sb_info *sbi, nid_t ino, int mode) { struct inode_management *im = &sbi->im[mode]; - struct ino_entry *e; + unsigned long index = INO_SLOT_INDEX(ino); + unsigned int ofs = INO_BIT_OFFSET(ino); + void *entry; + unsigned long bitmap; + + f2fs_bug_on(sbi, mode <= FLUSH_INO); spin_lock(&im->ino_lock); - e = radix_tree_lookup(&im->ino_root, ino); + entry = radix_tree_lookup(&im->ino_root, index); + if (!entry) { + spin_unlock(&im->ino_lock); + return false; + } + bitmap = xa_to_value(entry); spin_unlock(&im->ino_lock); - return e ? true : false; + + return bitmap & (1UL << ofs); } void f2fs_release_ino_entry(struct f2fs_sb_info *sbi, bool all) @@ -795,9 +884,11 @@ void f2fs_release_ino_entry(struct f2fs_sb_info *sbi, bool all) struct ino_entry *e, *tmp; int i; - for (i = all ? ORPHAN_INO : APPEND_INO; i < MAX_INO_ENTRY; i++) { + for (i = all ? ORPHAN_INO : FLUSH_INO; i <= FLUSH_INO; i++) { struct inode_management *im = &sbi->im[i]; + f2fs_wait_for_inode_record(sbi, i); + spin_lock(&im->ino_lock); list_for_each_entry_safe(e, tmp, &im->ino_list, list) { list_del(&e->list); @@ -807,12 +898,20 @@ void f2fs_release_ino_entry(struct f2fs_sb_info *sbi, bool all) } spin_unlock(&im->ino_lock); } + + for (i = APPEND_INO; i < MAX_INO_ENTRY; i++) { + struct inode_management *im = &sbi->im[i]; + + spin_lock(&im->ino_lock); + xa_destroy(&im->ino_root); + spin_unlock(&im->ino_lock); + } } void f2fs_set_dirty_device(struct f2fs_sb_info *sbi, nid_t ino, unsigned int devidx, int type) { - __add_ino_entry(sbi, ino, devidx, type); + __f2fs_add_ino_entry(sbi, ino, devidx, type); } bool f2fs_is_dirty_device(struct f2fs_sb_info *sbi, nid_t ino, @@ -864,14 +963,14 @@ void f2fs_release_orphan_inode(struct f2fs_sb_info *sbi) void f2fs_add_orphan_inode(struct inode *inode) { /* add new orphan ino entry into list */ - __add_ino_entry(F2FS_I_SB(inode), inode->i_ino, 0, ORPHAN_INO); + f2fs_add_ino_entry(F2FS_I_SB(inode), inode->i_ino, ORPHAN_INO); f2fs_update_inode_page(inode); } void f2fs_remove_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino) { /* remove orphan entry from orphan list */ - __remove_ino_entry(sbi, ino, ORPHAN_INO); + f2fs_remove_ino_entry(sbi, ino, ORPHAN_INO); } static int recover_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index dad5c2b..21f396e 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -384,11 +384,30 @@ static void f2fs_write_end_bio(struct bio *bio) bio_put(bio); } +static void f2fs_write_end_io_work(struct work_struct *work) +{ + struct bio *bio = &container_of(work, struct f2fs_bio, work)->bio; + + f2fs_write_end_bio(bio); +} + static void f2fs_write_end_io(struct bio *bio) { + struct f2fs_sb_info *sbi; + iostat_update_and_unbind_ctx(bio); - f2fs_write_end_bio(bio); + sbi = bio->bi_private; + + if (in_atomic() && bio->bi_iter.bi_size > sbi->max_atc_write_bio_size) { + struct work_struct *w; + + w = &container_of(bio, struct f2fs_bio, bio)->work; + INIT_WORK(w, f2fs_write_end_io_work); + queue_work(sbi->wq, w); + } else { + f2fs_write_end_bio(bio); + } } #ifdef CONFIG_BLK_DEV_ZONED @@ -1216,7 +1235,7 @@ int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count) if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) return -EPERM; - err = inc_valid_block_count(sbi, dn->inode, &count, true); + err = inc_valid_block_count(sbi, dn->inode, &count, true, false); if (unlikely(err)) return err; @@ -1288,10 +1307,11 @@ retry: if (folio_test_large(folio)) { pgoff_t folio_index = mapping_align_index(mapping, index); + unsigned long nr_pages = folio_nr_pages(folio); f2fs_folio_put(folio, true); invalidate_inode_pages2_range(mapping, folio_index, - folio_index + folio_nr_pages(folio) - 1); + folio_index + nr_pages - 1); f2fs_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT); goto retry; } @@ -1488,7 +1508,7 @@ static int __allocate_data_block(struct dnode_of_data *dn, int seg_type) dn->data_blkaddr = f2fs_data_blkaddr(dn); if (dn->data_blkaddr == NULL_ADDR) { - err = inc_valid_block_count(sbi, dn->inode, &count, true); + err = inc_valid_block_count(sbi, dn->inode, &count, true, false); if (unlikely(err)) return err; } @@ -1497,8 +1517,11 @@ static int __allocate_data_block(struct dnode_of_data *dn, int seg_type) old_blkaddr = dn->data_blkaddr; err = f2fs_allocate_data_block(sbi, NULL, old_blkaddr, &dn->data_blkaddr, &sum, seg_type, NULL); - if (err) + if (err) { + if (old_blkaddr == NULL_ADDR) + dec_valid_block_count(sbi, dn->inode, count); return err; + } if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) f2fs_invalidate_internal_cache(sbi, old_blkaddr, 1); @@ -3900,7 +3923,7 @@ repeat: * Will wait that below with our IO control. */ folio = f2fs_filemap_get_folio(mapping, index, - FGP_LOCK | FGP_WRITE | FGP_CREAT | FGP_NOFS, + FGP_LOCK | FGP_WRITE | FGP_CREAT, mapping_gfp_mask(mapping)); if (IS_ERR(folio)) { err = PTR_ERR(folio); @@ -4460,13 +4483,24 @@ int f2fs_init_wq(struct f2fs_sb_info *sbi) { sbi->wq = alloc_workqueue("f2fs_wq", WQ_UNBOUND | WQ_HIGHPRI, num_online_cpus()); - return sbi->wq ? 0 : -ENOMEM; + if (!sbi->wq) + return -ENOMEM; + + sbi->evict_wq = alloc_workqueue("f2fs_evict_wq", + WQ_UNBOUND | WQ_HIGHPRI, num_online_cpus()); + if (!sbi->evict_wq) { + destroy_workqueue(sbi->wq); + return -ENOMEM; + } + return 0; } void f2fs_destroy_wq(struct f2fs_sb_info *sbi) { if (sbi->wq) destroy_workqueue(sbi->wq); + if (sbi->evict_wq) + destroy_workqueue(sbi->evict_wq); } int __init f2fs_init_bio_entry_cache(void) diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c index af88db8..ff379af 100644 --- a/fs/f2fs/debug.c +++ b/fs/f2fs/debug.c @@ -352,10 +352,6 @@ static void update_mem_info(struct f2fs_sb_info *sbi) get_cache: si->cache_mem = 0; - /* build gc */ - if (sbi->gc_thread) - si->cache_mem += sizeof(struct f2fs_gc_kthread); - /* build merge flush thread */ if (SM_I(sbi)->fcc_info) si->cache_mem += sizeof(struct flush_cmd_control); diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c index a9563f7..fd0e2cd 100644 --- a/fs/f2fs/dir.c +++ b/fs/f2fs/dir.c @@ -320,6 +320,7 @@ start_find_bucket: de = find_in_block(dir, dentry_folio, fname, &max_slots, use_hash); if (IS_ERR(de)) { + f2fs_folio_put(dentry_folio, false); *res_folio = ERR_CAST(de); de = NULL; break; @@ -460,7 +461,7 @@ void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de, folio_mark_dirty(folio); inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); - f2fs_mark_inode_dirty_sync(dir, false); + f2fs_mark_inode_dirty_sync(dir, true); f2fs_folio_put(folio, true); } @@ -615,7 +616,7 @@ void f2fs_update_parent_metadata(struct inode *dir, struct inode *inode, clear_inode_flag(inode, FI_NEW_INODE); } inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); - f2fs_mark_inode_dirty_sync(dir, false); + f2fs_mark_inode_dirty_sync(dir, true); if (F2FS_I(dir)->i_current_depth != current_depth) f2fs_i_depth_write(dir, current_depth); @@ -927,7 +928,7 @@ void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct folio *folio, f2fs_folio_put(folio, true); inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); - f2fs_mark_inode_dirty_sync(dir, false); + f2fs_mark_inode_dirty_sync(dir, true); if (inode) f2fs_drop_nlink(dir, inode); diff --git a/fs/f2fs/extent_cache.c b/fs/f2fs/extent_cache.c index 61f6b97..37cf9fa 100644 --- a/fs/f2fs/extent_cache.c +++ b/fs/f2fs/extent_cache.c @@ -17,6 +17,7 @@ #include "f2fs.h" #include "node.h" +#include "segment.h" #include bool sanity_check_extent_cache(struct inode *inode, struct folio *ifolio) @@ -62,6 +63,14 @@ bool sanity_check_extent_cache(struct inode *inode, struct folio *ifolio) __func__, inode->i_ino, ei.blk, ei.fofs, ei.len); return false; } + + if ((GET_SEGOFF_FROM_SEG0(sbi, ei.blk) % BLKS_PER_SEC(sbi)) || + (ei.len % BLKS_PER_SEC(sbi))) { + f2fs_warn(sbi, "%s: device alias inode (ino=%llx)'s extent info [%u, %u, %u] is not aligned to section size %u", + __func__, inode->i_ino, ei.blk, ei.fofs, ei.len, + BLKS_PER_SEC(sbi)); + return false; + } return true; } diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 8376bbe..9940a6c 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -193,6 +193,7 @@ enum f2fs_lock_name { LOCK_NAME_GC_LOCK, LOCK_NAME_CP_GLOBAL, LOCK_NAME_IO_RWSEM, + LOCK_NAME_NAT_TREE_LOCK, LOCK_NAME_MAX, }; @@ -254,6 +255,7 @@ struct f2fs_mount_info { block_t unusable_cap; /* Amount of space allowed to be * unusable when disabling checkpoint */ + unsigned int resizable_tail_secno; /* number of resizable tail sections */ /* For compression */ unsigned char compress_algorithm; /* algorithm type */ @@ -388,14 +390,18 @@ enum { /* for the list of ino */ enum { ORPHAN_INO, /* for orphan ino list */ + FLUSH_INO, /* for multiple device flushing */ APPEND_INO, /* for append ino list */ UPDATE_INO, /* for update ino list */ TRANS_DIR_INO, /* for transactions dir ino list */ XATTR_DIR_INO, /* for xattr updated dir ino list */ - FLUSH_INO, /* for multiple device flushing */ MAX_INO_ENTRY, /* max. list */ }; +#define INO_BITS_PER_SLOT BITS_PER_XA_VALUE +#define INO_SLOT_INDEX(ino) ((ino) / INO_BITS_PER_SLOT) +#define INO_BIT_OFFSET(ino) ((ino) % INO_BITS_PER_SLOT) + struct ino_entry { struct list_head list; /* list head */ nid_t ino; /* inode number */ @@ -1403,6 +1409,8 @@ struct f2fs_dev_info { unsigned int total_segments; block_t start_blk; block_t end_blk; + bool has_alias; + bool is_reserving; #ifdef CONFIG_BLK_DEV_ZONED unsigned int nr_blkz; /* Total number of zones */ unsigned long *blkz_seq; /* Bitmap indicating sequential zones */ @@ -1747,6 +1755,33 @@ struct decompress_io_ctx { #define MAX_COMPRESS_LOG_SIZE 8 #define MAX_COMPRESS_WINDOW_SIZE(log_size) ((PAGE_SIZE) << (log_size)) +struct f2fs_gc_kthread { + struct task_struct *f2fs_gc_task; + wait_queue_head_t gc_wait_queue_head; + + /* for gc sleep time */ + unsigned int urgent_sleep_time; + unsigned int min_sleep_time; + unsigned int max_sleep_time; + unsigned int no_gc_sleep_time; + + /* for changing gc mode */ + bool gc_wake; + + /* for GC_MERGE mount option */ + wait_queue_head_t fggc_wq; /* + * caller of f2fs_balance_fs() + * will wait on this wait queue. + */ + + /* for gc control for zoned devices */ + unsigned int no_zoned_gc_percent; + unsigned int boost_zoned_gc_percent; + unsigned int valid_thresh_ratio; + unsigned int boost_gc_multiple; + unsigned int boost_gc_greedy; +}; + struct f2fs_sb_info { struct super_block *sb; /* pointer to VFS super block */ struct proc_dir_entry *s_proc; /* proc entry */ @@ -1772,6 +1807,8 @@ struct f2fs_sb_info { struct f2fs_sm_info *sm_info; /* segment manager */ /* for bio operations */ + /* Largest write bio size completed in atomic context (atc). */ + u32 max_atc_write_bio_size; struct f2fs_bio_info *write_io[NR_PAGE_TYPE]; /* for write bios */ /* keep migration IO order for LFS mode */ struct f2fs_rwsem io_order_lock; @@ -1854,6 +1891,7 @@ struct f2fs_sb_info { block_t last_valid_block_count; /* for recovery */ block_t reserved_blocks; /* configurable reserved blocks */ block_t current_reserved_blocks; /* current reserved blocks */ + block_t alias_reserved_blocks; /* reserved blocks for device alias */ /* Additional tracking for no checkpoint mode */ block_t unusable_block_count; /* # of blocks saved by last cp */ @@ -1882,7 +1920,7 @@ struct f2fs_sb_info { * semaphore for GC, avoid * race between GC and GC or CP */ - struct f2fs_gc_kthread *gc_thread; /* GC thread */ + struct f2fs_gc_kthread gc_thread; /* GC thread */ struct atgc_management am; /* atgc management */ unsigned int cur_victim_sec; /* current victim section num */ unsigned int gc_mode; /* current GC state */ @@ -1967,6 +2005,7 @@ struct f2fs_sb_info { spinlock_t dev_lock; /* protect dirty_device */ bool aligned_blksize; /* all devices has the same logical blksize */ unsigned int first_seq_zone_segno; /* first segno in sequential zone */ + unsigned int pinned_area_max_secno; /* upper bound section for pinned files */ unsigned int bggc_io_aware; /* For adjust the BG_GC priority when pending IO */ unsigned int allocate_section_hint; /* the boundary position between devices */ unsigned int allocate_section_policy; /* determine the section writing priority */ @@ -1980,6 +2019,8 @@ struct f2fs_sb_info { struct workqueue_struct *wq; /* bio completion workqueue */ + struct workqueue_struct *evict_wq; /* inode eviction workqueue */ + /* * If we are in irq context, let's update error information into * on-disk superblock in the work. @@ -2556,7 +2597,8 @@ static inline unsigned int get_available_block_count(struct f2fs_sb_info *sbi, block_t avail_user_block_count; avail_user_block_count = sbi->user_block_count - - sbi->current_reserved_blocks; + sbi->current_reserved_blocks - + sbi->alias_reserved_blocks; if (test_opt(sbi, RESERVE_ROOT) && !__allow_reserved_root(sbi, inode, cap)) avail_user_block_count -= F2FS_OPTION(sbi).root_reserved_blocks; @@ -2573,7 +2615,8 @@ static inline unsigned int get_available_block_count(struct f2fs_sb_info *sbi, static inline void f2fs_i_blocks_write(struct inode *, block_t, bool, bool); static inline int inc_valid_block_count(struct f2fs_sb_info *sbi, - struct inode *inode, blkcnt_t *count, bool partial) + struct inode *inode, blkcnt_t *count, + bool partial, bool alias_reserved) { long long diff = 0, release = 0; block_t avail_user_block_count; @@ -2596,10 +2639,16 @@ static inline int inc_valid_block_count(struct f2fs_sb_info *sbi, spin_lock(&sbi->stat_lock); + if (alias_reserved) + sbi->alias_reserved_blocks -= *count; + avail_user_block_count = get_available_block_count(sbi, inode, true); diff = (long long)sbi->total_valid_block_count + *count - avail_user_block_count; if (unlikely(diff > 0)) { + if (alias_reserved) + sbi->alias_reserved_blocks += *count; + if (!partial) { spin_unlock(&sbi->stat_lock); release = *count; @@ -3830,7 +3879,10 @@ void f2fs_update_inode_page(struct inode *inode); int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc); void f2fs_remove_donate_inode(struct inode *inode); void f2fs_evict_inode(struct inode *inode); -void f2fs_handle_failed_inode(struct inode *inode, struct f2fs_lock_context *lc); +void f2fs_handle_failed_inode(struct inode *inode, + struct f2fs_lock_context *lc, bool add_orphan); +int f2fs_init_evict_inode_work(void); +void f2fs_destroy_evict_inode_work(void); /* * namei.c @@ -4007,6 +4059,8 @@ int f2fs_flush_device_cache(struct f2fs_sb_info *sbi); void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free); void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr, unsigned int len); +void f2fs_reserve_device_alias(struct f2fs_sb_info *sbi, block_t addr, + unsigned int len); bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr); int f2fs_start_discard_thread(struct f2fs_sb_info *sbi); void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi); @@ -4095,10 +4149,26 @@ static inline struct inode *fio_inode(struct f2fs_io_info *fio) #define MIN_FRAGMENT_SIZE 1 #define MAX_FRAGMENT_SIZE 512 -static inline bool f2fs_need_rand_seg(struct f2fs_sb_info *sbi) +static inline bool f2fs_need_rand_blk(struct f2fs_sb_info *sbi, + enum log_type type) +{ + if (type == CURSEG_COLD_DATA_PINNED) + return false; + return F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK; +} + +static inline bool f2fs_need_rand_seg(struct f2fs_sb_info *sbi, + enum log_type type) +{ + if (type == CURSEG_COLD_DATA_PINNED) + return false; + return F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_SEG; +} + +static inline bool f2fs_need_rand_seg_blk(struct f2fs_sb_info *sbi, + enum log_type type) { - return F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_SEG || - F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK; + return f2fs_need_rand_blk(sbi, type) || f2fs_need_rand_seg(sbi, type); } /* @@ -4226,7 +4296,9 @@ int f2fs_gc(struct f2fs_sb_info *sbi, struct f2fs_gc_control *gc_control); void f2fs_build_gc_manager(struct f2fs_sb_info *sbi); int f2fs_gc_range(struct f2fs_sb_info *sbi, unsigned int start_seg, unsigned int end_seg, - bool dry_run, unsigned int dry_run_sections); + bool dry_run, unsigned int dry_run_sections, bool lock); +void f2fs_reset_gc_victim_resource(struct f2fs_sb_info *sbi, + unsigned int start, unsigned int end); int f2fs_resize_fs(struct file *filp, __u64 block_count); int __init f2fs_create_garbage_collection_cache(void); void f2fs_destroy_garbage_collection_cache(void); diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 0897593..edc3525 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -36,23 +36,69 @@ #include #include -static void f2fs_zero_post_eof_page(struct inode *inode, - loff_t new_size, bool lock) +static int fill_zero(struct inode *inode, pgoff_t index, + loff_t start, loff_t len); + +static int do_zero_post_eof_page(struct inode *inode, loff_t new_size) { loff_t old_size = i_size_read(inode); + unsigned int offset, len; + pgoff_t index; + int err; + + offset = old_size & (PAGE_SIZE - 1); + + if (!offset) + return 0; + + len = min_t(loff_t, PAGE_SIZE - offset, new_size - old_size); + index = old_size >> PAGE_SHIFT; + + if (f2fs_has_inline_data(inode)) { + /* data post eof should be always zero */ + if (new_size <= MAX_INLINE_DATA(inode)) + return 0; + err = f2fs_convert_inline_inode(inode); + if (err) + return err; + } + + err = fill_zero(inode, index, offset, len); + if (err) + return err; + return filemap_write_and_wait_range(inode->i_mapping, + old_size, old_size + len - 1); +} + +static int f2fs_zero_post_eof_page(struct inode *inode, + loff_t new_size, bool lock, bool writeback) +{ + loff_t old_size = i_size_read(inode); + bool strict = + F2FS_OPTION(F2FS_I_SB(inode)).fsync_mode == FSYNC_MODE_STRICT; if (old_size >= new_size) - return; + return 0; - if (mapping_empty(inode->i_mapping)) - return; + if (!strict && mapping_empty(inode->i_mapping)) + return 0; if (lock) filemap_invalidate_lock(inode->i_mapping); /* zero or drop pages only in range of [old_size, new_size] */ - truncate_inode_pages_range(inode->i_mapping, old_size, new_size); + truncate_inode_pages_range(inode->i_mapping, old_size, new_size - 1); if (lock) filemap_invalidate_unlock(inode->i_mapping); + + if (!writeback || !strict) + return 0; + /* + * In fsync_mode=strict, when we expand an unaligned EOF size, we + * should zero post EOF data and writeback the data immediately, + * so that it can avoid exposing stale data after metadata flush + * and POR. + */ + return do_zero_post_eof_page(inode, new_size); } static vm_fault_t f2fs_filemap_fault(struct vm_fault *vmf) @@ -132,7 +178,10 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf) f2fs_bug_on(sbi, f2fs_has_inline_data(inode)); - f2fs_zero_post_eof_page(inode, (folio->index + 1) << PAGE_SHIFT, true); + err = f2fs_zero_post_eof_page(inode, + (folio->index + 1) << PAGE_SHIFT, true, false); + if (err) + goto out_pagefault; file_update_time(vmf->vma->vm_file); filemap_invalidate_lock_shared(inode->i_mapping); @@ -189,7 +238,7 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf) out_sem: filemap_invalidate_unlock_shared(inode->i_mapping); - +out_pagefault: sb_end_pagefault(inode->i_sb); out: ret = vmf_fs_error(err); @@ -813,13 +862,19 @@ int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock) if (IS_DEVICE_ALIASING(inode)) { struct extent_tree *et = F2FS_I(inode)->extent_tree[EX_READ]; - struct extent_info ei = et->largest; + struct extent_info ei; + + read_lock(&et->lock); + ei = et->largest; + read_unlock(&et->lock); f2fs_invalidate_blocks(sbi, ei.blk, ei.len); dec_valid_block_count(sbi, inode, ei.len); f2fs_update_time(sbi, REQ_TIME); + f2fs_drop_extent_tree(inode); + f2fs_folio_put(ifolio, true); goto out; } @@ -1096,24 +1151,31 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, if ((attr->ia_valid & ATTR_SIZE)) { if (mapping_large_folio_support(inode->i_mapping)) return -EOPNOTSUPP; - if (!f2fs_is_compress_backend_ready(inode) || - IS_DEVICE_ALIASING(inode)) + if (IS_DEVICE_ALIASING(inode)) + return -EPERM; + if (!f2fs_is_compress_backend_ready(inode)) return -EOPNOTSUPP; if (is_inode_flag_set(inode, FI_COMPRESS_RELEASED) && !IS_ALIGNED(attr->ia_size, F2FS_BLK_TO_BYTES(fi->i_cluster_size))) return -EINVAL; - /* - * To prevent scattered pin block generation, we don't allow - * smaller/equal size unaligned truncation for pinned file. - * We only support overwrite IO to pinned file, so don't - * care about larger size truncation. - */ - if (f2fs_is_pinned_file(inode) && - attr->ia_size <= i_size_read(inode) && - !IS_ALIGNED(attr->ia_size, - F2FS_BLK_TO_BYTES(CAP_BLKS_PER_SEC(sbi)))) - return -EINVAL; + + if (f2fs_is_pinned_file(inode)) { + /* + * It may break section-aligned fallocate recovery + * mechanism, so do not allow larger size truncation. + */ + if (attr->ia_size > i_size_read(inode)) + return -EINVAL; + /* + * To prevent scattered pin block generation, we don't + * allow smaller/equal size unaligned truncation for + * pinned file. + */ + else if (!IS_ALIGNED(attr->ia_size, + F2FS_BLK_TO_BYTES(CAP_BLKS_PER_SEC(sbi)))) + return -EINVAL; + } } if (is_quota_modification(idmap, inode, attr)) { @@ -1165,8 +1227,12 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, f2fs_down_write(&fi->i_gc_rwsem[WRITE]); filemap_invalidate_lock(inode->i_mapping); - if (attr->ia_size > old_size) - f2fs_zero_post_eof_page(inode, attr->ia_size, false); + if (attr->ia_size > old_size) { + err = f2fs_zero_post_eof_page(inode, + attr->ia_size, false, true); + if (err) + goto err_out; + } truncate_setsize(inode, attr->ia_size); if (attr->ia_size <= old_size) @@ -1175,6 +1241,7 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, * do not trim all blocks after i_size if target size is * larger than i_size. */ +err_out: filemap_invalidate_unlock(inode->i_mapping); f2fs_up_write(&fi->i_gc_rwsem[WRITE]); if (err) @@ -1286,7 +1353,9 @@ static int f2fs_punch_hole(struct inode *inode, loff_t offset, loff_t len) if (ret) return ret; - f2fs_zero_post_eof_page(inode, offset + len, true); + ret = f2fs_zero_post_eof_page(inode, offset + len, true, false); + if (ret) + return ret; pg_start = ((unsigned long long) offset) >> PAGE_SHIFT; pg_end = ((unsigned long long) offset + len) >> PAGE_SHIFT; @@ -1573,7 +1642,9 @@ static int f2fs_do_collapse(struct inode *inode, loff_t offset, loff_t len) f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); filemap_invalidate_lock(inode->i_mapping); - f2fs_zero_post_eof_page(inode, offset + len, false); + ret = f2fs_zero_post_eof_page(inode, offset + len, false, false); + if (ret) + goto out_unlock; f2fs_lock_op(sbi, &lc); f2fs_drop_extent_tree(inode); @@ -1581,6 +1652,7 @@ static int f2fs_do_collapse(struct inode *inode, loff_t offset, loff_t len) ret = __exchange_data_block(inode, inode, end, start, nrpages - end, true); f2fs_unlock_op(sbi, &lc); +out_unlock: filemap_invalidate_unlock(inode->i_mapping); f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); return ret; @@ -1613,11 +1685,14 @@ static int f2fs_collapse_range(struct inode *inode, loff_t offset, loff_t len) /* write out all moved pages, if possible */ filemap_invalidate_lock(inode->i_mapping); - filemap_write_and_wait_range(inode->i_mapping, offset, LLONG_MAX); + ret = filemap_write_and_wait_range(inode->i_mapping, offset, LLONG_MAX); + if (ret) + goto out_unlock; truncate_pagecache(inode, offset); new_size = i_size_read(inode) - len; ret = f2fs_truncate_blocks(inode, new_size, true); +out_unlock: filemap_invalidate_unlock(inode->i_mapping); if (!ret) f2fs_i_size_write(inode, new_size); @@ -1699,7 +1774,9 @@ static int f2fs_zero_range(struct inode *inode, loff_t offset, loff_t len, if (ret) return ret; - f2fs_zero_post_eof_page(inode, offset + len, true); + ret = f2fs_zero_post_eof_page(inode, offset + len, true, false); + if (ret) + return ret; pg_start = ((unsigned long long) offset) >> PAGE_SHIFT; pg_end = ((unsigned long long) offset + len) >> PAGE_SHIFT; @@ -1834,7 +1911,9 @@ static int f2fs_insert_range(struct inode *inode, loff_t offset, loff_t len) f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); filemap_invalidate_lock(mapping); - f2fs_zero_post_eof_page(inode, offset + len, false); + ret = f2fs_zero_post_eof_page(inode, offset + len, false, false); + if (ret) + goto out_unlock; truncate_pagecache(inode, offset); while (!ret && idx > pg_start) { @@ -1852,6 +1931,7 @@ static int f2fs_insert_range(struct inode *inode, loff_t offset, loff_t len) idx + delta, nr, false); f2fs_unlock_op(sbi, &lc); } +out_unlock: filemap_invalidate_unlock(mapping); f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); if (ret) @@ -1894,7 +1974,9 @@ static int f2fs_expand_inode_data(struct inode *inode, loff_t offset, if (err) return err; - f2fs_zero_post_eof_page(inode, offset + len, true); + err = f2fs_zero_post_eof_page(inode, offset + len, true, true); + if (err) + return err; f2fs_balance_fs(sbi, true); @@ -1915,8 +1997,9 @@ static int f2fs_expand_inode_data(struct inode *inode, loff_t offset, block_t sec_len; if (map.m_lblk % sec_blks) { - map.m_lblk = rounddown(map.m_lblk, sec_blks); - map.m_len = pg_end - map.m_lblk; + pg_start = rounddown(map.m_lblk, sec_blks); + map.m_lblk = pg_start; + map.m_len = pg_end - pg_start; if (off_end) map.m_len++; } @@ -2126,6 +2209,9 @@ static int f2fs_setflags_common(struct inode *inode, u32 iflags, u32 mask) if (IS_NOQUOTA(inode)) return -EPERM; + if (IS_DEVICE_ALIASING(inode)) + return -EPERM; + if ((iflags ^ masked_flags) & F2FS_CASEFOLD_FL) { if (!f2fs_sb_has_casefold(F2FS_I_SB(inode))) return -EOPNOTSUPP; @@ -2674,6 +2760,17 @@ static int f2fs_ioc_get_encryption_policy(struct file *filp, unsigned long arg) return fscrypt_ioctl_get_policy(filp, (void __user *)arg); } +static int f2fs_ioc_get_dev_alias_status(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + + if (!IS_DEVICE_ALIASING(inode)) + return -EINVAL; + + return put_user(F2FS_HAS_BLOCKS(inode) ? F2FS_DEV_ALIAS_STATUS_RESERVED : + F2FS_DEV_ALIAS_STATUS_RELEASED, (u32 __user *)arg); +} + static int f2fs_ioc_get_encryption_pwsalt(struct file *filp, unsigned long arg) { struct inode *inode = file_inode(filp); @@ -3079,6 +3176,9 @@ static int f2fs_ioc_defragment(struct file *filp, unsigned long arg) if (f2fs_readonly(sbi->sb)) return -EROFS; + if (IS_DEVICE_ALIASING(inode)) + return -EOPNOTSUPP; + if (copy_from_user(&range, (struct f2fs_defragment __user *)arg, sizeof(range))) return -EFAULT; @@ -3117,8 +3217,9 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, struct inode *dst = file_inode(file_out); struct f2fs_sb_info *sbi = F2FS_I_SB(src); struct f2fs_lock_context lc; - size_t olen = len, dst_max_i_size = 0; - size_t dst_osize; + size_t olen = len; + loff_t dst_max_i_size = 0; + loff_t dst_osize, dst_end; int ret; if (file_in->f_path.mnt != file_out->f_path.mnt || @@ -3131,7 +3232,8 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, if (!S_ISREG(src->i_mode) || !S_ISREG(dst->i_mode)) return -EINVAL; - if (IS_ENCRYPTED(src) || IS_ENCRYPTED(dst)) + if (IS_ENCRYPTED(src) || IS_ENCRYPTED(dst) || + IS_DEVICE_ALIASING(src) || IS_DEVICE_ALIASING(dst)) return -EOPNOTSUPP; if (pos_out < 0 || pos_in < 0) @@ -3140,8 +3242,6 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, if (src == dst) { if (pos_in == pos_out) return 0; - if (pos_out > pos_in && pos_out < pos_in + len) - return -EINVAL; } inode_lock(src); @@ -3167,6 +3267,8 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, goto out_unlock; if (len == 0) olen = len = src->i_size - pos_in; + if (src == dst && pos_out > pos_in && pos_out < pos_in + len) + goto out_unlock; if (pos_in + len == src->i_size) len = ALIGN(src->i_size, F2FS_BLKSIZE) - pos_in; if (len == 0) { @@ -3175,8 +3277,15 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, } dst_osize = dst->i_size; - if (pos_out + olen > dst->i_size) - dst_max_i_size = pos_out + olen; + if (olen > LLONG_MAX - pos_out) + goto out_unlock; + dst_end = pos_out + olen; + if (dst_end > dst->i_size) { + ret = inode_newsize_ok(dst, dst_end); + if (ret) + goto out_unlock; + dst_max_i_size = dst_end; + } /* verify the end result is block aligned */ if (!IS_ALIGNED(pos_in, F2FS_BLKSIZE) || @@ -3612,6 +3721,241 @@ static int f2fs_ioc_get_dev_alias_file(struct file *filp, unsigned long arg) (u32 __user *)arg); } +static bool f2fs_get_dev_alias_extent(struct f2fs_sb_info *sbi, + struct dentry *dentry, + struct extent_info *ei) +{ + int i; + + for (i = 1; i < sbi->s_ndevs; i++) { + char *name = strrchr(FDEV(i).path, '/'); + + name = name ? name + 1 : FDEV(i).path; + if (strcmp(name, dentry->d_name.name)) + continue; + + ei->blk = FDEV(i).start_blk; + ei->len = FDEV(i).total_segments << sbi->log_blocks_per_seg; + ei->fofs = 0; + return true; + } + return false; +} + +static int f2fs_ioc_reserve_dev_alias(struct file *filp) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct extent_tree *et = F2FS_I(inode)->extent_tree[EX_READ]; + struct extent_info ei; + struct cp_control cpc = { CP_SYNC, 0, 0, 0 }; + struct f2fs_lock_context lc, glc; + blkcnt_t count; + unsigned int start, end; + int type, err; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) + return -EINVAL; + + err = mnt_want_write_file(filp); + if (err) + return err; + + inode_lock(inode); + + if (!IS_DEVICE_ALIASING(inode)) { + err = -EINVAL; + goto out_inode_unlock; + } + + if (F2FS_HAS_BLOCKS(inode)) { + err = 0; + goto out_inode_unlock; + } + + if (!f2fs_get_dev_alias_extent(sbi, filp->f_path.dentry, &ei)) { + f2fs_warn(sbi, "device alias file (%s, ino=%llu) has no matching device", + filp->f_path.dentry->d_name.name, + (unsigned long long)inode->i_ino); + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); + err = -EFSCORRUPTED; + goto out_inode_unlock; + } + + spin_lock(&sbi->stat_lock); + if (sbi->total_valid_block_count + ei.len > + get_available_block_count(sbi, inode, true)) { + spin_unlock(&sbi->stat_lock); + err = -ENOSPC; + goto out_inode_unlock; + } + sbi->alias_reserved_blocks += ei.len; + spin_unlock(&sbi->stat_lock); + + spin_lock(&FREE_I(sbi)->segmap_lock); + FDEV(f2fs_target_device_index(sbi, ei.blk)).is_reserving = true; + spin_unlock(&FREE_I(sbi)->segmap_lock); + + start = GET_SEGNO(sbi, ei.blk); + end = GET_SEGNO(sbi, ei.blk + ei.len - 1); + + /* Acquire gc_lock for victim reset, curseg resize, and range GC */ + f2fs_down_write_trace(&sbi->gc_lock, &glc); + + /* Reset the victim information to prevent GC from targeting the range */ + f2fs_reset_gc_victim_resource(sbi, start, end); + + /* Move out cursegs from the target range */ + for (type = CURSEG_HOT_DATA; type < NR_CURSEG_PERSIST_TYPE; type++) { + err = f2fs_allocate_segment_for_resize(sbi, type, start, end); + if (err) + goto out_gc_unlock; + } + + f2fs_lock_op(sbi, &lc); + + if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { + err = -EINVAL; + f2fs_unlock_op(sbi, &lc); + goto out_gc_unlock; + } + + /* do GC to move out valid blocks in the range all at once! */ + err = f2fs_gc_range(sbi, start, end, false, 0, false); + if (err) { + f2fs_unlock_op(sbi, &lc); + goto out_gc_unlock; + } + + count = ei.len; + err = inc_valid_block_count(sbi, inode, &count, false, true); + if (err) { + f2fs_unlock_op(sbi, &lc); + goto out_gc_unlock; + } + + write_lock(&et->lock); + et->largest = ei; + write_unlock(&et->lock); + clear_inode_flag(inode, FI_NO_EXTENT); + + f2fs_reserve_device_alias(sbi, ei.blk, ei.len); + + i_size_write(inode, (loff_t)ei.len << sbi->log_blocksize); + f2fs_update_inode_page(inode); + + spin_lock(&FREE_I(sbi)->segmap_lock); + FDEV(f2fs_target_device_index(sbi, ei.blk)).is_reserving = false; + spin_unlock(&FREE_I(sbi)->segmap_lock); + + f2fs_unlock_op(sbi, &lc); + f2fs_up_write_trace(&sbi->gc_lock, &glc); + + inode_unlock(inode); + mnt_drop_write_file(filp); + + return f2fs_write_checkpoint(sbi, &cpc); + +out_gc_unlock: + spin_lock(&sbi->stat_lock); + sbi->alias_reserved_blocks -= ei.len; + spin_unlock(&sbi->stat_lock); + + spin_lock(&FREE_I(sbi)->segmap_lock); + FDEV(f2fs_target_device_index(sbi, ei.blk)).is_reserving = false; + spin_unlock(&FREE_I(sbi)->segmap_lock); + f2fs_up_write_trace(&sbi->gc_lock, &glc); + +out_inode_unlock: + inode_unlock(inode); + mnt_drop_write_file(filp); + return err; +} + +static int f2fs_ioc_release_dev_alias(struct file *filp) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct extent_tree *et = F2FS_I(inode)->extent_tree[EX_READ]; + struct extent_info ei = {0, }; + struct cp_control cpc = { CP_SYNC, 0, 0, 0 }; + struct f2fs_lock_context lc, glc; + int err; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) + return -EINVAL; + + err = mnt_want_write_file(filp); + if (err) + return err; + + inode_lock(inode); + + if (!IS_DEVICE_ALIASING(inode)) { + err = -EINVAL; + goto out_inode_unlock; + } + + if (!F2FS_HAS_BLOCKS(inode)) { + err = 0; + goto out_inode_unlock; + } + + err = filemap_write_and_wait(inode->i_mapping); + if (err) + goto out_inode_unlock; + + read_lock(&et->lock); + ei = et->largest; + read_unlock(&et->lock); + + f2fs_down_write_trace(&sbi->gc_lock, &glc); + f2fs_lock_op(sbi, &lc); + + if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { + err = -EINVAL; + f2fs_unlock_op(sbi, &lc); + f2fs_up_write_trace(&sbi->gc_lock, &glc); + goto out_inode_unlock; + } + + filemap_invalidate_lock(inode->i_mapping); + truncate_setsize(inode, 0); + + err = f2fs_truncate_blocks(inode, 0, false); + if (err) + i_size_write(inode, (loff_t)ei.len << sbi->log_blocksize); + filemap_invalidate_unlock(inode->i_mapping); + + if (err) { + f2fs_unlock_op(sbi, &lc); + f2fs_up_write_trace(&sbi->gc_lock, &glc); + goto out_inode_unlock; + } + + f2fs_update_inode_page(inode); + + f2fs_unlock_op(sbi, &lc); + f2fs_up_write_trace(&sbi->gc_lock, &glc); + + inode_unlock(inode); + mnt_drop_write_file(filp); + + return f2fs_write_checkpoint(sbi, &cpc); + +out_inode_unlock: + inode_unlock(inode); + mnt_drop_write_file(filp); + return err; +} + static int f2fs_ioc_io_prio(struct file *filp, unsigned long arg) { struct inode *inode = file_inode(filp); @@ -4037,7 +4381,7 @@ static int reserve_compress_blocks(struct dnode_of_data *dn, pgoff_t count, } ret = inc_valid_block_count(sbi, dn->inode, - &to_reserved, false); + &to_reserved, false, false); if (unlikely(ret)) return ret; @@ -4469,7 +4813,7 @@ static int redirty_blocks(struct inode *inode, pgoff_t page_idx, int len) page_idx = folio_next_index(folio); } while (page_len < len); - do { + while (redirty_idx < page_idx) { folio = filemap_lock_folio(mapping, redirty_idx); /* It will never fail, when folio has pinned above */ @@ -4482,7 +4826,7 @@ static int redirty_blocks(struct inode *inode, pgoff_t page_idx, int len) redirty_idx = folio_next_index(folio); folio_unlock(folio); folio_put_refs(folio, 2); - } while (redirty_idx < page_idx); + } return ret; } @@ -4738,8 +5082,14 @@ static long __f2fs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) return f2fs_ioc_compress_file(filp); case F2FS_IOC_GET_DEV_ALIAS_FILE: return f2fs_ioc_get_dev_alias_file(filp, arg); + case F2FS_IOC_GET_DEV_ALIAS_STATUS: + return f2fs_ioc_get_dev_alias_status(filp, arg); case F2FS_IOC_IO_PRIO: return f2fs_ioc_io_prio(filp, arg); + case F2FS_IOC_RESERVE_DEV_ALIAS: + return f2fs_ioc_reserve_dev_alias(filp); + case F2FS_IOC_RELEASE_DEV_ALIAS: + return f2fs_ioc_release_dev_alias(filp); default: return -ENOTTY; } @@ -4998,8 +5348,10 @@ static ssize_t f2fs_write_checks(struct kiocb *iocb, struct iov_iter *from) if (err) return err; - f2fs_zero_post_eof_page(inode, - iocb->ki_pos + iov_iter_count(from), true); + err = f2fs_zero_post_eof_page(inode, + iocb->ki_pos + iov_iter_count(from), true, true); + if (err) + return err; return count; } @@ -5285,9 +5637,8 @@ static ssize_t f2fs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) bool dio; bool may_need_sync = true; int preallocated; - const loff_t pos = iocb->ki_pos; - const ssize_t count = iov_iter_count(from); ssize_t ret; + loff_t bufio_start_pos; if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) { ret = -EIO; @@ -5308,15 +5659,17 @@ static ssize_t f2fs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) inode_lock(inode); } + ret = f2fs_write_checks(iocb, from); + if (ret <= 0) + goto out_unlock; + if (f2fs_is_pinned_file(inode) && - !f2fs_overwrite_io(inode, pos, count)) { + !f2fs_overwrite_io(inode, iocb->ki_pos, iov_iter_count(from))) { ret = -EIO; goto out_unlock; } - ret = f2fs_write_checks(iocb, from); - if (ret <= 0) - goto out_unlock; + bufio_start_pos = iocb->ki_pos; /* Determine whether we will do a direct write or a buffered write. */ dio = f2fs_should_use_dio(inode, iocb, from); @@ -5371,8 +5724,8 @@ out: */ if (ret > 0 && !dio && (iocb->ki_flags & IOCB_DIRECT)) f2fs_flush_buffered_write(iocb->ki_filp->f_mapping, - orig_pos, - orig_pos + ret - 1); + bufio_start_pos, + bufio_start_pos + ret - 1); return ret; } @@ -5526,7 +5879,10 @@ long f2fs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) case F2FS_IOC_DECOMPRESS_FILE: case F2FS_IOC_COMPRESS_FILE: case F2FS_IOC_GET_DEV_ALIAS_FILE: + case F2FS_IOC_GET_DEV_ALIAS_STATUS: case F2FS_IOC_IO_PRIO: + case F2FS_IOC_RESERVE_DEV_ALIAS: + case F2FS_IOC_RELEASE_DEV_ALIAS: break; default: return -ENOIOCTLCMD; diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index ffaa7ba..0a00180 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -31,9 +31,9 @@ static unsigned int count_bits(const unsigned long *addr, static int gc_thread_func(void *data) { struct f2fs_sb_info *sbi = data; - struct f2fs_gc_kthread *gc_th = sbi->gc_thread; - wait_queue_head_t *wq = &sbi->gc_thread->gc_wait_queue_head; - wait_queue_head_t *fggc_wq = &sbi->gc_thread->fggc_wq; + struct f2fs_gc_kthread *gc_th = &sbi->gc_thread; + wait_queue_head_t *wq = &sbi->gc_thread.gc_wait_queue_head; + wait_queue_head_t *fggc_wq = &sbi->gc_thread.fggc_wq; unsigned int wait_ms; struct f2fs_gc_control gc_control = { .victim_segno = NULL_SEGNO, @@ -193,13 +193,9 @@ next: int f2fs_start_gc_thread(struct f2fs_sb_info *sbi) { - struct f2fs_gc_kthread *gc_th; + struct f2fs_gc_kthread *gc_th = &sbi->gc_thread; dev_t dev = sbi->sb->s_bdev->bd_dev; - gc_th = f2fs_kmalloc(sbi, sizeof(struct f2fs_gc_kthread), GFP_KERNEL); - if (!gc_th) - return -ENOMEM; - gc_th->urgent_sleep_time = DEF_GC_THREAD_URGENT_SLEEP_TIME; gc_th->valid_thresh_ratio = DEF_GC_THREAD_VALID_THRESH_RATIO; gc_th->boost_gc_multiple = BOOST_GC_MULTIPLE; @@ -221,16 +217,14 @@ int f2fs_start_gc_thread(struct f2fs_sb_info *sbi) gc_th->gc_wake = false; - sbi->gc_thread = gc_th; - init_waitqueue_head(&sbi->gc_thread->gc_wait_queue_head); - init_waitqueue_head(&sbi->gc_thread->fggc_wq); - sbi->gc_thread->f2fs_gc_task = kthread_run(gc_thread_func, sbi, + init_waitqueue_head(&gc_th->gc_wait_queue_head); + init_waitqueue_head(&gc_th->fggc_wq); + gc_th->f2fs_gc_task = kthread_run(gc_thread_func, sbi, "f2fs_gc-%u:%u", MAJOR(dev), MINOR(dev)); if (IS_ERR(gc_th->f2fs_gc_task)) { int err = PTR_ERR(gc_th->f2fs_gc_task); - kfree(gc_th); - sbi->gc_thread = NULL; + gc_th->f2fs_gc_task = NULL; return err; } @@ -241,14 +235,14 @@ int f2fs_start_gc_thread(struct f2fs_sb_info *sbi) void f2fs_stop_gc_thread(struct f2fs_sb_info *sbi) { - struct f2fs_gc_kthread *gc_th = sbi->gc_thread; + struct f2fs_gc_kthread *gc_th = &sbi->gc_thread; - if (!gc_th) + if (!gc_th->f2fs_gc_task) return; + kthread_stop(gc_th->f2fs_gc_task); + gc_th->f2fs_gc_task = NULL; wake_up_all(&gc_th->fggc_wq); - kfree(gc_th); - sbi->gc_thread = NULL; } static int select_gc_type(struct f2fs_sb_info *sbi, int gc_type) @@ -316,7 +310,7 @@ static void select_policy(struct f2fs_sb_info *sbi, int gc_type, p->max_search = sbi->max_victim_search; /* let's select beginning hot/small space first. */ - if (f2fs_need_rand_seg(sbi)) { + if (f2fs_need_rand_seg_blk(sbi, type)) { p->offset = get_random_u32_below(MAIN_SECS(sbi) * SEGS_PER_SEC(sbi)); SIT_I(sbi)->last_victim[p->gc_mode] = p->offset; @@ -796,7 +790,7 @@ int f2fs_get_victim(struct f2fs_sb_info *sbi, unsigned int *result, if (one_time) { p.one_time_gc = one_time; if (has_enough_free_secs(sbi, 0, NR_PERSISTENT_LOG)) - valid_thresh_ratio = sbi->gc_thread->valid_thresh_ratio; + valid_thresh_ratio = sbi->gc_thread.valid_thresh_ratio; } retry: @@ -1807,9 +1801,9 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, if (f2fs_sb_has_blkzoned(sbi) && !has_enough_free_blocks(sbi, - sbi->gc_thread->boost_zoned_gc_percent)) + sbi->gc_thread.boost_zoned_gc_percent)) window_granularity *= - sbi->gc_thread->boost_gc_multiple; + sbi->gc_thread.boost_gc_multiple; end_segno = start_segno + window_granularity; } @@ -2156,8 +2150,9 @@ void f2fs_build_gc_manager(struct f2fs_sb_info *sbi) int f2fs_gc_range(struct f2fs_sb_info *sbi, unsigned int start_seg, unsigned int end_seg, - bool dry_run, unsigned int dry_run_sections) + bool dry_run, unsigned int dry_run_sections, bool lock) { + struct f2fs_lock_context lc; unsigned int segno; unsigned int gc_secs = dry_run_sections; @@ -2170,66 +2165,102 @@ int f2fs_gc_range(struct f2fs_sb_info *sbi, .ilist = LIST_HEAD_INIT(gc_list.ilist), .iroot = RADIX_TREE_INIT(gc_list.iroot, GFP_NOFS), }; + int err = 0; + + if (lock) + f2fs_down_write_trace(&sbi->gc_lock, &lc); /* * avoid migrating empty section, as it can be allocated by * log in parallel. */ if (!get_valid_blocks(sbi, segno, true)) - continue; + goto next; if (is_cursec(sbi, GET_SEC_FROM_SEG(sbi, segno))) - continue; + goto next; do_garbage_collect(sbi, segno, &gc_list, FG_GC, true, false); put_gc_inode(&gc_list); - if (!dry_run && get_valid_blocks(sbi, segno, true)) - return -EAGAIN; + /* reset all pinned status during fggc */ + f2fs_unpin_all_sections(sbi, true); + + if (!dry_run && get_valid_blocks(sbi, segno, true)) { + err = -EAGAIN; + goto next; + } if (dry_run && dry_run_sections && - !get_valid_blocks(sbi, segno, true) && --gc_secs == 0) - break; + !get_valid_blocks(sbi, segno, true)) { + --gc_secs; + goto next; + } if (fatal_signal_pending(current)) - return -ERESTARTSYS; + err = -ERESTARTSYS; +next: + if (lock) + f2fs_up_write_trace(&sbi->gc_lock, &lc); + if (err) + return err; + if (dry_run && dry_run_sections && !gc_secs) + return 0; } return 0; } +void f2fs_reset_gc_victim_resource(struct f2fs_sb_info *sbi, + unsigned int start, unsigned int end) +{ + int i; + + mutex_lock(&DIRTY_I(sbi)->seglist_lock); + for (i = 0; i < MAX_GC_POLICY; i++) + if (SIT_I(sbi)->last_victim[i] >= start && + SIT_I(sbi)->last_victim[i] <= end) + SIT_I(sbi)->last_victim[i] = 0; + + for (i = BG_GC; i <= FG_GC; i++) + if (sbi->next_victim_seg[i] >= start && + sbi->next_victim_seg[i] <= end) + sbi->next_victim_seg[i] = NULL_SEGNO; + mutex_unlock(&DIRTY_I(sbi)->seglist_lock); +} + static int free_segment_range(struct f2fs_sb_info *sbi, unsigned int secs, bool dry_run) { - unsigned int next_inuse, start, end; + unsigned int secno, next_inuse, start, end, end_secno; struct cp_control cpc = { CP_RESIZE, 0, 0, 0 }; - int gc_mode, gc_type; + unsigned int freed_secs = 0; int err = 0; int type; - /* Force block allocation for GC */ MAIN_SECS(sbi) -= secs; start = MAIN_SECS(sbi) * SEGS_PER_SEC(sbi); end = MAIN_SEGS(sbi) - 1; + end_secno = GET_SEC_FROM_SEG(sbi, end); - mutex_lock(&DIRTY_I(sbi)->seglist_lock); - for (gc_mode = 0; gc_mode < MAX_GC_POLICY; gc_mode++) - if (SIT_I(sbi)->last_victim[gc_mode] >= start) - SIT_I(sbi)->last_victim[gc_mode] = 0; + f2fs_reset_gc_victim_resource(sbi, start, end); - for (gc_type = BG_GC; gc_type <= FG_GC; gc_type++) - if (sbi->next_victim_seg[gc_type] >= start) - sbi->next_victim_seg[gc_type] = NULL_SEGNO; - mutex_unlock(&DIRTY_I(sbi)->seglist_lock); + spin_lock(&FREE_I(sbi)->segmap_lock); + for (secno = MAIN_SECS(sbi); secno <= end_secno; secno++) { + if (!test_bit(secno, FREE_I(sbi)->free_secmap)) + freed_secs++; + } + FREE_I(sbi)->free_sections -= freed_secs; + spin_unlock(&FREE_I(sbi)->segmap_lock); /* Move out cursegs from the target range */ - for (type = CURSEG_HOT_DATA; type < NR_CURSEG_PERSIST_TYPE; type++) { + for (type = CURSEG_HOT_DATA; type < NR_CURSEG_TYPE; type++) { err = f2fs_allocate_segment_for_resize(sbi, type, start, end); if (err) goto out; } /* do GC to move out valid blocks in the range */ - err = f2fs_gc_range(sbi, start, end, dry_run, 0); + err = f2fs_gc_range(sbi, start, end, dry_run, 0, false); if (err || dry_run) goto out; @@ -2245,6 +2276,9 @@ static int free_segment_range(struct f2fs_sb_info *sbi, f2fs_bug_on(sbi, 1); } out: + spin_lock(&FREE_I(sbi)->segmap_lock); + FREE_I(sbi)->free_sections += freed_secs; + spin_unlock(&FREE_I(sbi)->segmap_lock); MAIN_SECS(sbi) += secs; return err; } @@ -2422,7 +2456,7 @@ out_drop_write: set_sbi_flag(sbi, SBI_IS_RESIZEFS); err = free_segment_range(sbi, secs, false); if (err) - goto recover_out; + goto recover_user_blocks; update_sb_metadata(sbi, -secs); @@ -2444,11 +2478,14 @@ out_drop_write: f2fs_commit_super(sbi, false); } recover_out: - clear_sbi_flag(sbi, SBI_IS_RESIZEFS); if (err) { + f2fs_bug_on(sbi, err == -EAGAIN); set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_err(sbi, "resize_fs failed, should run fsck to repair!"); - + } +recover_user_blocks: + clear_sbi_flag(sbi, SBI_IS_RESIZEFS); + if (err) { spin_lock(&sbi->stat_lock); sbi->user_block_count += shrunk_blocks; spin_unlock(&sbi->stat_lock); diff --git a/fs/f2fs/gc.h b/fs/f2fs/gc.h index 6c4d456..b015742 100644 --- a/fs/f2fs/gc.h +++ b/fs/f2fs/gc.h @@ -45,32 +45,7 @@ #define NR_GC_CHECKPOINT_SECS (3) /* data/node/dentry sections */ -struct f2fs_gc_kthread { - struct task_struct *f2fs_gc_task; - wait_queue_head_t gc_wait_queue_head; - - /* for gc sleep time */ - unsigned int urgent_sleep_time; - unsigned int min_sleep_time; - unsigned int max_sleep_time; - unsigned int no_gc_sleep_time; - - /* for changing gc mode */ - bool gc_wake; - - /* for GC_MERGE mount option */ - wait_queue_head_t fggc_wq; /* - * caller of f2fs_balance_fs() - * will wait on this wait queue. - */ - /* for gc control for zoned devices */ - unsigned int no_zoned_gc_percent; - unsigned int boost_zoned_gc_percent; - unsigned int valid_thresh_ratio; - unsigned int boost_gc_multiple; - unsigned int boost_gc_greedy; -}; struct gc_inode_list { struct list_head ilist; @@ -197,6 +172,6 @@ static inline bool need_to_boost_gc(struct f2fs_sb_info *sbi) { if (f2fs_sb_has_blkzoned(sbi)) return !has_enough_free_blocks(sbi, - sbi->gc_thread->boost_zoned_gc_percent); + sbi->gc_thread.boost_zoned_gc_percent); return has_enough_invalid_blocks(sbi); } diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c index e2f7bed..aec06fb 100644 --- a/fs/f2fs/inline.c +++ b/fs/f2fs/inline.c @@ -732,7 +732,7 @@ void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry, f2fs_folio_put(folio, true); inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); - f2fs_mark_inode_dirty_sync(dir, false); + f2fs_mark_inode_dirty_sync(dir, true); if (inode) f2fs_drop_nlink(dir, inode); diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c index c95e0b1..96cc0e7 100644 --- a/fs/f2fs/inode.c +++ b/fs/f2fs/inode.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,18 @@ extern const struct address_space_operations f2fs_compress_aops; #endif +#define NUM_PREALLOC_EVICT_INODE_WORK 8 + +static struct kmem_cache *evict_inode_work_cache; +static mempool_t *evict_inode_work_pool; + +struct evict_inode_work { + struct work_struct work; + struct f2fs_sb_info *sbi; + nid_t ino; + unsigned int add_ino_entry_bits; +}; + void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync) { if (is_inode_flag_set(inode, FI_NEW_INODE)) @@ -637,6 +650,9 @@ make_now: inode->i_fop = &f2fs_dir_operations; inode->i_mapping->a_ops = &f2fs_dblock_aops; mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS); + + /* Let's prepare APPEND/UPDATE_INO before future access. */ + flush_workqueue(sbi->evict_wq); } else if (S_ISLNK(inode->i_mode)) { if (file_is_encrypt(inode)) inode->i_op = &f2fs_encrypted_symlink_inode_operations; @@ -854,16 +870,32 @@ void f2fs_remove_donate_inode(struct inode *inode) spin_unlock(&sbi->inode_lock[DONATE_INODE]); } +static void f2fs_record_inode_state(struct f2fs_sb_info *sbi, nid_t ino, + unsigned int bits) +{ + if (bits & BIT(APPEND_INO)) + f2fs_add_ino_entry(sbi, ino, APPEND_INO); + if (bits & BIT(UPDATE_INO)) + f2fs_add_ino_entry(sbi, ino, UPDATE_INO); +} + +static void f2fs_evict_inode_work(struct work_struct *work) +{ + struct evict_inode_work *ew = + container_of(work, struct evict_inode_work, work); + + f2fs_record_inode_state(ew->sbi, ew->ino, ew->add_ino_entry_bits); + + mempool_free(ew, evict_inode_work_pool); +} + /* - * Called at the last iput() if i_nlink is zero + * Return true, if we shouldn't go through post_evict_inode. */ -void f2fs_evict_inode(struct inode *inode) +static bool f2fs_pre_evict_inode(struct inode *inode) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_inode_info *fi = F2FS_I(inode); - nid_t xnid = fi->i_xattr_nid; - int err = 0; - bool freeze_protected = false; f2fs_abort_atomic_write(inode, true); @@ -883,13 +915,13 @@ void f2fs_evict_inode(struct inode *inode) truncate_inode_pages_final(&inode->i_data); if ((inode->i_nlink || is_bad_inode(inode)) && - test_opt(sbi, COMPRESS_CACHE) && f2fs_compressed_file(inode)) + test_opt(sbi, COMPRESS_CACHE) && f2fs_compressed_file(inode)) f2fs_invalidate_compress_pages(sbi, inode->i_ino); if (inode->i_ino == F2FS_NODE_INO(sbi) || - inode->i_ino == F2FS_META_INO(sbi) || - inode->i_ino == F2FS_COMPRESS_INO(sbi)) - goto out_clear; + inode->i_ino == F2FS_META_INO(sbi) || + inode->i_ino == F2FS_COMPRESS_INO(sbi)) + return true; f2fs_bug_on(sbi, get_dirty_pages(inode)); f2fs_remove_dirty_inode(inode); @@ -898,14 +930,18 @@ void f2fs_evict_inode(struct inode *inode) if (!IS_DEVICE_ALIASING(inode)) f2fs_destroy_extent_tree(inode); - if (inode->i_nlink || is_bad_inode(inode)) - goto no_delete; + return false; +} - err = f2fs_dquot_initialize(inode); - if (err) { - err = 0; +static void f2fs_delete_inode(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + bool freeze_protected = false; + struct f2fs_lock_context lc; + int err = 0; + + if (f2fs_dquot_initialize(inode)) set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR); - } f2fs_remove_ino_entry(sbi, inode->i_ino, APPEND_INO); f2fs_remove_ino_entry(sbi, inode->i_ino, UPDATE_INO); @@ -924,30 +960,30 @@ retry: if (time_to_inject(sbi, FAULT_EVICT_INODE)) err = -EIO; - if (!err) { - struct f2fs_lock_context lc; - - f2fs_lock_op(sbi, &lc); - err = f2fs_remove_inode_page(inode); - f2fs_unlock_op(sbi, &lc); - if (err == -ENOENT) { - err = 0; - - /* - * in fuzzed image, another node may has the same - * block address as inode's, if it was truncated - * previously, truncation of inode node will fail. - */ - if (is_inode_flag_set(inode, FI_DIRTY_INODE)) { - f2fs_warn(F2FS_I_SB(inode), - "f2fs_evict_inode: inconsistent node id, ino:%llu", - inode->i_ino); - f2fs_inode_synced(inode); - set_sbi_flag(sbi, SBI_NEED_FSCK); - } + if (err) + goto error_check; + + f2fs_lock_op(sbi, &lc); + err = f2fs_remove_inode_page(inode); + f2fs_unlock_op(sbi, &lc); + + if (err == -ENOENT) { + err = 0; + + /* + * in fuzzed image, another node may has the same + * block address as inode's, if it was truncated + * previously, truncation of inode node will fail. + */ + if (is_inode_flag_set(inode, FI_DIRTY_INODE)) { + f2fs_warn(F2FS_I_SB(inode), + "f2fs_evict_inode: inconsistent node id, ino:%llu", + inode->i_ino); + f2fs_inode_synced(inode); + set_sbi_flag(sbi, SBI_NEED_FSCK); } } - +error_check: /* give more chances, if ENOMEM case */ if (err == -ENOMEM) { err = 0; @@ -957,27 +993,38 @@ retry: if (IS_DEVICE_ALIASING(inode)) f2fs_destroy_extent_tree(inode); - if (err) { - f2fs_update_inode_page(inode); - if (dquot_initialize_needed(inode)) - set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR); + if (!err) + goto unfreeze_out; - /* - * If both f2fs_truncate() and f2fs_update_inode_page() failed - * due to fuzzed corrupted inode, call f2fs_inode_synced() to - * avoid triggering later f2fs_bug_on(). - */ - if (is_inode_flag_set(inode, FI_DIRTY_INODE)) { - f2fs_warn(sbi, - "f2fs_evict_inode: inode is dirty, ino:%llu", - inode->i_ino); - f2fs_inode_synced(inode); - set_sbi_flag(sbi, SBI_NEED_FSCK); - } + f2fs_update_inode_page(inode); + + if (dquot_initialize_needed(inode)) + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR); + + /* + * If both f2fs_truncate() and f2fs_update_inode_page() failed + * due to fuzzed corrupted inode, call f2fs_inode_synced() to + * avoid triggering later f2fs_bug_on(). + */ + if (is_inode_flag_set(inode, FI_DIRTY_INODE)) { + f2fs_warn(sbi, + "f2fs_evict_inode: inode is dirty, ino:%llu", + inode->i_ino); + f2fs_inode_synced(inode); + set_sbi_flag(sbi, SBI_NEED_FSCK); } +unfreeze_out: if (freeze_protected) sb_end_intwrite(inode->i_sb); -no_delete: +} + +static void f2fs_post_evict_inode(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_inode_info *fi = F2FS_I(inode); + nid_t xnid = fi->i_xattr_nid; + unsigned int record_bits = 0; + dquot_drop(inode); stat_dec_inline_xattr(inode); @@ -1003,12 +1050,32 @@ no_delete: inode->i_ino); if (xnid) invalidate_mapping_pages(NODE_MAPPING(sbi), xnid, xnid); - if (inode->i_nlink) { - if (is_inode_flag_set(inode, FI_APPEND_WRITE)) - f2fs_add_ino_entry(sbi, inode->i_ino, APPEND_INO); - if (is_inode_flag_set(inode, FI_UPDATE_WRITE)) - f2fs_add_ino_entry(sbi, inode->i_ino, UPDATE_INO); + + if (!inode->i_nlink) + goto skip_record; + + if (is_inode_flag_set(inode, FI_APPEND_WRITE)) + record_bits = BIT(APPEND_INO); + if (is_inode_flag_set(inode, FI_UPDATE_WRITE)) + record_bits = BIT(UPDATE_INO); + + if (!record_bits) + goto skip_record; + + /* Let's do this in workqueue out of the direct reclaim path. */ + if (current_is_kswapd()) { + f2fs_record_inode_state(sbi, inode->i_ino, record_bits); + } else { + struct evict_inode_work *ew = + mempool_alloc(evict_inode_work_pool, GFP_NOFS); + + ew->sbi = sbi; + ew->ino = inode->i_ino; + ew->add_ino_entry_bits = record_bits; + INIT_WORK(&ew->work, f2fs_evict_inode_work); + queue_work(sbi->evict_wq, &ew->work); } +skip_record: if (is_inode_flag_set(inode, FI_FREE_NID)) { f2fs_alloc_nid_failed(sbi, inode->i_ino); clear_inode_flag(inode, FI_FREE_NID); @@ -1019,13 +1086,29 @@ no_delete: * In that case, f2fs_check_nid_range() is enough to give a clue. */ } -out_clear: +} + +/* + * Called at the last iput() if i_nlink is zero + */ +void f2fs_evict_inode(struct inode *inode) +{ + if (f2fs_pre_evict_inode(inode)) + goto clear_out; + + if (!inode->i_nlink && !is_bad_inode(inode)) + f2fs_delete_inode(inode); + + f2fs_post_evict_inode(inode); + +clear_out: fscrypt_put_encryption_info(inode); clear_inode(inode); } /* caller should call f2fs_lock_op() */ -void f2fs_handle_failed_inode(struct inode *inode, struct f2fs_lock_context *lc) +void f2fs_handle_failed_inode(struct inode *inode, + struct f2fs_lock_context *lc, bool orphan_free) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct node_info ni; @@ -1047,6 +1130,9 @@ void f2fs_handle_failed_inode(struct inode *inode, struct f2fs_lock_context *lc) /* don't make bad inode, since it becomes a regular file. */ unlock_new_inode(inode); + if (!orphan_free) + goto out; + /* * Note: we should add inode to orphan list before f2fs_unlock_op() * so we can prevent losing this orphan when encoutering checkpoint @@ -1079,3 +1165,29 @@ out: /* iput will drop the inode object */ iput(inode); } + +int __init f2fs_init_evict_inode_work(void) +{ + evict_inode_work_cache = + kmem_cache_create("f2fs_evict_inode_work", + sizeof(struct evict_inode_work), 0, 0, NULL); + if (!evict_inode_work_cache) + goto fail; + evict_inode_work_pool = + mempool_create_slab_pool(NUM_PREALLOC_EVICT_INODE_WORK, + evict_inode_work_cache); + if (!evict_inode_work_pool) + goto fail_free_cache; + return 0; + +fail_free_cache: + kmem_cache_destroy(evict_inode_work_cache); +fail: + return -ENOMEM; +} + +void f2fs_destroy_evict_inode_work(void) +{ + mempool_destroy(evict_inode_work_pool); + kmem_cache_destroy(evict_inode_work_cache); +} diff --git a/fs/f2fs/iostat.c b/fs/f2fs/iostat.c index ae265e3..12d4e18 100644 --- a/fs/f2fs/iostat.c +++ b/fs/f2fs/iostat.c @@ -332,6 +332,12 @@ void f2fs_destroy_iostat_processing(void) int f2fs_init_iostat(struct f2fs_sb_info *sbi) { + /* + * The f2fs_iostat tracepoint emits a fixed number of read folio order + * buckets; make sure every order fits so none is silently dropped. + */ + BUILD_BUG_ON(NR_PAGE_ORDERS > F2FS_IOSTAT_RD_FOLIO_ORDERS); + /* init iostat info */ spin_lock_init(&sbi->iostat_lock); spin_lock_init(&sbi->iostat_lat_lock); diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c index 5ae647a..ff86ee0 100644 --- a/fs/f2fs/namei.c +++ b/fs/f2fs/namei.c @@ -402,13 +402,16 @@ static int f2fs_create(struct mnt_idmap *idmap, struct inode *dir, d_instantiate_new(dentry, inode); - if (IS_DIRSYNC(dir)) - f2fs_sync_fs(sbi->sb, 1); + if (IS_DIRSYNC(dir)) { + err = f2fs_sync_fs(sbi->sb, 1); + if (err) + return err; + } f2fs_balance_fs(sbi, true); return 0; out: - f2fs_handle_failed_inode(inode, &lc); + f2fs_handle_failed_inode(inode, &lc, true); return err; } @@ -425,6 +428,9 @@ static int f2fs_link(struct dentry *old_dentry, struct inode *dir, if (!f2fs_is_checkpoint_ready(sbi)) return -ENOSPC; + if (IS_DEVICE_ALIASING(inode)) + return -EPERM; + err = fscrypt_prepare_link(old_dentry, dir, dentry); if (err) return err; @@ -452,8 +458,11 @@ static int f2fs_link(struct dentry *old_dentry, struct inode *dir, d_instantiate(dentry, inode); - if (IS_DIRSYNC(dir)) - f2fs_sync_fs(sbi->sb, 1); + if (IS_DIRSYNC(dir)) { + err = f2fs_sync_fs(sbi->sb, 1); + if (err) + return err; + } return 0; out: clear_inode_flag(inode, FI_INC_LINK); @@ -557,35 +566,31 @@ out: return ERR_PTR(err); } -static int f2fs_unlink(struct inode *dir, struct dentry *dentry) +static int __do_unlink(struct inode *dir, struct inode *inode, + const struct qstr *name) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); - struct inode *inode = d_inode(dentry); struct f2fs_dir_entry *de; struct f2fs_lock_context lc; struct folio *folio; int err; - trace_f2fs_unlink_enter(dir, dentry); + if (IS_DEVICE_ALIASING(inode)) + return -EPERM; - if (unlikely(f2fs_cp_error(sbi))) { - err = -EIO; - goto out; - } + if (unlikely(f2fs_cp_error(sbi))) + return -EIO; err = f2fs_dquot_initialize(dir); if (err) - goto out; + return err; err = f2fs_dquot_initialize(inode); if (err) - goto out; + return err; - de = f2fs_find_entry(dir, &dentry->d_name, &folio); - if (!de) { - if (IS_ERR(folio)) - err = PTR_ERR(folio); - goto out; - } + de = f2fs_find_entry(dir, name, &folio); + if (!de) + return IS_ERR(folio) ? PTR_ERR(folio) : 0; if (unlikely(inode->i_nlink == 0)) { f2fs_warn(sbi, "%s: inode (ino=%llx) has zero i_nlink", @@ -603,11 +608,28 @@ static int f2fs_unlink(struct inode *dir, struct dentry *dentry) err = f2fs_acquire_orphan_inode(sbi); if (err) { f2fs_unlock_op(sbi, &lc); - f2fs_folio_put(folio, false); - goto out; + goto err_out; } f2fs_delete_entry(de, folio, dir, inode); f2fs_unlock_op(sbi, &lc); + return 0; + +corrupted: + err = -EFSCORRUPTED; + set_sbi_flag(sbi, SBI_NEED_FSCK); +err_out: + f2fs_folio_put(folio, false); + return err; +} + +static int f2fs_unlink(struct inode *dir, struct dentry *dentry) +{ + int err; + + trace_f2fs_unlink_enter(dir, dentry); + err = __do_unlink(dir, d_inode(dentry), &dentry->d_name); + if (err) + goto out; /* VFS negative dentries are incompatible with Encoding and * Case-insensitiveness. Eventually we'll want avoid @@ -619,15 +641,9 @@ static int f2fs_unlink(struct inode *dir, struct dentry *dentry) d_invalidate(dentry); if (IS_DIRSYNC(dir)) - f2fs_sync_fs(sbi->sb, 1); - - goto out; -corrupted: - err = -EFSCORRUPTED; - set_sbi_flag(sbi, SBI_NEED_FSCK); - f2fs_folio_put(folio, false); + err = f2fs_sync_fs(F2FS_I_SB(dir)->sb, 1); out: - trace_f2fs_unlink_exit(inode, err); + trace_f2fs_unlink_exit(d_inode(dentry), err); return err; } @@ -654,6 +670,8 @@ static int f2fs_symlink(struct mnt_idmap *idmap, struct inode *dir, struct inode *inode; size_t len = strlen(symname); struct fscrypt_str disk_link; + bool orphan_free = true; + int ret = -EAGAIN; int err; if (unlikely(f2fs_cp_error(sbi))) @@ -684,17 +702,19 @@ static int f2fs_symlink(struct mnt_idmap *idmap, struct inode *dir, f2fs_lock_op(sbi, &lc); err = f2fs_add_link(dentry, inode); if (err) - goto out_f2fs_handle_failed_inode; + goto free_inode; f2fs_unlock_op(sbi, &lc); f2fs_alloc_nid_done(sbi, inode->i_ino); + /* Write the symlink path to the new inode. */ err = fscrypt_encrypt_symlink(inode, symname, len, &disk_link); if (err) - goto err_out; + goto unlink_free_inode; err = page_symlink(inode, disk_link.name, disk_link.len); + if (err) + goto unlink_free_inode; -err_out: d_instantiate_new(dentry, inode); /* @@ -706,25 +726,29 @@ err_out: * If the symlink path is stored into inline_data, there is no * performance regression. */ - if (!err) { - filemap_write_and_wait_range(inode->i_mapping, 0, - disk_link.len - 1); - - if (IS_DIRSYNC(dir)) - f2fs_sync_fs(sbi->sb, 1); - } else { - f2fs_unlink(dir, dentry); - } + ret = filemap_write_and_wait_range(inode->i_mapping, 0, + disk_link.len - 1); + if (!ret && IS_DIRSYNC(dir)) + err = f2fs_sync_fs(sbi->sb, 1); f2fs_balance_fs(sbi, true); - goto out_free_encrypted_link; - -out_f2fs_handle_failed_inode: - f2fs_handle_failed_inode(inode, &lc); -out_free_encrypted_link: +out: if (disk_link.name != (unsigned char *)symname) kfree(disk_link.name); return err; + +unlink_free_inode: + ret = __do_unlink(dir, inode, &dentry->d_name); + if (ret) { + /* Give up and leave a broken symlink. */ + d_instantiate_new(dentry, inode); + goto out; + } + orphan_free = false; + f2fs_lock_op(sbi, &lc); +free_inode: + f2fs_handle_failed_inode(inode, &lc, orphan_free); + goto out; } static struct dentry *f2fs_mkdir(struct mnt_idmap *idmap, struct inode *dir, @@ -762,15 +786,18 @@ static struct dentry *f2fs_mkdir(struct mnt_idmap *idmap, struct inode *dir, d_instantiate_new(dentry, inode); - if (IS_DIRSYNC(dir)) - f2fs_sync_fs(sbi->sb, 1); + if (IS_DIRSYNC(dir)) { + err = f2fs_sync_fs(sbi->sb, 1); + if (err) + return ERR_PTR(err); + } f2fs_balance_fs(sbi, true); return NULL; out_fail: clear_inode_flag(inode, FI_INC_LINK); - f2fs_handle_failed_inode(inode, &lc); + f2fs_handle_failed_inode(inode, &lc, true); return ERR_PTR(err); } @@ -817,13 +844,16 @@ static int f2fs_mknod(struct mnt_idmap *idmap, struct inode *dir, d_instantiate_new(dentry, inode); - if (IS_DIRSYNC(dir)) - f2fs_sync_fs(sbi->sb, 1); + if (IS_DIRSYNC(dir)) { + err = f2fs_sync_fs(sbi->sb, 1); + if (err) + return err; + } f2fs_balance_fs(sbi, true); return 0; out: - f2fs_handle_failed_inode(inode, &lc); + f2fs_handle_failed_inode(inode, &lc, true); return err; } @@ -894,7 +924,7 @@ static int __f2fs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, release_out: f2fs_release_orphan_inode(sbi); out: - f2fs_handle_failed_inode(inode, &lc); + f2fs_handle_failed_inode(inode, &lc, true); return err; } @@ -946,6 +976,9 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir, bool old_is_dir = S_ISDIR(old_inode->i_mode); int err; + if (IS_DEVICE_ALIASING(old_inode)) + return -EPERM; + if (unlikely(f2fs_cp_error(sbi))) return -EIO; if (!f2fs_is_checkpoint_ready(sbi)) @@ -1016,6 +1049,10 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir, } if (new_inode) { + if (IS_DEVICE_ALIASING(new_inode)) { + err = -EPERM; + goto out_dir; + } err = -ENOTEMPTY; if (old_is_dir && !f2fs_empty_dir(new_inode)) @@ -1076,7 +1113,7 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir, f2fs_up_write(&F2FS_I(old_inode)->i_sem); inode_set_ctime_current(old_inode); - f2fs_mark_inode_dirty_sync(old_inode, false); + f2fs_mark_inode_dirty_sync(old_inode, true); f2fs_delete_entry(old_entry, old_folio, old_dir, NULL); old_folio = NULL; @@ -1110,8 +1147,11 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir, f2fs_unlock_op(sbi, &lc); - if (IS_DIRSYNC(old_dir) || IS_DIRSYNC(new_dir)) - f2fs_sync_fs(sbi->sb, 1); + if (IS_DIRSYNC(old_dir) || IS_DIRSYNC(new_dir)) { + err = f2fs_sync_fs(sbi->sb, 1); + if (err) + return err; + } f2fs_update_time(sbi, REQ_TIME); return 0; @@ -1143,6 +1183,9 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, int old_nlink = 0, new_nlink = 0; int err; + if (IS_DEVICE_ALIASING(old_inode) || IS_DEVICE_ALIASING(new_inode)) + return -EPERM; + if (unlikely(f2fs_cp_error(sbi))) return -EIO; if (!f2fs_is_checkpoint_ready(sbi)) @@ -1246,7 +1289,7 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, f2fs_i_links_write(old_dir, old_nlink > 0); f2fs_up_write(&F2FS_I(old_dir)->i_sem); } - f2fs_mark_inode_dirty_sync(old_dir, false); + f2fs_mark_inode_dirty_sync(old_dir, true); /* update directory entry info of new dir inode */ f2fs_set_link(new_dir, new_entry, new_folio, old_inode); @@ -1265,7 +1308,7 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, f2fs_i_links_write(new_dir, new_nlink > 0); f2fs_up_write(&F2FS_I(new_dir)->i_sem); } - f2fs_mark_inode_dirty_sync(new_dir, false); + f2fs_mark_inode_dirty_sync(new_dir, true); if (F2FS_OPTION(sbi).fsync_mode == FSYNC_MODE_STRICT) { f2fs_add_ino_entry(sbi, old_dir->i_ino, TRANS_DIR_INO); @@ -1274,8 +1317,11 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, f2fs_unlock_op(sbi, &lc); - if (IS_DIRSYNC(old_dir) || IS_DIRSYNC(new_dir)) - f2fs_sync_fs(sbi->sb, 1); + if (IS_DIRSYNC(old_dir) || IS_DIRSYNC(new_dir)) { + err = f2fs_sync_fs(sbi->sb, 1); + if (err) + return err; + } f2fs_update_time(sbi, REQ_TIME); return 0; diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 38917e4..86c2e67 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -401,15 +401,16 @@ bool f2fs_need_dentry_mark(struct f2fs_sb_info *sbi, nid_t nid) struct f2fs_nm_info *nm_i = NM_I(sbi); struct nat_entry *e; bool need = false; + struct f2fs_lock_context lc; - f2fs_down_read(&nm_i->nat_tree_lock); + f2fs_down_read_trace(&nm_i->nat_tree_lock, &lc); e = __lookup_nat_cache(nm_i, nid, false); if (e) { if (!get_nat_flag(e, IS_CHECKPOINTED) && !get_nat_flag(e, HAS_FSYNCED_INODE)) need = true; } - f2fs_up_read(&nm_i->nat_tree_lock); + f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc); return need; } @@ -418,12 +419,13 @@ bool f2fs_is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid) struct f2fs_nm_info *nm_i = NM_I(sbi); struct nat_entry *e; bool is_cp = true; + struct f2fs_lock_context lc; - f2fs_down_read(&nm_i->nat_tree_lock); + f2fs_down_read_trace(&nm_i->nat_tree_lock, &lc); e = __lookup_nat_cache(nm_i, nid, false); if (e && !get_nat_flag(e, IS_CHECKPOINTED)) is_cp = false; - f2fs_up_read(&nm_i->nat_tree_lock); + f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc); return is_cp; } @@ -432,16 +434,16 @@ bool f2fs_need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino) struct f2fs_nm_info *nm_i = NM_I(sbi); struct nat_entry *e; bool need_update = true; - struct f2fs_lock_context lc; + struct f2fs_lock_context lc, nlc; f2fs_down_read_trace(&sbi->node_write, &lc); - f2fs_down_read(&nm_i->nat_tree_lock); + f2fs_down_read_trace(&nm_i->nat_tree_lock, &nlc); e = __lookup_nat_cache(nm_i, ino, false); if (e && get_nat_flag(e, HAS_LAST_FSYNC) && (get_nat_flag(e, IS_CHECKPOINTED) || get_nat_flag(e, HAS_FSYNCED_INODE))) need_update = false; - f2fs_up_read(&nm_i->nat_tree_lock); + f2fs_up_read_trace(&nm_i->nat_tree_lock, &nlc); f2fs_up_read_trace(&sbi->node_write, &lc); return need_update; } @@ -452,6 +454,7 @@ static void cache_nat_entry(struct f2fs_sb_info *sbi, nid_t nid, { struct f2fs_nm_info *nm_i = NM_I(sbi); struct nat_entry *new, *e; + struct f2fs_lock_context lc; /* Let's mitigate lock contention of nat_tree_lock during checkpoint */ if (f2fs_rwsem_is_locked(&sbi->cp_global_sem)) @@ -461,7 +464,7 @@ static void cache_nat_entry(struct f2fs_sb_info *sbi, nid_t nid, if (!new) return; - f2fs_down_write(&nm_i->nat_tree_lock); + f2fs_down_write_trace(&nm_i->nat_tree_lock, &lc); e = __lookup_nat_cache(nm_i, nid, false); if (!e) e = __init_nat_entry(nm_i, new, ne, false, false); @@ -470,7 +473,7 @@ static void cache_nat_entry(struct f2fs_sb_info *sbi, nid_t nid, nat_get_blkaddr(e) != le32_to_cpu(ne->block_addr) || nat_get_version(e) != ne->version); - f2fs_up_write(&nm_i->nat_tree_lock); + f2fs_up_write_trace(&nm_i->nat_tree_lock, &lc); if (e != new) __free_nat_entry(new); } @@ -482,8 +485,9 @@ static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni, struct nat_entry *e; struct nat_entry *new = __alloc_nat_entry(sbi, ni->nid, true); bool init_dirty = false; + struct f2fs_lock_context lc; - f2fs_down_write(&nm_i->nat_tree_lock); + f2fs_down_write_trace(&nm_i->nat_tree_lock, &lc); e = __lookup_nat_cache(nm_i, ni->nid, true); if (!e) { init_dirty = true; @@ -533,15 +537,16 @@ static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni, set_nat_flag(e, HAS_FSYNCED_INODE, true); set_nat_flag(e, HAS_LAST_FSYNC, fsync_done); } - f2fs_up_write(&nm_i->nat_tree_lock); + f2fs_up_write_trace(&nm_i->nat_tree_lock, &lc); } int f2fs_try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink) { struct f2fs_nm_info *nm_i = NM_I(sbi); int nr = nr_shrink; + struct f2fs_lock_context lc; - if (!f2fs_down_write_trylock(&nm_i->nat_tree_lock)) + if (!f2fs_down_write_trylock_trace(&nm_i->nat_tree_lock, &lc)) return 0; spin_lock(&nm_i->nat_list_lock); @@ -563,7 +568,7 @@ int f2fs_try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink) } spin_unlock(&nm_i->nat_list_lock); - f2fs_up_write(&nm_i->nat_tree_lock); + f2fs_up_write_trace(&nm_i->nat_tree_lock, &lc); return nr - nr_shrink; } @@ -581,18 +586,19 @@ int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid, pgoff_t index; int i; bool need_cache = true; + struct f2fs_lock_context lc; ni->flag = 0; ni->nid = nid; retry: /* Check nat cache */ - f2fs_down_read(&nm_i->nat_tree_lock); + f2fs_down_read_trace(&nm_i->nat_tree_lock, &lc); e = __lookup_nat_cache(nm_i, nid, false); if (e) { ni->ino = nat_get_ino(e); ni->blk_addr = nat_get_blkaddr(e); ni->version = nat_get_version(e); - f2fs_up_read(&nm_i->nat_tree_lock); + f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc); if (IS_ENABLED(CONFIG_F2FS_CHECK_FS)) { need_cache = false; goto sanity_check; @@ -610,7 +616,7 @@ retry: down_read(&curseg->journal_rwsem); } else if (f2fs_rwsem_is_contended(&nm_i->nat_tree_lock) || !down_read_trylock(&curseg->journal_rwsem)) { - f2fs_up_read(&nm_i->nat_tree_lock); + f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc); goto retry; } @@ -621,13 +627,13 @@ retry: } up_read(&curseg->journal_rwsem); if (i >= 0) { - f2fs_up_read(&nm_i->nat_tree_lock); + f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc); goto sanity_check; } /* Fill node_info from nat page */ index = current_nat_addr(sbi, nid); - f2fs_up_read(&nm_i->nat_tree_lock); + f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc); folio = f2fs_get_meta_folio(sbi, index); if (IS_ERR(folio)) @@ -660,6 +666,7 @@ sanity_check: __builtin_return_address(0), ni->ino, ni->nid, ni->blk_addr, ni->version, ni->flag); f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT); + return -EFSCORRUPTED; } /* cache nat entry */ @@ -1617,7 +1624,7 @@ page_hit: if (!err) return folio; out_err: - folio_clear_uptodate(folio); + clear_node_folio_dirty(folio); out_put_err: /* ENOENT comes from read_node_folio which is not an error. */ if (err != -ENOENT) @@ -1789,7 +1796,7 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool do_fsync, /* get old block addr of this node page */ nid = nid_of_node(folio); - if (f2fs_sanity_check_node_footer(sbi, folio, nid, + if (f2fs_sanity_check_node_footer(sbi, folio, folio->index, NODE_TYPE_REGULAR, false)) { fserror_report_metadata(sbi->sb, -EFSCORRUPTED, GFP_NOFS); f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_CORRUPTED_NID); @@ -2009,6 +2016,11 @@ continue_unlock: f2fs_debug(sbi, "Retry to write fsync mark: ino=%u, idx=%lx", ino, last_folio->index); folio_lock(last_folio); + if (unlikely(!is_node_folio(last_folio))) { + f2fs_folio_put(last_folio, true); + ret = -EAGAIN; + goto out; + } f2fs_folio_wait_writeback(last_folio, NODE, true, true); folio_mark_dirty(last_folio); folio_unlock(last_folio); @@ -2566,8 +2578,9 @@ static void scan_free_nid_bits(struct f2fs_sb_info *sbi) struct f2fs_nm_info *nm_i = NM_I(sbi); unsigned int i, idx; nid_t nid; + struct f2fs_lock_context lc; - f2fs_down_read(&nm_i->nat_tree_lock); + f2fs_down_read_trace(&nm_i->nat_tree_lock, &lc); for (i = 0; i < nm_i->nat_blocks; i++) { if (!test_bit_le(i, nm_i->nat_block_bitmap)) @@ -2590,7 +2603,7 @@ static void scan_free_nid_bits(struct f2fs_sb_info *sbi) out: scan_curseg_cache(sbi); - f2fs_up_read(&nm_i->nat_tree_lock); + f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc); } static int __f2fs_build_free_nids(struct f2fs_sb_info *sbi, @@ -2599,6 +2612,7 @@ static int __f2fs_build_free_nids(struct f2fs_sb_info *sbi, struct f2fs_nm_info *nm_i = NM_I(sbi); int i = 0, ret; nid_t nid = nm_i->next_scan_nid; + struct f2fs_lock_context lc; if (unlikely(nid >= nm_i->max_nid)) nid = 0; @@ -2625,7 +2639,7 @@ static int __f2fs_build_free_nids(struct f2fs_sb_info *sbi, f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), FREE_NID_PAGES, META_NAT, true); - f2fs_down_read(&nm_i->nat_tree_lock); + f2fs_down_read_trace(&nm_i->nat_tree_lock, &lc); while (1) { if (!test_bit_le(NAT_BLOCK_OFFSET(nid), @@ -2641,7 +2655,7 @@ static int __f2fs_build_free_nids(struct f2fs_sb_info *sbi, } if (ret) { - f2fs_up_read(&nm_i->nat_tree_lock); + f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc); if (ret == -EFSCORRUPTED) { f2fs_err(sbi, "NAT is corrupt, run fsck to fix it"); @@ -2668,7 +2682,7 @@ static int __f2fs_build_free_nids(struct f2fs_sb_info *sbi, /* find free nids from current sum_pages */ scan_curseg_cache(sbi); - f2fs_up_read(&nm_i->nat_tree_lock); + f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc); f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nm_i->next_scan_nid), nm_i->ra_nid_pages, META_NAT, false); @@ -3205,21 +3219,22 @@ int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) nid_t set_idx = 0; LIST_HEAD(sets); int err = 0; + struct f2fs_lock_context lc; /* * during unmount, let's flush nat_bits before checking * nat_cnt[DIRTY_NAT]. */ if (enabled_nat_bits(sbi, cpc)) { - f2fs_down_write(&nm_i->nat_tree_lock); + f2fs_down_write_trace(&nm_i->nat_tree_lock, &lc); remove_nats_in_journal(sbi); - f2fs_up_write(&nm_i->nat_tree_lock); + f2fs_up_write_trace(&nm_i->nat_tree_lock, &lc); } if (!nm_i->nat_cnt[DIRTY_NAT]) return 0; - f2fs_down_write(&nm_i->nat_tree_lock); + f2fs_down_write_trace(&nm_i->nat_tree_lock, &lc); /* * if there are no enough space in journal to store dirty nat @@ -3260,7 +3275,7 @@ int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) break; } - f2fs_up_write(&nm_i->nat_tree_lock); + f2fs_up_write_trace(&nm_i->nat_tree_lock, &lc); /* Allow dirty nats by node block allocation in write_begin */ return err; @@ -3379,7 +3394,8 @@ static int init_node_manager(struct f2fs_sb_info *sbi) mutex_init(&nm_i->build_lock); spin_lock_init(&nm_i->nid_list_lock); - init_f2fs_rwsem(&nm_i->nat_tree_lock); + init_f2fs_rwsem_trace(&nm_i->nat_tree_lock, sbi, + LOCK_NAME_NAT_TREE_LOCK); nm_i->next_scan_nid = le32_to_cpu(sbi->ckpt->next_free_nid); nm_i->bitmap_size = __bitmap_size(sbi, NAT_BITMAP); @@ -3471,6 +3487,7 @@ void f2fs_destroy_node_manager(struct f2fs_sb_info *sbi) struct nat_entry_set **setvec = (struct nat_entry_set **)vec; nid_t nid = 0; unsigned int found; + struct f2fs_lock_context lc; if (!nm_i) return; @@ -3489,7 +3506,7 @@ void f2fs_destroy_node_manager(struct f2fs_sb_info *sbi) spin_unlock(&nm_i->nid_list_lock); /* destroy nat cache */ - f2fs_down_write(&nm_i->nat_tree_lock); + f2fs_down_write_trace(&nm_i->nat_tree_lock, &lc); while ((found = __gang_lookup_nat_cache(nm_i, nid, NAT_VEC_SIZE, natvec))) { unsigned idx; @@ -3520,7 +3537,7 @@ void f2fs_destroy_node_manager(struct f2fs_sb_info *sbi) kmem_cache_free(nat_entry_set_slab, setvec[idx]); } } - f2fs_up_write(&nm_i->nat_tree_lock); + f2fs_up_write_trace(&nm_i->nat_tree_lock, &lc); kvfree(nm_i->nat_block_bitmap); if (nm_i->free_nid_bitmap) { diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c index 89af840..aaa5227 100644 --- a/fs/f2fs/recovery.c +++ b/fs/f2fs/recovery.c @@ -116,18 +116,26 @@ static void del_fsync_inode(struct fsync_inode_entry *entry, int drop) } static int init_recovered_filename(const struct inode *dir, + struct inode *inode, struct f2fs_inode *raw_inode, struct f2fs_filename *fname, struct qstr *usr_fname) { + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); int err; memset(fname, 0, sizeof(*fname)); fname->disk_name.len = le32_to_cpu(raw_inode->i_namelen); fname->disk_name.name = raw_inode->i_name; - if (WARN_ON(fname->disk_name.len > F2FS_NAME_LEN)) - return -ENAMETOOLONG; + if (unlikely(!fname->disk_name.len || + fname->disk_name.len > F2FS_NAME_LEN)) { + f2fs_err(sbi, "invalid recovered filename length %u for ino %llu", + fname->disk_name.len, inode->i_ino); + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); + return -EFSCORRUPTED; + } if (!IS_ENCRYPTED(dir)) { usr_fname->name = fname->disk_name.name; @@ -158,6 +166,22 @@ static int init_recovered_filename(const struct inode *dir, return 0; } +static const char *recover_printable_name(struct inode *inode, + struct f2fs_inode *raw, + int *name_len) +{ + static const char encrypted_name[] = ""; + + if (file_enc_name(inode)) { + *name_len = sizeof(encrypted_name) - 1; + return encrypted_name; + } + + *name_len = min_t(unsigned int, le32_to_cpu(raw->i_namelen), + F2FS_NAME_LEN); + return raw->i_name; +} + static int recover_dentry(struct inode *inode, struct folio *ifolio, struct list_head *dir_list) { @@ -170,7 +194,8 @@ static int recover_dentry(struct inode *inode, struct folio *ifolio, struct inode *dir, *einode; struct fsync_inode_entry *entry; int err = 0; - char *name; + const char *name; + int name_len; entry = get_fsync_inode(dir_list, pino); if (!entry) { @@ -184,7 +209,7 @@ static int recover_dentry(struct inode *inode, struct folio *ifolio, } dir = entry->inode; - err = init_recovered_filename(dir, raw_inode, &fname, &usr_fname); + err = init_recovered_filename(dir, inode, raw_inode, &fname, &usr_fname); if (err) goto out; retry: @@ -229,12 +254,9 @@ retry: out_put: f2fs_folio_put(folio, false); out: - if (file_enc_name(inode)) - name = ""; - else - name = raw_inode->i_name; - f2fs_notice(F2FS_I_SB(inode), "%s: ino = %x, name = %s, dir = %llu, err = %d", - __func__, ino_of_node(ifolio), name, + name = recover_printable_name(inode, raw_inode, &name_len); + f2fs_notice(F2FS_I_SB(inode), "%s: ino = %x, name = %.*s, dir = %llu, err = %d", + __func__, ino_of_node(ifolio), name_len, name, IS_ERR(dir) ? 0 : dir->i_ino, err); return err; } @@ -282,7 +304,8 @@ static int recover_inode(struct inode *inode, struct folio *folio) { struct f2fs_inode *raw = F2FS_INODE(folio); struct f2fs_inode_info *fi = F2FS_I(inode); - char *name; + const char *name; + int name_len; int err; inode->i_mode = le16_to_cpu(raw->i_mode); @@ -331,13 +354,11 @@ static int recover_inode(struct inode *inode, struct folio *folio) f2fs_mark_inode_dirty_sync(inode, true); - if (file_enc_name(inode)) - name = ""; - else - name = F2FS_INODE(folio)->i_name; + name = recover_printable_name(inode, raw, &name_len); - f2fs_notice(F2FS_I_SB(inode), "recover_inode: ino = %x, name = %s, inline = %x", - ino_of_node(folio), name, raw->i_inline); + f2fs_notice(F2FS_I_SB(inode), "%s: ino = %x, name = %.*s, inline = %x", + __func__, ino_of_node(folio), name_len, name, + raw->i_inline); return 0; } diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 642668a..63b712d 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -261,7 +261,7 @@ retry: } else { blkcnt_t count = 1; - err = inc_valid_block_count(sbi, inode, &count, true); + err = inc_valid_block_count(sbi, inode, &count, true, false); if (err) { f2fs_put_dnode(&dn); return err; @@ -452,15 +452,14 @@ void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) f2fs_submit_merged_write(sbi, DATA); f2fs_submit_all_merged_ipu_writes(sbi); - if (test_opt(sbi, GC_MERGE) && sbi->gc_thread && - sbi->gc_thread->f2fs_gc_task) { + if (test_opt(sbi, GC_MERGE) && sbi->gc_thread.f2fs_gc_task) { DEFINE_WAIT(wait); - prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait, + prepare_to_wait(&sbi->gc_thread.fggc_wq, &wait, TASK_UNINTERRUPTIBLE); - wake_up(&sbi->gc_thread->gc_wait_queue_head); + wake_up(&sbi->gc_thread.gc_wait_queue_head); io_schedule(); - finish_wait(&sbi->gc_thread->fggc_wq, &wait); + finish_wait(&sbi->gc_thread.fggc_wq, &wait); } else { struct f2fs_gc_control gc_control = { .victim_segno = NULL_SEGNO, @@ -566,21 +565,59 @@ static int __submit_flush_wait(struct f2fs_sb_info *sbi, return ret; } +static void f2fs_flush_end_io(struct bio *bio) +{ + complete(bio->bi_private); +} + +struct f2fs_flush_bio { + struct bio bio; + struct completion wait; +}; + static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino) { + struct f2fs_flush_bio *flush_bio; + unsigned long devices = 0; int ret = 0; int i; if (!f2fs_is_multi_device(sbi)) return __submit_flush_wait(sbi, sbi->sb->s_bdev); + flush_bio = kmalloc(array_size(sbi->s_ndevs, sizeof(*flush_bio)), + GFP_NOFS | __GFP_NOFAIL); + for (i = 0; i < sbi->s_ndevs; i++) { if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO)) continue; - ret = __submit_flush_wait(sbi, FDEV(i).bdev); - if (ret) - break; + + bio_init(&flush_bio[i].bio, FDEV(i).bdev, NULL, 0, + REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH); + init_completion(&flush_bio[i].wait); + flush_bio[i].bio.bi_private = &flush_bio[i].wait; + flush_bio[i].bio.bi_end_io = f2fs_flush_end_io; + devices |= BIT(i); + submit_bio(&flush_bio[i].bio); } + + for (i = 0; i < sbi->s_ndevs; i++) { + int err; + + if (!(devices & BIT(i))) + continue; + + wait_for_completion(&flush_bio[i].wait); + err = blk_status_to_errno(flush_bio[i].bio.bi_status); + trace_f2fs_issue_flush(FDEV(i).bdev, test_opt(sbi, NOBARRIER), + test_opt(sbi, FLUSH_MERGE), err); + if (!err) + f2fs_update_iostat(sbi, NULL, FS_FLUSH_IO, 0); + else if (!ret) + ret = err; + bio_uninit(&flush_bio[i].bio); + } + kfree(flush_bio); return ret; } @@ -2502,35 +2539,42 @@ static int update_sit_entry_for_alloc(struct f2fs_sb_info *sbi, struct seg_entry unsigned int segno, block_t blkaddr, unsigned int offset, int del) { bool exist; + int del_count = del; + int i; - exist = f2fs_test_and_set_bit(offset, se->cur_valid_map); - if (unlikely(exist)) { - f2fs_err(sbi, "Bitmap was wrongly set, blk:%u", blkaddr); - f2fs_bug_on(sbi, 1); - se->valid_blocks--; - del = 0; - } + f2fs_bug_on(sbi, GET_SEGNO(sbi, blkaddr) != GET_SEGNO(sbi, blkaddr + del_count - 1)); - if (f2fs_block_unit_discard(sbi) && - !f2fs_test_and_set_bit(offset, se->discard_map)) - sbi->discard_blks--; + for (i = 0; i < del_count; i++) { + exist = f2fs_test_and_set_bit(offset + i, se->cur_valid_map); + if (unlikely(exist)) { + f2fs_err(sbi, "Bitmap was wrongly set, blk:%u", blkaddr + i); + f2fs_bug_on(sbi, 1); + se->valid_blocks--; + del -= 1; + continue; + } - /* - * SSR should never reuse block which is checkpointed - * or newly invalidated. - */ - if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) { - if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map)) { - se->ckpt_valid_blocks++; - if (__is_large_section(sbi)) - get_sec_entry(sbi, segno)->ckpt_valid_blocks++; + if (f2fs_block_unit_discard(sbi) && + !f2fs_test_and_set_bit(offset + i, se->discard_map)) + sbi->discard_blks--; + + /* + * SSR should never reuse block which is checkpointed + * or newly invalidated. + */ + if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) { + if (!f2fs_test_and_set_bit(offset + i, se->ckpt_valid_map)) { + se->ckpt_valid_blocks++; + if (__is_large_section(sbi)) + get_sec_entry(sbi, segno)->ckpt_valid_blocks++; + } } - } - if (!f2fs_test_bit(offset, se->ckpt_valid_map)) { - se->ckpt_valid_blocks += del; - if (__is_large_section(sbi)) - get_sec_entry(sbi, segno)->ckpt_valid_blocks += del; + if (!f2fs_test_bit(offset + i, se->ckpt_valid_map)) { + se->ckpt_valid_blocks += 1; + if (__is_large_section(sbi)) + get_sec_entry(sbi, segno)->ckpt_valid_blocks += 1; + } } if (__is_large_section(sbi)) @@ -2585,9 +2629,14 @@ void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr, unsigned int segno = GET_SEGNO(sbi, addr); struct sit_info *sit_i = SIT_I(sbi); block_t addr_start = addr, addr_end = addr + len - 1; - unsigned int seg_num = GET_SEGNO(sbi, addr_end) - segno + 1; + unsigned int seg_num; unsigned int i = 1, max_blocks = sbi->blocks_per_seg, cnt; + if (len == 0) + return; + + seg_num = GET_SEGNO(sbi, addr_end) - segno + 1; + f2fs_bug_on(sbi, addr == NULL_ADDR); if (addr == NEW_ADDR || addr == COMPRESS_ADDR) return; @@ -2620,6 +2669,52 @@ void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr, up_write(&sit_i->sentry_lock); } +void f2fs_reserve_device_alias(struct f2fs_sb_info *sbi, block_t addr, + unsigned int len) +{ + unsigned int segno = GET_SEGNO(sbi, addr); + struct sit_info *sit_i = SIT_I(sbi); + block_t addr_start = addr, addr_end = addr + len - 1; + unsigned int seg_num; + unsigned int i = 1, max_blocks = sbi->blocks_per_seg, cnt; + + if (len == 0) + return; + + seg_num = GET_SEGNO(sbi, addr_end) - segno + 1; + + down_write(&sit_i->sentry_lock); + + if (seg_num == 1) + cnt = len; + else + cnt = max_blocks - GET_BLKOFF_FROM_SEG0(sbi, addr); + + do { + update_segment_mtime(sbi, addr_start, 0); + update_sit_entry(sbi, addr_start, cnt); + __set_test_and_inuse(sbi, segno); + + /* Remove the segment from PRE (prefree) to prevent checkpoint from freeing it! */ + mutex_lock(&DIRTY_I(sbi)->seglist_lock); + if (test_and_clear_bit(segno, DIRTY_I(sbi)->dirty_segmap[PRE])) + DIRTY_I(sbi)->nr_dirty[PRE]--; + mutex_unlock(&DIRTY_I(sbi)->seglist_lock); + + /* add it into dirty seglist */ + locate_dirty_segment(sbi, segno); + + /* update @addr_start and @cnt and @segno */ + addr_start = START_BLOCK(sbi, ++segno); + if (++i == seg_num) + cnt = GET_BLKOFF_FROM_SEG0(sbi, addr_end) + 1; + else + cnt = max_blocks; + } while (i <= seg_num); + + up_write(&sit_i->sentry_lock); +} + bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr) { struct sit_info *sit_i = SIT_I(sbi); @@ -2758,8 +2853,13 @@ static int is_next_segment_free(struct f2fs_sb_info *sbi, unsigned int segno = curseg->segno + 1; struct free_segmap_info *free_i = FREE_I(sbi); - if (segno < MAIN_SEGS(sbi) && segno % SEGS_PER_SEC(sbi)) + if (segno < MAIN_SEGS(sbi) && segno % SEGS_PER_SEC(sbi)) { + int devi = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno)); + + if (f2fs_dev_is_reserving(sbi, devi)) + return 0; return !test_bit(segno, free_i->free_segmap); + } return 0; } @@ -2777,8 +2877,10 @@ static int get_new_segment(struct f2fs_sb_info *sbi, unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg); unsigned int alloc_policy = sbi->allocate_section_policy; unsigned int alloc_hint = sbi->allocate_section_hint; + unsigned int max_secno = MAIN_SECS(sbi); bool init = true; - int i; + bool looped = false; + int i, devi; int ret = 0; spin_lock(&free_i->segmap_lock); @@ -2791,8 +2893,13 @@ static int get_new_segment(struct f2fs_sb_info *sbi, if (!new_sec && ((*newseg + 1) % SEGS_PER_SEC(sbi))) { segno = find_next_zero_bit(free_i->free_segmap, GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1); - if (segno < GET_SEG_FROM_SEC(sbi, hint + 1)) + if (segno < GET_SEG_FROM_SEC(sbi, hint + 1)) { + devi = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno)); + + if (f2fs_dev_is_alloc_blocked(sbi, devi, pinning)) + goto find_other_zone; goto got_it; + } } #ifdef CONFIG_BLK_DEV_ZONED @@ -2802,7 +2909,7 @@ static int get_new_segment(struct f2fs_sb_info *sbi, */ if (f2fs_sb_has_blkzoned(sbi)) { /* Prioritize writing to conventional zones */ - if (sbi->blkzone_alloc_policy == BLKZONE_ALLOC_PRIOR_CONV || pinning) + if (sbi->blkzone_alloc_policy == BLKZONE_ALLOC_PRIOR_CONV) segno = 0; else segno = max(sbi->first_seq_zone_segno, *newseg); @@ -2818,43 +2925,57 @@ static int get_new_segment(struct f2fs_sb_info *sbi, alloc_hint > MAIN_SECS(sbi)) alloc_hint = MAIN_SECS(sbi); - if (alloc_policy == ALLOCATE_FORWARD_FROM_HINT && - hint < alloc_hint) + if (pinning) { + max_secno = sbi->pinned_area_max_secno; + hint = 0; + } else if (alloc_policy == ALLOCATE_FORWARD_FROM_HINT && + hint < alloc_hint) { hint = alloc_hint; - else if (alloc_policy == ALLOCATE_FORWARD_WITHIN_HINT && - hint >= alloc_hint) + } else if (alloc_policy == ALLOCATE_FORWARD_WITHIN_HINT && + hint >= alloc_hint) { hint = 0; + } find_other_zone: - secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint); + secno = find_next_zero_bit(free_i->free_secmap, max_secno, hint); + if (secno >= max_secno) { + if (looped) { + ret = (pinning && has_unpinned_area(sbi)) ? + -EAGAIN : -ENOSPC; + f2fs_bug_on(sbi, !pinning); + goto out_unlock; + } + hint = 0; #ifdef CONFIG_BLK_DEV_ZONED - if (secno >= MAIN_SECS(sbi) && f2fs_sb_has_blkzoned(sbi)) { /* Write only to sequential zones */ - if (sbi->blkzone_alloc_policy == BLKZONE_ALLOC_ONLY_SEQ) { + if (f2fs_sb_has_blkzoned(sbi) && + sbi->blkzone_alloc_policy == BLKZONE_ALLOC_ONLY_SEQ) hint = GET_SEC_FROM_SEG(sbi, sbi->first_seq_zone_segno); - secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint); - } else - secno = find_first_zero_bit(free_i->free_secmap, - MAIN_SECS(sbi)); - if (secno >= MAIN_SECS(sbi)) { - ret = -ENOSPC; - f2fs_bug_on(sbi, 1); - goto out_unlock; - } - } #endif + looped = true; + goto find_other_zone; + } - if (secno >= MAIN_SECS(sbi)) { - secno = find_first_zero_bit(free_i->free_secmap, - MAIN_SECS(sbi)); - if (secno >= MAIN_SECS(sbi)) { - ret = -ENOSPC; - f2fs_bug_on(sbi, !pinning); - goto out_unlock; + segno = GET_SEG_FROM_SEC(sbi, secno); + + devi = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno)); + + if (f2fs_dev_is_alloc_blocked(sbi, devi, pinning)) { + while (devi < sbi->s_ndevs && + f2fs_dev_is_alloc_blocked(sbi, devi, pinning)) { + unsigned int end_segno = GET_SEGNO(sbi, FDEV(devi).end_blk); + + hint = GET_SEC_FROM_SEG(sbi, end_segno) + 1; + devi++; } + goto find_other_zone; + } + + if (sec_usage_check(sbi, secno)) { + hint = secno + 1; + goto find_other_zone; } - segno = GET_SEG_FROM_SEC(sbi, secno); zoneno = GET_ZONE_FROM_SEC(sbi, secno); /* give up on finding another zone */ @@ -2886,12 +3007,6 @@ got_it: goto out_unlock; } - /* no free section in conventional device or conventional zone */ - if (new_sec && pinning && - f2fs_is_sequential_zone_area(sbi, START_BLOCK(sbi, segno))) { - ret = -EAGAIN; - goto out_unlock; - } __set_inuse(sbi, segno); *newseg = segno; out_unlock: @@ -2937,7 +3052,7 @@ static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type) sanity_check_seg_type(sbi, seg_type); if (__is_large_section(sbi)) { - if (f2fs_need_rand_seg(sbi)) { + if (f2fs_need_rand_seg_blk(sbi, type)) { unsigned int hint = GET_SEC_FROM_SEG(sbi, curseg->segno); if (GET_SEC_FROM_SEG(sbi, curseg->segno + 1) != hint) @@ -2946,7 +3061,7 @@ static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type) GET_SEG_FROM_SEC(sbi, hint + 1) - 1); } return curseg->segno; - } else if (f2fs_need_rand_seg(sbi)) { + } else if (f2fs_need_rand_seg_blk(sbi, type)) { return get_random_u32_below(MAIN_SECS(sbi) * SEGS_PER_SEC(sbi)); } @@ -3002,7 +3117,7 @@ static int new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) curseg->next_segno = segno; reset_curseg(sbi, type, 1); curseg->alloc_type = LFS; - if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) + if (f2fs_need_rand_blk(sbi, type)) curseg->fragment_remained_chunk = get_random_u32_inclusive(1, sbi->max_fragment_chunk); return 0; @@ -3357,15 +3472,17 @@ retry: err = f2fs_allocate_new_section(sbi, CURSEG_COLD_DATA_PINNED, false); f2fs_unlock_op(sbi, &lc); - if (f2fs_sb_has_blkzoned(sbi) && err == -EAGAIN && gc_required) { - f2fs_down_write_trace(&sbi->gc_lock, &lc); - err = f2fs_gc_range(sbi, 0, sbi->first_seq_zone_segno - 1, - true, ZONED_PIN_SEC_REQUIRED_COUNT); - f2fs_up_write_trace(&sbi->gc_lock, &lc); - - gc_required = false; - if (!err) + if (has_unpinned_area(sbi) && err == -EAGAIN && gc_required) { + err = f2fs_gc_range(sbi, 0, + sbi->pinned_area_max_secno * SEGS_PER_SEC(sbi) - 1, + true, ZONED_PIN_SEC_REQUIRED_COUNT, true); + if (err) + return err; + err = f2fs_sync_fs(sbi->sb, 1); + if (!err) { + gc_required = false; goto retry; + } } return err; @@ -3820,7 +3937,7 @@ int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio, curseg->next_blkoff = f2fs_find_next_ssr_block(sbi, curseg); } else { curseg->next_blkoff++; - if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) + if (f2fs_need_rand_blk(sbi, type)) f2fs_randomize_chunk(sbi, curseg); } if (curseg->next_blkoff >= f2fs_usable_blks_in_seg(sbi, curseg->segno)) diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h index b0c06b3..5949aa5 100644 --- a/fs/f2fs/segment.h +++ b/fs/f2fs/segment.h @@ -43,6 +43,7 @@ static inline void sanity_check_seg_type(struct f2fs_sb_info *sbi, #define MAIN_SEGS(sbi) (SM_I(sbi)->main_segments) #define MAIN_SECS(sbi) ((sbi)->total_sections) +#define has_unpinned_area(sbi) ((sbi)->pinned_area_max_secno < MAIN_SECS(sbi)) #define TOTAL_SEGS(sbi) \ (SM_I(sbi) ? SM_I(sbi)->segment_count : \ @@ -796,20 +797,6 @@ F2FS_IPU_POLICY(F2FS_IPU_ASYNC); F2FS_IPU_POLICY(F2FS_IPU_NOCACHE); F2FS_IPU_POLICY(F2FS_IPU_HONOR_OPU_WRITE); -static inline unsigned int curseg_segno(struct f2fs_sb_info *sbi, - int type) -{ - struct curseg_info *curseg = CURSEG_I(sbi, type); - return curseg->segno; -} - -static inline unsigned char curseg_alloc_type(struct f2fs_sb_info *sbi, - int type) -{ - struct curseg_info *curseg = CURSEG_I(sbi, type); - return curseg->alloc_type; -} - static inline bool valid_main_segno(struct f2fs_sb_info *sbi, unsigned int segno) { @@ -954,10 +941,32 @@ static inline block_t sum_blk_addr(struct f2fs_sb_info *sbi, int base, int type) - (base + 1) + type; } +static inline bool f2fs_dev_is_reserving(struct f2fs_sb_info *sbi, int devi) +{ + if (!f2fs_sb_has_device_alias(sbi)) + return false; + return FDEV(devi).is_reserving; +} + +static inline bool f2fs_dev_is_alloc_blocked(struct f2fs_sb_info *sbi, + int devi, bool pinning) +{ + if (!f2fs_sb_has_device_alias(sbi)) + return false; + return (pinning && FDEV(devi).has_alias) || FDEV(devi).is_reserving; +} + static inline bool sec_usage_check(struct f2fs_sb_info *sbi, unsigned int secno) { if (is_cursec(sbi, secno) || (sbi->cur_victim_sec == secno)) return true; + if (f2fs_sb_has_device_alias(sbi)) { + block_t start_blk = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno)); + int devi = f2fs_target_device_index(sbi, start_blk); + + if (f2fs_dev_is_reserving(sbi, devi)) + return true; + } return false; } diff --git a/fs/f2fs/shrinker.c b/fs/f2fs/shrinker.c index b88babc..4f6bf59 100644 --- a/fs/f2fs/shrinker.c +++ b/fs/f2fs/shrinker.c @@ -109,7 +109,8 @@ unsigned long f2fs_shrink_scan(struct shrinker *shrink, freed += f2fs_shrink_age_extent_tree(sbi, nr >> 2); /* shrink read extent cache entries */ - freed += f2fs_shrink_read_extent_tree(sbi, nr >> 2); + if (freed < nr) + freed += f2fs_shrink_read_extent_tree(sbi, nr >> 2); /* shrink clean nat cache entries */ if (freed < nr) diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index be22c31..1314b6c 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -235,6 +235,7 @@ enum { Opt_jqfmt, Opt_checkpoint, Opt_lookup_mode, + Opt_resizable_tail_secno, Opt_err, }; @@ -366,6 +367,7 @@ static const struct fs_parameter_spec f2fs_param_specs[] = { fsparam_flag("age_extent_cache", Opt_age_extent_cache), fsparam_enum("errors", Opt_errors, f2fs_param_errors), fsparam_enum("lookup_mode", Opt_lookup_mode, f2fs_param_lookup_mode), + fsparam_u32("resizable_tail_secno", Opt_resizable_tail_secno), {} }; @@ -404,6 +406,7 @@ static match_table_t f2fs_checkpoint_tokens = { #define F2FS_SPEC_errors (1 << 23) #define F2FS_SPEC_lookup_mode (1 << 24) #define F2FS_SPEC_reserve_node (1 << 25) +#define F2FS_SPEC_resizable_tail_secno (1 << 26) struct f2fs_fs_context { struct f2fs_mount_info info; @@ -551,6 +554,17 @@ static inline void adjust_unusable_cap_perc(struct f2fs_sb_info *sbi) F2FS_OPTION(sbi).unusable_cap_perc); } +static inline void adjust_pinned_area_boundary(struct f2fs_sb_info *sbi) +{ + sbi->pinned_area_max_secno = MAIN_SECS(sbi); + if (f2fs_sb_has_blkzoned(sbi) && sbi->first_seq_zone_segno != NULL_SEGNO) + sbi->pinned_area_max_secno = min(sbi->pinned_area_max_secno, + GET_SEC_FROM_SEG(sbi, sbi->first_seq_zone_segno)); + if (F2FS_OPTION(sbi).resizable_tail_secno) + sbi->pinned_area_max_secno = min(sbi->pinned_area_max_secno, + MAIN_SECS(sbi) - F2FS_OPTION(sbi).resizable_tail_secno); +} + static void init_once(void *foo) { struct f2fs_inode_info *fi = (struct f2fs_inode_info *) foo; @@ -1235,6 +1249,10 @@ static int f2fs_parse_param(struct fs_context *fc, struct fs_parameter *param) F2FS_CTX_INFO(ctx).lookup_mode = result.uint_32; ctx->spec_mask |= F2FS_SPEC_lookup_mode; break; + case Opt_resizable_tail_secno: + F2FS_CTX_INFO(ctx).resizable_tail_secno = result.uint_32; + ctx->spec_mask |= F2FS_SPEC_resizable_tail_secno; + break; } return 0; } @@ -1763,6 +1781,9 @@ static void f2fs_apply_options(struct fs_context *fc, struct super_block *sb) F2FS_OPTION(sbi).errors = F2FS_CTX_INFO(ctx).errors; if (ctx->spec_mask & F2FS_SPEC_lookup_mode) F2FS_OPTION(sbi).lookup_mode = F2FS_CTX_INFO(ctx).lookup_mode; + if (ctx->spec_mask & F2FS_SPEC_resizable_tail_secno) + F2FS_OPTION(sbi).resizable_tail_secno = + F2FS_CTX_INFO(ctx).resizable_tail_secno; f2fs_apply_compression(fc, sb); f2fs_apply_test_dummy_encryption(fc, sb); @@ -1771,6 +1792,13 @@ static void f2fs_apply_options(struct fs_context *fc, struct super_block *sb) static int f2fs_sanity_check_options(struct f2fs_sb_info *sbi, bool remount) { + unsigned int total_sections = le32_to_cpu(sbi->raw_super->section_count); + + if (F2FS_OPTION(sbi).resizable_tail_secno >= total_sections) { + f2fs_err(sbi, "Option resizable_tail_secno is larger than or equal to total sections (%u >= %u)", + F2FS_OPTION(sbi).resizable_tail_secno, total_sections); + return -EINVAL; + } if (f2fs_sb_has_device_alias(sbi) && !test_opt(sbi, READ_EXTENT_CACHE)) { f2fs_err(sbi, "device aliasing requires extent cache"); @@ -2058,7 +2086,8 @@ static void f2fs_put_super(struct super_block *sb) if (!get_pages(sbi, i)) continue; f2fs_err(sbi, "detect filesystem reference count leak during " - "umount, type: %d, count: %lld", i, get_pages(sbi, i)); + "umount, type: %d, count: %lld, err: %d, cp_err: %d", + i, get_pages(sbi, i), err, f2fs_cp_error(sbi)); f2fs_bug_on(sbi, 1); } @@ -2456,9 +2485,9 @@ static int f2fs_show_options(struct seq_file *seq, struct dentry *root) seq_puts(seq, "adaptive"); else if (F2FS_OPTION(sbi).fs_mode == FS_MODE_LFS) seq_puts(seq, "lfs"); - else if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_SEG) + else if (f2fs_need_rand_seg(sbi, NO_CHECK_TYPE)) seq_puts(seq, "fragment:segment"); - else if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) + else if (f2fs_need_rand_blk(sbi, NO_CHECK_TYPE)) seq_puts(seq, "fragment:block"); seq_printf(seq, ",active_logs=%u", F2FS_OPTION(sbi).active_logs); if (test_opt(sbi, RESERVE_ROOT) || test_opt(sbi, RESERVE_NODE)) @@ -2543,6 +2572,10 @@ static int f2fs_show_options(struct seq_file *seq, struct dentry *root) else if (F2FS_OPTION(sbi).lookup_mode == LOOKUP_AUTO) seq_show_option(seq, "lookup_mode", "auto"); + if (F2FS_OPTION(sbi).resizable_tail_secno) + seq_printf(seq, ",resizable_tail_secno=%u", + F2FS_OPTION(sbi).resizable_tail_secno); + return 0; } @@ -2585,6 +2618,7 @@ static void default_options(struct f2fs_sb_info *sbi, bool remount) F2FS_OPTION(sbi).bggc_mode = BGGC_MODE_ON; F2FS_OPTION(sbi).memory_mode = MEMORY_MODE_NORMAL; F2FS_OPTION(sbi).errors = MOUNT_ERRORS_CONTINUE; + F2FS_OPTION(sbi).resizable_tail_secno = 0; set_opt(sbi, INLINE_XATTR); set_opt(sbi, INLINE_DATA); @@ -2943,11 +2977,11 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb) if ((flags & SB_RDONLY) || (F2FS_OPTION(sbi).bggc_mode == BGGC_MODE_OFF && !test_opt(sbi, GC_MERGE))) { - if (sbi->gc_thread) { + if (sbi->gc_thread.f2fs_gc_task) { f2fs_stop_gc_thread(sbi); need_restart_gc = true; } - } else if (!sbi->gc_thread) { + } else if (!sbi->gc_thread.f2fs_gc_task) { err = f2fs_start_gc_thread(sbi); if (err) goto restore_opts; @@ -2959,7 +2993,9 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb) set_sbi_flag(sbi, SBI_IS_DIRTY); set_sbi_flag(sbi, SBI_IS_CLOSE); - f2fs_sync_fs(sb, 1); + err = f2fs_sync_fs(sb, 1); + if (err) + goto restore_gc; clear_sbi_flag(sbi, SBI_IS_CLOSE); } @@ -3042,6 +3078,7 @@ skip: sb->s_flags = (sb->s_flags & ~SB_POSIXACL) | (test_opt(sbi, POSIX_ACL) ? SB_POSIXACL : 0); + adjust_pinned_area_boundary(sbi); limit_reserve_root(sbi); fc->sb_flags = (flags & ~SB_LAZYTIME) | (sb->s_flags & SB_LAZYTIME); @@ -3168,7 +3205,7 @@ static ssize_t f2fs_quota_read(struct super_block *sb, int type, char *data, repeat: folio = mapping_read_folio_gfp(mapping, off >> PAGE_SHIFT, - GFP_NOFS); + GFP_KERNEL); if (IS_ERR(folio)) { if (PTR_ERR(folio) == -ENOMEM) { memalloc_retry_wait(GFP_NOFS); @@ -5004,6 +5041,39 @@ static void f2fs_tuning_parameters(struct f2fs_sb_info *sbi) sbi->readdir_ra = true; } +static void f2fs_restore_device_alias(struct f2fs_sb_info *sbi) +{ + struct inode *root = d_inode(sbi->sb->s_root); + struct f2fs_dir_entry *de; + struct folio *folio; + int i; + + if (!f2fs_sb_has_device_alias(sbi)) + return; + + for (i = 1; i < sbi->s_ndevs; i++) { + char *name = strrchr(FDEV(i).path, '/'); + struct inode *inode; + struct qstr qstr; + + name = name ? name + 1 : FDEV(i).path; + qstr.name = name; + qstr.len = strlen(name); + + de = f2fs_find_entry(root, &qstr, &folio); + if (!de) + continue; + + inode = f2fs_iget(sbi->sb, le32_to_cpu(de->ino)); + if (!IS_ERR(inode)) { + if (IS_DEVICE_ALIASING(inode)) + FDEV(i).has_alias = true; + iput(inode); + } + f2fs_folio_put(folio, 0); + } +} + static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) { struct f2fs_fs_context *ctx = fc->fs_private; @@ -5068,6 +5138,7 @@ try_onemore: sb->s_fs_info = sbi; sbi->raw_super = raw_super; + sbi->max_atc_write_bio_size = UINT_MAX; INIT_WORK(&sbi->s_error_work, f2fs_record_error_work); memcpy(sbi->errors, raw_super->s_errors, MAX_F2FS_ERRORS); @@ -5207,6 +5278,7 @@ try_onemore: sbi->last_valid_block_count = sbi->total_valid_block_count; sbi->reserved_blocks = 0; sbi->current_reserved_blocks = 0; + sbi->alias_reserved_blocks = 0; limit_reserve_root(sbi); adjust_unusable_cap_perc(sbi); @@ -5249,6 +5321,8 @@ try_onemore: /* get segno of first zoned block device */ sbi->first_seq_zone_segno = get_first_seq_zone_segno(sbi); + adjust_pinned_area_boundary(sbi); + sbi->reserved_pin_section = f2fs_sb_has_blkzoned(sbi) ? ZONED_PIN_SEC_REQUIRED_COUNT : GET_SEC_FROM_SEG(sbi, overprovision_segments(sbi)); @@ -5434,6 +5508,8 @@ reset_checkpoint: f2fs_update_time(sbi, REQ_TIME); clear_sbi_flag(sbi, SBI_CP_DISABLED_QUICK); + f2fs_restore_device_alias(sbi); + sbi->umount_lock_holder = NULL; return 0; @@ -5701,10 +5777,16 @@ static int __init init_f2fs_fs(void) err = f2fs_init_xattr_cache(); if (err) goto free_casefold_cache; - err = register_filesystem(&f2fs_fs_type); + err = f2fs_init_evict_inode_work(); if (err) goto free_xattr_cache; + err = register_filesystem(&f2fs_fs_type); + if (err) + goto free_evict_inode_cache; return 0; + +free_evict_inode_cache: + f2fs_destroy_evict_inode_work(); free_xattr_cache: f2fs_destroy_xattr_cache(); free_casefold_cache: @@ -5747,6 +5829,7 @@ fail: static void __exit exit_f2fs_fs(void) { unregister_filesystem(&f2fs_fs_type); + f2fs_destroy_evict_inode_work(); f2fs_destroy_xattr_cache(); f2fs_destroy_casefold_cache(); f2fs_destroy_compress_cache(); diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index 6656872..811e350 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -75,7 +75,7 @@ static ssize_t f2fs_sbi_show(struct f2fs_attr *a, static unsigned char *__struct_ptr(struct f2fs_sb_info *sbi, int struct_type) { if (struct_type == GC_THREAD) - return (unsigned char *)sbi->gc_thread; + return (unsigned char *)&sbi->gc_thread; else if (struct_type == SM_INFO) return (unsigned char *)SM_I(sbi); else if (struct_type == DCC_INFO) @@ -557,7 +557,7 @@ out: return -EINVAL; cprc->ckpt_thread_ioprio = IOPRIO_PRIO_VALUE(class, level); - if (test_opt(sbi, MERGE_CHECKPOINT)) { + if (cprc->f2fs_issue_ckpt) { ret = set_task_ioprio(cprc->f2fs_issue_ckpt, cprc->ckpt_thread_ioprio); if (ret) @@ -664,20 +664,20 @@ out: sbi->gc_mode = GC_NORMAL; } else if (t == 1) { sbi->gc_mode = GC_URGENT_HIGH; - if (sbi->gc_thread) { - sbi->gc_thread->gc_wake = true; + if (sbi->gc_thread.f2fs_gc_task) { + sbi->gc_thread.gc_wake = true; wake_up_interruptible_all( - &sbi->gc_thread->gc_wait_queue_head); + &sbi->gc_thread.gc_wait_queue_head); wake_up_discard_thread(sbi, true); } } else if (t == 2) { sbi->gc_mode = GC_URGENT_LOW; } else if (t == 3) { sbi->gc_mode = GC_URGENT_MID; - if (sbi->gc_thread) { - sbi->gc_thread->gc_wake = true; + if (sbi->gc_thread.f2fs_gc_task) { + sbi->gc_thread.gc_wake = true; wake_up_interruptible_all( - &sbi->gc_thread->gc_wait_queue_head); + &sbi->gc_thread.gc_wait_queue_head); } } else { return -EINVAL; @@ -934,14 +934,14 @@ out: if (!strcmp(a->attr.name, "gc_boost_gc_multiple")) { if (t < 1 || t > SEGS_PER_SEC(sbi)) return -EINVAL; - sbi->gc_thread->boost_gc_multiple = (unsigned int)t; + sbi->gc_thread.boost_gc_multiple = (unsigned int)t; return count; } if (!strcmp(a->attr.name, "gc_boost_gc_greedy")) { if (t > GC_GREEDY) return -EINVAL; - sbi->gc_thread->boost_gc_greedy = (unsigned int)t; + sbi->gc_thread.boost_gc_greedy = (unsigned int)t; return count; } @@ -989,8 +989,8 @@ out: if (sbi->cprc_info.f2fs_issue_ckpt) set_user_nice(sbi->cprc_info.f2fs_issue_ckpt, PRIO_TO_NICE(sbi->critical_task_priority)); - if (sbi->gc_thread && sbi->gc_thread->f2fs_gc_task) - set_user_nice(sbi->gc_thread->f2fs_gc_task, + if (sbi->gc_thread.f2fs_gc_task) + set_user_nice(sbi->gc_thread.f2fs_gc_task, PRIO_TO_NICE(sbi->critical_task_priority)); return count; } @@ -1007,13 +1007,15 @@ static ssize_t f2fs_sbi_store(struct f2fs_attr *a, ssize_t ret; bool gc_entry = (!strcmp(a->attr.name, "gc_urgent") || a->struct_type == GC_THREAD); + bool thread_entry = !strcmp(a->attr.name, "ckpt_thread_ioprio") || + !strcmp(a->attr.name, "critical_task_priority"); - if (gc_entry) { + if (gc_entry || thread_entry) { if (!down_read_trylock(&sbi->sb->s_umount)) return -EAGAIN; } ret = __sbi_store(a, sbi, buf, count); - if (gc_entry) + if (gc_entry || thread_entry) up_read(&sbi->sb->s_umount); return ret; @@ -1266,6 +1268,7 @@ F2FS_SBI_RW_ATTR(gc_idle_interval, interval_time[GC_TIME]); F2FS_SBI_RW_ATTR(umount_discard_timeout, interval_time[UMOUNT_DISCARD_TIMEOUT]); F2FS_SBI_RW_ATTR(gc_pin_file_thresh, gc_pin_file_threshold); F2FS_SBI_RW_ATTR(gc_reclaimed_segments, gc_reclaimed_segs); +F2FS_SBI_RW_ATTR(max_atc_write_bio_size, max_atc_write_bio_size); F2FS_SBI_GENERAL_RW_ATTR(max_victim_search); F2FS_SBI_GENERAL_RW_ATTR(migration_granularity); F2FS_SBI_GENERAL_RW_ATTR(migration_window_granularity); @@ -1310,6 +1313,7 @@ F2FS_SBI_GENERAL_RW_ATTR(blkzone_alloc_policy); #endif F2FS_SBI_GENERAL_RW_ATTR(carve_out); F2FS_SBI_GENERAL_RW_ATTR(reserved_pin_section); +F2FS_SBI_GENERAL_RO_ATTR(pinned_area_max_secno); F2FS_SBI_GENERAL_RW_ATTR(bggc_io_aware); F2FS_SBI_GENERAL_RW_ATTR(max_lock_elapsed_time); F2FS_SBI_GENERAL_RW_ATTR(lock_duration_priority); @@ -1509,6 +1513,7 @@ static struct attribute *f2fs_attrs[] = { ATTR_LIST(seq_file_ra_mul), ATTR_LIST(gc_segment_mode), ATTR_LIST(gc_reclaimed_segments), + ATTR_LIST(max_atc_write_bio_size), ATTR_LIST(max_fragment_chunk), ATTR_LIST(max_fragment_hole), ATTR_LIST(current_atomic_write), @@ -1521,6 +1526,7 @@ static struct attribute *f2fs_attrs[] = { ATTR_LIST(max_read_extent_count), ATTR_LIST(carve_out), ATTR_LIST(reserved_pin_section), + ATTR_LIST(pinned_area_max_secno), ATTR_LIST(allocate_section_hint), ATTR_LIST(allocate_section_policy), ATTR_LIST(max_lock_elapsed_time), diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c index ed33e51..6728d14 100644 --- a/fs/f2fs/xattr.c +++ b/fs/f2fs/xattr.c @@ -119,7 +119,7 @@ static int f2fs_xattr_advise_set(const struct xattr_handler *handler, unsigned char old_advise = F2FS_I(inode)->i_advise; unsigned char new_advise; - if (!inode_owner_or_capable(&nop_mnt_idmap, inode)) + if (!inode_owner_or_capable(idmap, inode)) return -EPERM; if (value == NULL) return -EINVAL; diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h index 270c1a2..d53be93 100644 --- a/include/trace/events/f2fs.h +++ b/include/trace/events/f2fs.h @@ -191,7 +191,8 @@ TRACE_DEFINE_ENUM(CP_PHASE_FINISH_CHECKPOINT); { LOCK_NAME_NODE_WRITE, "node_write" }, \ { LOCK_NAME_GC_LOCK, "gc_lock" }, \ { LOCK_NAME_CP_GLOBAL, "cp_global" }, \ - { LOCK_NAME_IO_RWSEM, "io_rwsem" }) + { LOCK_NAME_IO_RWSEM, "io_rwsem" }, \ + { LOCK_NAME_NAT_TREE_LOCK, "nat_tree_lock" }) struct f2fs_sb_info; struct f2fs_io_info; @@ -2114,6 +2115,14 @@ DEFINE_EVENT(f2fs_zip_end, f2fs_decompress_pages_end, ); #ifdef CONFIG_F2FS_IOSTAT +/* + * Number of read folio order buckets emitted by the f2fs_iostat tracepoint. + * TP_printk() cannot loop, so the field count is fixed here and must be >= + * the largest possible NR_PAGE_ORDERS (14 on arm64 with 64K pages). The + * BUILD_BUG_ON() in f2fs_update_read_folio_count() enforces this. + */ +#define F2FS_IOSTAT_RD_FOLIO_ORDERS 16 + TRACE_EVENT(f2fs_iostat, TP_PROTO(struct f2fs_sb_info *sbi, unsigned long long *iostat, @@ -2151,7 +2160,7 @@ TRACE_EVENT(f2fs_iostat, __field(unsigned long long, fs_mrio) __field(unsigned long long, fs_discard) __field(unsigned long long, fs_reset_zone) - __array(unsigned long long, read_folio_count, 11) + __array(unsigned long long, read_folio_count, F2FS_IOSTAT_RD_FOLIO_ORDERS) ), TP_fast_assign( @@ -2186,7 +2195,8 @@ TRACE_EVENT(f2fs_iostat, __entry->fs_reset_zone = iostat[FS_ZONE_RESET_IO]; memset(__entry->read_folio_count, 0, sizeof(__entry->read_folio_count)); memcpy(__entry->read_folio_count, read_folio_count, - sizeof(unsigned long long) * min_t(int, NR_PAGE_ORDERS, 11)); + sizeof(unsigned long long) * + min_t(int, NR_PAGE_ORDERS, F2FS_IOSTAT_RD_FOLIO_ORDERS)); ), TP_printk("dev = (%d,%d), " @@ -2201,7 +2211,8 @@ TRACE_EVENT(f2fs_iostat, "fs [data=%llu, (gc_data=%llu, cdata=%llu), " "node=%llu, meta=%llu], " "read_folio_count [0=%llu, 1=%llu, 2=%llu, 3=%llu, 4=%llu, " - "5=%llu, 6=%llu, 7=%llu, 8=%llu, 9=%llu, 10=%llu]", + "5=%llu, 6=%llu, 7=%llu, 8=%llu, 9=%llu, 10=%llu, 11=%llu, " + "12=%llu, 13=%llu, 14=%llu, 15=%llu]", show_dev(__entry->dev), __entry->app_wio, __entry->app_dio, __entry->app_bio, __entry->app_mio, __entry->app_bcdio, __entry->app_mcdio, __entry->fs_dio, __entry->fs_cdio, @@ -2218,7 +2229,9 @@ TRACE_EVENT(f2fs_iostat, __entry->read_folio_count[4], __entry->read_folio_count[5], __entry->read_folio_count[6], __entry->read_folio_count[7], __entry->read_folio_count[8], __entry->read_folio_count[9], - __entry->read_folio_count[10]) + __entry->read_folio_count[10], __entry->read_folio_count[11], + __entry->read_folio_count[12], __entry->read_folio_count[13], + __entry->read_folio_count[14], __entry->read_folio_count[15]) ); #ifndef __F2FS_IOSTAT_LATENCY_TYPE diff --git a/include/uapi/linux/f2fs.h b/include/uapi/linux/f2fs.h index 795e262..4409ada 100644 --- a/include/uapi/linux/f2fs.h +++ b/include/uapi/linux/f2fs.h @@ -45,6 +45,9 @@ #define F2FS_IOC_START_ATOMIC_REPLACE _IO(F2FS_IOCTL_MAGIC, 25) #define F2FS_IOC_GET_DEV_ALIAS_FILE _IOR(F2FS_IOCTL_MAGIC, 26, __u32) #define F2FS_IOC_IO_PRIO _IOW(F2FS_IOCTL_MAGIC, 27, __u32) +#define F2FS_IOC_RESERVE_DEV_ALIAS _IO(F2FS_IOCTL_MAGIC, 28) +#define F2FS_IOC_RELEASE_DEV_ALIAS _IO(F2FS_IOCTL_MAGIC, 29) +#define F2FS_IOC_GET_DEV_ALIAS_STATUS _IOR(F2FS_IOCTL_MAGIC, 30, __u32) /* * should be same as XFS_IOC_GOINGDOWN. @@ -70,6 +73,10 @@ enum { F2FS_IOPRIO_MAX, }; +/* for F2FS_IOC_GET_DEV_ALIAS_STATUS */ +#define F2FS_DEV_ALIAS_STATUS_RELEASED 0 +#define F2FS_DEV_ALIAS_STATUS_RESERVED 1 + struct f2fs_gc_range { __u32 sync; __u64 start;