diff --git a/.mailmap b/.mailmap index 1f5540b..11750e5 100644 --- a/.mailmap +++ b/.mailmap @@ -355,6 +355,8 @@ Hans Verkuil Hans Verkuil Hans Verkuil Hao Ge +Haowen Bai +Haowen Bai Harry Yoo <42.hyeyoo@gmail.com> Harry Yoo Heiko Carstens @@ -982,6 +984,8 @@ Wesley Cheng Will Deacon Wolfram Sang Wolfram Sang +Xu Xin +Xu Xin xu xin Yakir Yang Yanteng Si Ying Huang diff --git a/MAINTAINERS b/MAINTAINERS index cc3cae2..140eafc 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17194,7 +17194,7 @@ F: tools/testing/selftests/mm/gup_test.c MEMORY MANAGEMENT - KSM (Kernel Samepage Merging) M: Andrew Morton M: David Hildenbrand -R: Xu Xin +R: Xu Xin R: Chengming Zhou L: linux-mm@kvack.org S: Maintained @@ -17242,6 +17242,8 @@ R: Barry Song R: Axel Rasmussen R: Yuanchu Xie R: Wei Xu +R: Baoquan He +R: Baolin Wang L: linux-mm@kvack.org S: Maintained W: http://www.linux-mm.org diff --git a/fs/ocfs2/namei.c b/fs/ocfs2/namei.c index e9c7774..58c6061 100644 --- a/fs/ocfs2/namei.c +++ b/fs/ocfs2/namei.c @@ -336,13 +336,8 @@ static int ocfs2_mknod(struct mnt_idmap *idmap, goto leave; /* calculate meta data/clusters for setting security and acl xattr */ - status = ocfs2_calc_xattr_init(dir, mode, &si, &want_clusters, - &xattr_credits, &want_meta, - &acl_state); - if (status < 0) { - mlog_errno(status); - goto leave; - } + ocfs2_calc_xattr_init(dir, mode, &si, &want_clusters, &xattr_credits, + &want_meta, &acl_state); /* Reserve a cluster if creating an extent based directory. */ if (S_ISDIR(mode) && !ocfs2_supports_inline_data(osb)) { diff --git a/fs/ocfs2/xattr.c b/fs/ocfs2/xattr.c index 35bcbb0..bfafe05 100644 --- a/fs/ocfs2/xattr.c +++ b/fs/ocfs2/xattr.c @@ -635,12 +635,11 @@ int ocfs2_calc_security_init(struct inode *dir, return ret; } -int ocfs2_calc_xattr_init(struct inode *dir, umode_t mode, - struct ocfs2_security_xattr_info *si, - int *want_clusters, int *xattr_credits, - int *want_meta, struct ocfs2_acl_state *acl_state) +void ocfs2_calc_xattr_init(struct inode *dir, umode_t mode, + struct ocfs2_security_xattr_info *si, + int *want_clusters, int *xattr_credits, + int *want_meta, struct ocfs2_acl_state *acl_state) { - int ret = 0; struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); int s_size = 0, a_size = 0, acl_len = 0, new_clusters; @@ -662,7 +661,7 @@ int ocfs2_calc_xattr_init(struct inode *dir, umode_t mode, } if (!(s_size + a_size)) - return ret; + return; /* * The max space of security xattr taken inline is @@ -728,8 +727,6 @@ int ocfs2_calc_xattr_init(struct inode *dir, umode_t mode, } } } - - return ret; } static int ocfs2_xattr_extend_allocation(struct inode *inode, diff --git a/fs/ocfs2/xattr.h b/fs/ocfs2/xattr.h index 5e18513..887cc1a 100644 --- a/fs/ocfs2/xattr.h +++ b/fs/ocfs2/xattr.h @@ -59,10 +59,10 @@ int ocfs2_calc_security_init(struct inode *, int *, int *, struct ocfs2_alloc_context **); struct ocfs2_acl_state; -int ocfs2_calc_xattr_init(struct inode *dir, umode_t mode, - struct ocfs2_security_xattr_info *si, - int *want_clusters, int *xattr_credits, - int *want_meta, struct ocfs2_acl_state *acl_state); +void ocfs2_calc_xattr_init(struct inode *dir, umode_t mode, + struct ocfs2_security_xattr_info *si, + int *want_clusters, int *xattr_credits, + int *want_meta, struct ocfs2_acl_state *acl_state); /* * xattrs can live inside an inode, as part of an external xattr block, diff --git a/mm/backing-dev.c b/mm/backing-dev.c index cecbcf9..18e9990 100644 --- a/mm/backing-dev.c +++ b/mm/backing-dev.c @@ -910,8 +910,9 @@ static void cleanup_offline_cgwbs_workfn(struct work_struct *work) continue; spin_unlock_irq(&cgwb_lock); - while (cleanup_offline_cgwb(wb)) - cond_resched(); + do { + cond_resched_tasks_rcu_qs(); + } while (cleanup_offline_cgwb(wb)); spin_lock_irq(&cgwb_lock); wb_put(wb); diff --git a/mm/damon/core.c b/mm/damon/core.c index 644daf5..1764620 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2342,36 +2342,39 @@ static bool damos_skip_charged_region(struct damon_target *t, { struct damos_quota *quota = &s->quota; unsigned long sz_to_skip; + bool skip = false; /* Skip previously charged regions */ if (quota->charge_target_from) { if (t != quota->charge_target_from) return true; - if (r == damon_last_region(t)) { - quota->charge_target_from = NULL; - quota->charge_addr_from = 0; - return true; - } if (quota->charge_addr_from && - r->ar.end <= quota->charge_addr_from) - return true; + r->ar.end <= quota->charge_addr_from) { + skip = true; + goto out; + } if (quota->charge_addr_from && r->ar.start < quota->charge_addr_from) { sz_to_skip = ALIGN_DOWN(quota->charge_addr_from - r->ar.start, min_region_sz); if (!sz_to_skip) { - if (damon_sz_region(r) <= min_region_sz) - return true; + if (damon_sz_region(r) <= min_region_sz) { + skip = true; + goto out; + } sz_to_skip = min_region_sz; } damon_split_region_at(t, r, sz_to_skip); - return true; + skip = true; } + } +out: + if (r == damon_last_region(t)) { quota->charge_target_from = NULL; quota->charge_addr_from = 0; } - return false; + return skip; } static void damos_update_stat(struct damos *s, @@ -3091,6 +3094,7 @@ static void damos_set_effective_quota(struct damon_ctx *ctx, struct damos *s) struct damos_quota *quota = &s->quota; unsigned long throughput; unsigned long esz = ULONG_MAX; + unsigned long esz_time; if (!quota->ms && list_empty("a->goals)) { quota->esz = quota->sz; @@ -3111,8 +3115,8 @@ static void damos_set_effective_quota(struct damon_ctx *ctx, struct damos *s) 1000000, quota->total_charged_ns); else throughput = PAGE_SIZE * 1024; - esz = min(throughput * quota->ms, esz); - esz = max(ctx->min_region_sz, esz); + esz_time = max(throughput * quota->ms, ctx->min_region_sz); + esz = min(esz_time, esz); } if (quota->sz && quota->sz < esz) @@ -3241,8 +3245,15 @@ static void kdamond_apply_schemes(struct damon_ctx *c) max_region_sz = damon_region_sz_limit(c); mutex_lock(&c->walk_control_lock); damon_for_each_target(t, c) { - if (c->ops.target_valid && c->ops.target_valid(t) == false) + if (c->ops.target_valid && c->ops.target_valid(t) == false) { + damon_for_each_scheme(s, c) { + if (s->quota.charge_target_from != t) + continue; + s->quota.charge_target_from = NULL; + s->quota.charge_addr_from = 0; + } continue; + } damos_apply_target(c, t, max_region_sz); } diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index fbda70d..8fc61d0 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -61,7 +61,12 @@ void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr * device aspects. */ if (likely(pte_present(pteval))) - young |= ptep_test_and_clear_young(vma, addr, pte); + /* + * Arch implementation of ptep_test_and_clear_young() may + * require aligned @addr + */ + young |= ptep_test_and_clear_young(vma, PAGE_ALIGN_DOWN(addr), + pte); young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + PAGE_SIZE); if (young) folio_set_young(folio); diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 0648400..04ee2a2 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -293,22 +293,29 @@ out: } #ifdef CONFIG_HUGETLB_PAGE +static bool damon_hugetlb_ptep_mkold(pte_t *pte, struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr, pte_t *entry) +{ + unsigned long psize = huge_page_size(hstate_vma(vma)); + + if (!pte_young(*entry)) + return false; + *entry = huge_ptep_get_and_clear(mm, addr, pte, psize); + *entry = pte_mkold(*entry); + set_huge_pte_at(mm, addr, pte, *entry, psize); + return true; +} + static void damon_hugetlb_mkold(pte_t *pte, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr) { bool referenced = false; pte_t entry = huge_ptep_get(mm, addr, pte); struct folio *folio = pfn_folio(pte_pfn(entry)); - unsigned long psize = huge_page_size(hstate_vma(vma)); folio_get(folio); - if (pte_young(entry)) { - referenced = true; - entry = pte_mkold(entry); - set_huge_pte_at(mm, addr, pte, entry, psize); - } - + referenced = damon_hugetlb_ptep_mkold(pte, mm, vma, addr, &entry); if (mmu_notifier_clear_young(mm, addr, addr + huge_page_size(hstate_vma(vma)))) referenced = true; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 4f6f58b..cea2577 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1967,6 +1967,15 @@ retry: struct hstate *h = folio_hstate(folio); bool adjust_surplus = false; + /* + * remove_hugetlb_folio()/update_and_free_hugetlb_folio() bail + * for gigantic hstates without runtime support, so dissolving one + * here would leave it on the free list and, on vmemmap restore + * failure, the add_hugetlb_folio() rollback corrupts that list. + */ + if (hstate_is_gigantic_no_runtime(h)) + goto out; + if (!available_huge_pages(h)) goto out; @@ -5161,18 +5170,21 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, hugetlb_vma_lock_write(vma); i_mmap_lock_write(mapping); for (; old_addr < old_end; old_addr += sz, new_addr += sz) { + const unsigned long offset_to_last_entry = + (old_addr | last_addr_mask) - old_addr; + src_pte = hugetlb_walk(vma, old_addr, sz); if (!src_pte) { - old_addr |= last_addr_mask; - new_addr |= last_addr_mask; + old_addr += offset_to_last_entry; + new_addr += offset_to_last_entry; continue; } if (huge_pte_none(huge_ptep_get(mm, old_addr, src_pte))) continue; if (huge_pmd_unshare(&tlb, vma, old_addr, src_pte)) { - old_addr |= last_addr_mask; - new_addr |= last_addr_mask; + old_addr += offset_to_last_entry; + new_addr += offset_to_last_entry; continue; } diff --git a/mm/rmap.c b/mm/rmap.c index d1819fd..f3b21aa 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -209,7 +209,11 @@ int __anon_vma_prepare(struct vm_area_struct *vma) /* page_table_lock to protect against threads */ spin_lock(&mm->page_table_lock); if (likely(!vma->anon_vma)) { - vma->anon_vma = anon_vma; + /* + * Make anon_vma fields visible before anon_vma is published. + * Paired with an address dependency in reusable_anon_vma(). + */ + smp_store_release(&vma->anon_vma, anon_vma); anon_vma_chain_assign(vma, avc, anon_vma); anon_rmap_tree_insert(avc, anon_vma); anon_vma->num_active_vmas++; diff --git a/mm/vma.c b/mm/vma.c index f29abb3..9f0a0ac 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -2094,6 +2094,13 @@ static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct * * acceptable for merging, so we can do all of this optimistically. But * we do that READ_ONCE() to make sure that we never re-load the pointer. * + * The READ_ONCE() establishes an address dependency between anon_vma and + * any access to its fields, which pairs with the assignment to + * vma->anon_vma performed with release semantics in __anon_vma_prepare(). + * + * This is especially important as anon_vma's are SLAB_TYPESAFE_BY_RCU so + * accessing an uninitialised anon_vma's fields may result in a UAF. + * * IOW: that the "list_is_singular()" test on the anon_vma_chain only * matters for the 'stable anon_vma' case (ie the thing we want to avoid * is to return an anon_vma that is "complex" due to having gone through @@ -2108,6 +2115,7 @@ static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *b) { if (anon_vma_compatible(a, b)) { + /* Paired with a memory barrier in __anon_vma_prepare(). */ struct anon_vma *anon_vma = READ_ONCE(old->anon_vma); if (anon_vma && list_is_singular(&old->anon_vma_chain)) diff --git a/tools/testing/selftests/cgroup/test_zswap.c b/tools/testing/selftests/cgroup/test_zswap.c index 609c48f..8df54b5 100644 --- a/tools/testing/selftests/cgroup/test_zswap.c +++ b/tools/testing/selftests/cgroup/test_zswap.c @@ -20,6 +20,7 @@ static int page_size; #define PATH_ZSWAP "/sys/module/zswap" #define PATH_ZSWAP_ENABLED "/sys/module/zswap/parameters/enabled" +#define PATH_ZSWAP_SHRINKER_ENABLED "/sys/module/zswap/parameters/shrinker_enabled" #define PATH_ZSWAP_STORED_PAGES "/sys/kernel/debug/zswap/stored_pages" static int read_int(const char *path, size_t *value) @@ -444,6 +445,16 @@ static int test_zswap_writeback_disabled(const char *root) return test_zswap_writeback(root, false); } +static bool zswap_shrinker_enabled(void) +{ + char value[2]; + + if (read_text(PATH_ZSWAP_SHRINKER_ENABLED, value, sizeof(value)) <= 0) + return 0; + + return value[0] == 'Y'; +} + /* * When trying to store a memcg page in zswap, if the memcg hits its memory * limit in zswap, writeback should affect only the zswapped pages of that @@ -453,6 +464,7 @@ static int test_no_invasive_cgroup_shrink(const char *root) { int ret = KSFT_FAIL; unsigned int off; + long zswpwb_before, zswpwb_after, zswpwb_target; size_t allocation_size = page_size * 1024; unsigned int nr_pages = allocation_size / page_size; char zswap_max_buf[32], mem_max_buf[32]; @@ -488,6 +500,14 @@ static int test_no_invasive_cgroup_shrink(const char *root) if (cg_read_key_long(zw_group, "memory.stat", "zswapped") < 1) goto out; + /* If the shrinker is enabled, try to let the writebacks finish first */ + if (zswap_shrinker_enabled()) + sleep(5); + + zswpwb_before = get_cg_wb_count(zw_group); + if (zswpwb_before < 0) + goto out; + /* Push wb_group memory into zswap with hard-to-compress data to trigger wb */ if (cg_enter_current(wb_group)) goto out; @@ -500,9 +520,13 @@ static int test_no_invasive_cgroup_shrink(const char *root) getrandom(&wb_allocation[off], page_size/4, 0); } - /* Verify that only zswapped memory from gwb_group has been written back */ - if (wait_for_writeback(wb_group, 5000) > 0 && get_cg_wb_count(zw_group) == 0) + /* Verify that only zswapped memory from wb_group has been written back */ + zswpwb_target = wait_for_writeback(wb_group, 5000); + zswpwb_after = get_cg_wb_count(zw_group); + + if (zswpwb_target > 0 && zswpwb_before == zswpwb_after) ret = KSFT_PASS; + out: cg_enter_current(root); if (zw_group) {