diff --git a/.mailmap b/.mailmap index 34625ea..0a0be49 100644 --- a/.mailmap +++ b/.mailmap @@ -101,6 +101,7 @@ Andy Chiu Andy Chiu Andy Shevchenko Andy Shevchenko +Andy Yan Anilkumar Kolli Anirudh Ghayal Antoine Tenart @@ -461,7 +462,8 @@ Johan Hovold Johan Hovold John Crispin John Fastabend -John Garry +John Garry +John Garry John Keeping John Moon John Paul Adrian Glaubitz diff --git a/drivers/char/mem.c b/drivers/char/mem.c index 63253d1..5b93c92 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c @@ -503,7 +503,7 @@ static int mmap_zero_prepare(struct vm_area_desc *desc) #ifndef CONFIG_MMU return -ENOSYS; #endif - if (vma_desc_test(desc, VMA_SHARED_BIT)) + if (vma_desc_test(desc, VMA_MAYSHARE_BIT)) return shmem_zero_setup_desc(desc); /* diff --git a/mm/mremap.c b/mm/mremap.c index 7c36844..444f37d 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -1037,6 +1037,7 @@ static void vrm_stat_account(struct vma_remap_struct *vrm, } static bool __check_map_count_against_split(struct mm_struct *mm, + bool pre_split, bool before_unmaps) { const int sys_map_count = get_sysctl_max_map_count(); @@ -1088,26 +1089,42 @@ static bool __check_map_count_against_split(struct mm_struct *mm, * Therefore we must check to ensure we have headroom of 2 additional * VMAs. */ - return map_count + 2 <= sys_map_count; + map_count += 2; + + /* If pre-split, the -1 observed above doesn't apply. */ + if (pre_split) + map_count++; + + return map_count <= sys_map_count; +} + +static bool needs_pre_split(struct vma_remap_struct *vrm) +{ + /* + * An MREMAP_DONTUNMAP of a mlock()'d VMA needs to unlock the + * source VMA, so split in this case. + */ + return (vrm->flags & MREMAP_DONTUNMAP) && + vma_test(vrm->vma, VMA_LOCKED_BIT); } /* Do we violate the map count limit if we split VMAs when moving the VMA? */ -static bool check_map_count_against_split(void) +static bool check_map_count_against_split(struct vma_remap_struct *vrm) { return __check_map_count_against_split(current->mm, - /*before_unmaps=*/false); + needs_pre_split(vrm), /*before_unmaps=*/false); } /* Do we violate the map count limit if we split VMAs prior to early unmaps? */ -static bool check_map_count_against_split_early(void) +static bool check_map_count_against_split_early(struct vma_remap_struct *vrm) { return __check_map_count_against_split(current->mm, - /*before_unmaps=*/true); + vrm->flags & MREMAP_DONTUNMAP, /*before_unmaps=*/true); } /* - * Perform checks before attempting to write a VMA prior to it being - * moved. + * Perform checks and preparation before attempting to write a VMA prior to it + * being moved. */ static unsigned long prep_move_vma(struct vma_remap_struct *vrm) { @@ -1116,19 +1133,17 @@ static unsigned long prep_move_vma(struct vma_remap_struct *vrm) unsigned long old_addr = vrm->addr; unsigned long old_len = vrm->old_len; vm_flags_t dummy = vma->vm_flags; + const bool split_before = vma->vm_start != old_addr; + const bool split_after = vma->vm_end != old_addr + old_len; - /* - * We'd prefer to avoid failure later on in do_munmap: we copy a VMA, - * which may not merge, then (if MREMAP_DONTUNMAP is not set) unmap the - * source, which may split, causing a net increase of 2 mappings. - */ - if (!check_map_count_against_split()) + /* Avoid failure later on. */ + if (!check_map_count_against_split(vrm)) return -ENOMEM; if (vma->vm_ops && vma->vm_ops->may_split) { - if (vma->vm_start != old_addr) + if (split_before) err = vma->vm_ops->may_split(vma, old_addr); - if (!err && vma->vm_end != old_addr + old_len) + if (!err && split_after) err = vma->vm_ops->may_split(vma, old_addr + old_len); if (err) return err; @@ -1146,6 +1161,21 @@ static unsigned long prep_move_vma(struct vma_remap_struct *vrm) if (err) return err; + /* + * To account mlock()'d pages correctly in the MREMAP_DONTUNMAP + * case perform any split ahead of time for an mlock()'d VMA. + */ + if (needs_pre_split(vrm)) { + VMA_ITERATOR(vmi, vma->vm_mm, old_addr); + + if (split_before) + err = split_vma(&vmi, vma, old_addr, 1); + if (!err && split_after) + err = split_vma(&vmi, vma, old_addr + old_len, 0); + vrm->vmi_needs_invalidate = true; + return err; + } + return 0; } @@ -1275,7 +1305,8 @@ static int copy_vma_and_data(struct vma_remap_struct *vrm, PAGETABLE_MOVE(pmc, NULL, NULL, vrm->addr, vrm->new_addr, vrm->old_len); new_vma = copy_vma(&vma, vrm->new_addr, vrm->new_len, new_pgoff, - new_anon_pgoff, &pmc.need_rmap_locks); + new_anon_pgoff, &pmc.need_rmap_locks, + vrm->flags & MREMAP_DONTUNMAP); if (!new_vma) { vrm_uncharge(vrm); *new_vma_ptr = NULL; @@ -1335,6 +1366,9 @@ static void dontunmap_complete(struct vma_remap_struct *vrm, unsigned long old_start = vma->vm_start; unsigned long old_end = vma->vm_end; + /* Self-merge is disallowed. */ + VM_WARN_ON_ONCE(new_vma == vma); + /* We always clear VMA_LOCKED[ONFAULT]_BIT on the old VMA. */ vma_clear_flags_mask(vma, VMA_LOCKED_MASK); @@ -1342,7 +1376,7 @@ static void dontunmap_complete(struct vma_remap_struct *vrm, * anon_vma links of the old vma is no longer needed after its page * table has been moved. */ - if (new_vma != vma && start == old_start && end == old_end) { + if (start == old_start && end == old_end) { const pgoff_t pgoff_unfaulted = vma->vm_start >> PAGE_SHIFT; unlink_anon_vmas(vma); @@ -2002,7 +2036,7 @@ static unsigned long do_mremap(struct vma_remap_struct *vrm) return -EINTR; vrm->mmap_locked = true; - if (!check_map_count_against_split_early()) { + if (!check_map_count_against_split_early(vrm)) { mmap_write_unlock(mm); return -ENOMEM; } diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index b91b1a9..cd227fc 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -438,12 +438,30 @@ static void kernel_pgtable_work_func(struct work_struct *work) __pagetable_free(pt); } +static void schedule_kernel_pgtable_free(void) +{ + schedule_work(&kernel_pgtable_work.work); +} + void pagetable_free_kernel(struct ptdesc *pt) { spin_lock(&kernel_pgtable_work.lock); list_add(&pt->pt_list, &kernel_pgtable_work.list); spin_unlock(&kernel_pgtable_work.lock); - schedule_work(&kernel_pgtable_work.work); + /* + * The workqueue may not exist yet while the system is booting. + * kernel_pgtable_drain_early() schedules the work once it does. + */ + if (system_state != SYSTEM_BOOTING) + schedule_kernel_pgtable_free(); +} + +static int __init kernel_pgtable_drain_early(void) +{ + /* Free the kernel page tables queued while booting. */ + schedule_kernel_pgtable_free(); + return 0; } +core_initcall(kernel_pgtable_drain_early); #endif diff --git a/mm/vma.c b/mm/vma.c index 6cde678..6c35f0d 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -634,8 +634,8 @@ out_free_vma: * Split a vma into two pieces at address 'addr', a new vma is allocated * either for the first part or the tail. */ -static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, - unsigned long addr, int new_below) +int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, + unsigned long addr, int new_below) { if (vma->vm_mm->map_count >= get_sysctl_max_map_count()) return -ENOMEM; @@ -1943,7 +1943,7 @@ static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) */ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, unsigned long addr, unsigned long len, pgoff_t pgoff, - pgoff_t anon_pgoff, bool *need_rmap_locks) + pgoff_t anon_pgoff, bool *need_rmap_locks, bool keep_source) { struct vm_area_struct *vma = *vmap; unsigned long old_vma_start = vma->vm_start; @@ -1981,6 +1981,21 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, vmg.pgoff = pgoff; vmg.anon_pgoff = anon_pgoff; vmg.next = vma_iter_next_rewind(&vmi, NULL); + + /* + * If the original VMA is kept (MREMAP_DONTUNMAP), the source and + * destination VMA must be treated distinctly. + * + * A merge violates this, so in this case disallow a self-merge. + */ + if (can_self_merge && keep_source) { + if (vmg.prev == vma) + vmg.prev = NULL; + if (vmg.next == vma) + vmg.next = NULL; + can_self_merge = false; + } + new_vma = vma_merge_copied_range(&vmg); if (new_vma) { diff --git a/mm/vma.h b/mm/vma.h index 024fabe..36973ab 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -535,7 +535,7 @@ void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb, struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, unsigned long addr, unsigned long len, pgoff_t pgoff, - pgoff_t anon_pgoff, bool *need_rmap_locks); + pgoff_t anon_pgoff, bool *need_rmap_locks, bool keep_source); struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma); @@ -556,6 +556,11 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma, unsigned long unmapped_area(struct vm_unmapped_area_info *info); unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info); +#ifdef CONFIG_MMU +int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, + unsigned long addr, int new_below); +#endif + static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma) { /* diff --git a/tools/testing/selftests/mm/hugetlb-mmap.c b/tools/testing/selftests/mm/hugetlb-mmap.c index 0f2aad1..2edbc99 100644 --- a/tools/testing/selftests/mm/hugetlb-mmap.c +++ b/tools/testing/selftests/mm/hugetlb-mmap.c @@ -122,7 +122,7 @@ int main(int argc, char **argv) hugepage_size = default_huge_page_size(); if (!hugepage_size) ksft_exit_skip("Could not detect default hugetlb page size."); - ksft_print_msg("Default size hugepages (%lu kB)\n", hugepage_size >> 10); + ksft_print_msg("Default size hugepages (%zu kB)\n", hugepage_size >> 10); } /* munmap will fail if the length is not page aligned */ @@ -131,7 +131,7 @@ int main(int argc, char **argv) hugetlb_set_nr_pages(hugepage_size, nr); if (hugetlb_free_pages(hugepage_size) < nr) - ksft_exit_skip("Not enough %lu Kb pages\n", hugepage_size >> 10); + ksft_exit_skip("Not enough %zu Kb pages\n", hugepage_size >> 10); ksft_set_plan(2); ksft_print_msg("Mapping %lu Mbytes\n", (unsigned long)length >> 20); diff --git a/tools/testing/vma/tests/vma.c b/tools/testing/vma/tests/vma.c index c8ef7b8..e973e0a 100644 --- a/tools/testing/vma/tests/vma.c +++ b/tools/testing/vma/tests/vma.c @@ -40,7 +40,7 @@ static bool test_copy_vma(void) vma = alloc_and_link_vma(&mm, 0x1000, 0x2000, 1, vma_flags); vma_set_anonymous(vma); vma_orig = vma; - vma_new = copy_vma(&vma, 0x2000, 0x1000, 1, 1, &need_locks); + vma_new = copy_vma(&vma, 0x2000, 0x1000, 1, 1, &need_locks, false); ASSERT_EQ(vma_new, vma_orig); ASSERT_EQ(vma, vma_orig); ASSERT_EQ(vma_new->vm_start, 0x1000); @@ -53,7 +53,7 @@ static bool test_copy_vma(void) vma = alloc_and_link_vma(&mm, 0x2000, 0x3000, 2, vma_flags); vma_set_anonymous(vma); vma_orig = vma; - vma_new = copy_vma(&vma, 0x1000, 0x1000, 2, 2, &need_locks); + vma_new = copy_vma(&vma, 0x1000, 0x1000, 2, 2, &need_locks, false); ASSERT_EQ(vma_new, vma_orig); ASSERT_EQ(vma, vma_orig); ASSERT_EQ(vma_new->vm_start, 0x1000); @@ -71,7 +71,7 @@ static bool test_copy_vma(void) vma = alloc_and_link_vma(&mm, 0x3000, 0x4000, 3, vma_flags); vma_set_anonymous(vma); vma_orig = vma; - vma_new = copy_vma(&vma, 0x2000, 0x1000, 3, 3, &need_locks); + vma_new = copy_vma(&vma, 0x2000, 0x1000, 3, 3, &need_locks, false); ASSERT_NE(vma_new, vma_orig); ASSERT_EQ(vma_new, vma); ASSERT_EQ(vma_new->vm_start, 0x1000); @@ -82,7 +82,7 @@ static bool test_copy_vma(void) /* Move backwards and do not merge. */ vma = alloc_and_link_vma(&mm, 0x3000, 0x5000, 3, vma_flags); - vma_new = copy_vma(&vma, 0, 0x2000, 0, 3, &need_locks); + vma_new = copy_vma(&vma, 0, 0x2000, 0, 3, &need_locks, false); ASSERT_NE(vma_new, vma); ASSERT_EQ(vma_new->vm_start, 0); ASSERT_EQ(vma_new->vm_end, 0x2000); @@ -95,7 +95,7 @@ static bool test_copy_vma(void) vma = alloc_and_link_vma(&mm, 0, 0x2000, 0, vma_flags); vma_next = alloc_and_link_vma(&mm, 0x6000, 0x8000, 6, vma_flags); - vma_new = copy_vma(&vma, 0x4000, 0x2000, 4, 4, &need_locks); + vma_new = copy_vma(&vma, 0x4000, 0x2000, 4, 4, &need_locks, false); vma_assert_attached(vma_new); ASSERT_EQ(vma_new, vma_next);