projects/linux·blob assoc-array-fixes · refs

157 lines · 3.9 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2
3/*
4 * Functions for initializing, allocating, freeing and duplicating VMAs. Shared
5 * between CONFIG_MMU and non-CONFIG_MMU kernel configurations.
6 */
7
8/*
9 * To allow for userland testing we place internal dependencies in
10 * vma_internal.h and external VMA API declarations in vma.h.
11 */
12#include "vma_internal.h"
13#include "vma.h"
14
15/* SLAB cache for vm_area_struct structures */
16static struct kmem_cache *vm_area_cachep;
17
18void __init vma_state_init(void)
19{
20 struct kmem_cache_args args = {
21 .use_freeptr_offset = true,
22 .freeptr_offset = offsetof(struct vm_area_struct, vm_freeptr),
23 .sheaf_capacity = 32,
24 };
25
26 vm_area_cachep = kmem_cache_create("vm_area_struct",
27 sizeof(struct vm_area_struct), &args,
28 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU|
29 SLAB_ACCOUNT);
30}
31
32struct vm_area_struct *vm_area_alloc(struct mm_struct *mm)
33{
34 struct vm_area_struct *vma;
35
36 vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
37 if (!vma)
38 return NULL;
39
40 vma_init(vma, mm);
41
42 return vma;
43}
44
45static void vm_area_init_from(const struct vm_area_struct *src,
46 struct vm_area_struct *dest)
47{
48 dest->vm_mm = src->vm_mm;
49 dest->vm_ops = src->vm_ops;
50 dest->vm_start = src->vm_start;
51 dest->vm_end = src->vm_end;
52 dest->anon_vma = src->anon_vma;
53 dest->vm_pgoff = vma_start_pgoff(src);
54 __vma_set_anon_pgoff(dest, vma_start_anon_pgoff(src));
55 dest->vm_file = src->vm_file;
56 dest->vm_private_data = src->vm_private_data;
57 vm_flags_init(dest, src->vm_flags);
58 memcpy(&dest->vm_page_prot, &src->vm_page_prot,
59 sizeof(dest->vm_page_prot));
60 /*
61 * src->shared.rb may be modified concurrently when called from
62 * dup_mmap(), but the clone will reinitialize it.
63 */
64 data_race(memcpy(&dest->shared, &src->shared, sizeof(dest->shared)));
65 memcpy(&dest->vm_userfaultfd_ctx, &src->vm_userfaultfd_ctx,
66 sizeof(dest->vm_userfaultfd_ctx));
67#ifdef CONFIG_ANON_VMA_NAME
68 dest->anon_name = src->anon_name;
69#endif
70#ifdef CONFIG_SWAP
71 memcpy(&dest->swap_readahead_info, &src->swap_readahead_info,
72 sizeof(dest->swap_readahead_info));
73#endif
74#ifndef CONFIG_MMU
75 dest->vm_region = src->vm_region;
76#endif
77#ifdef CONFIG_NUMA
78 dest->vm_policy = src->vm_policy;
79#endif
80#ifdef __HAVE_PFNMAP_TRACKING
81 dest->pfnmap_track_ctx = NULL;
82#endif
83}
84
85#ifdef __HAVE_PFNMAP_TRACKING
86static inline int vma_pfnmap_track_ctx_dup(struct vm_area_struct *orig,
87 struct vm_area_struct *new)
88{
89 struct pfnmap_track_ctx *ctx = orig->pfnmap_track_ctx;
90
91 if (likely(!ctx))
92 return 0;
93
94 /*
95 * We don't expect to ever hit this. If ever required, we would have
96 * to duplicate the tracking.
97 */
98 if (unlikely(kref_read(&ctx->kref) >= REFCOUNT_MAX))
99 return -ENOMEM;
100 kref_get(&ctx->kref);
101 new->pfnmap_track_ctx = ctx;
102 return 0;
103}
104
105static inline void vma_pfnmap_track_ctx_release(struct vm_area_struct *vma)
106{
107 struct pfnmap_track_ctx *ctx = vma->pfnmap_track_ctx;
108
109 if (likely(!ctx))
110 return;
111
112 kref_put(&ctx->kref, pfnmap_track_ctx_release);
113 vma->pfnmap_track_ctx = NULL;
114}
115#else
116static inline int vma_pfnmap_track_ctx_dup(struct vm_area_struct *orig,
117 struct vm_area_struct *new)
118{
119 return 0;
120}
121static inline void vma_pfnmap_track_ctx_release(struct vm_area_struct *vma)
122{
123}
124#endif
125
126struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig)
127{
128 struct vm_area_struct *new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
129
130 if (!new)
131 return NULL;
132
133 ASSERT_EXCLUSIVE_WRITER(orig->vm_flags);
134 ASSERT_EXCLUSIVE_WRITER(orig->vm_file);
135 vm_area_init_from(orig, new);
136
137 if (vma_pfnmap_track_ctx_dup(orig, new)) {
138 kmem_cache_free(vm_area_cachep, new);
139 return NULL;
140 }
141 vma_lock_init(new, true);
142 INIT_LIST_HEAD(&new->anon_vma_chain);
143 vma_numab_state_init(new);
144 dup_anon_vma_name(orig, new);
145
146 return new;
147}
148
149void vm_area_free(struct vm_area_struct *vma)
150{
151 /* The vma should be detached while being destroyed. */
152 vma_assert_detached(vma);
153 vma_numab_state_free(vma);
154 free_anon_vma_name(vma);
155 vma_pfnmap_track_ctx_release(vma);
156 kmem_cache_free(vm_area_cachep, vma);
157}