projects/linux·blob 2c455ddc · refs

854 lines · 27.2 KiB

1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * vma.h
4 *
5 * Core VMA manipulation API implemented in vma.c, vma_init.c and vma_exec.c.
6 *
7 * Note that, in order for VMA logic to be userland testable, this header
8 * intentionally includes no dependencies.
9 *
10 * This is specifically scoped to mm-only. Users of this functionality (other
11 * than the core VMA implementation itself) should not include this header
12 * directly, but rather include internal.h.
13 */
14#ifndef __MM_VMA_H
15#define __MM_VMA_H
16
17/*
18 * VMA lock generalization
19 */
20struct vma_prepare {
21 struct vm_area_struct *vma;
22 struct vm_area_struct *adj_next;
23 struct file *file;
24 struct address_space *mapping;
25 struct anon_vma *anon_vma;
26 struct vm_area_struct *insert;
27 struct vm_area_struct *remove;
28 struct vm_area_struct *remove2;
29
30 bool skip_vma_uprobe :1;
31};
32
33struct unlink_vma_file_batch {
34 int count;
35 struct vm_area_struct *vmas[8];
36};
37
38/*
39 * vma munmap operation
40 */
41struct vma_munmap_struct {
42 struct vma_iterator *vmi;
43 struct vm_area_struct *vma; /* The first vma to munmap */
44 struct vm_area_struct *prev; /* vma before the munmap area */
45 struct vm_area_struct *next; /* vma after the munmap area */
46 struct list_head *uf; /* Userfaultfd list_head */
47 unsigned long start; /* Aligned start addr (inclusive) */
48 unsigned long end; /* Aligned end addr (exclusive) */
49 unsigned long unmap_start; /* Unmap PTE start */
50 unsigned long unmap_end; /* Unmap PTE end */
51 int vma_count; /* Number of vmas that will be removed */
52 bool unlock; /* Unlock after the munmap */
53 bool clear_ptes; /* If there are outstanding PTE to be cleared */
54 /* 2 byte hole */
55 unsigned long nr_pages; /* Number of pages being removed */
56 unsigned long locked_vm; /* Number of locked pages */
57 unsigned long nr_accounted; /* Number of VM_ACCOUNT pages */
58 unsigned long exec_vm;
59 unsigned long stack_vm;
60 unsigned long data_vm;
61};
62
63enum vma_merge_state {
64 VMA_MERGE_START,
65 VMA_MERGE_ERROR_NOMEM,
66 VMA_MERGE_NOMERGE,
67 VMA_MERGE_SUCCESS,
68};
69
70/*
71 * Describes a VMA merge operation and is threaded throughout it.
72 *
73 * Any of the fields may be mutated by the merge operation, so no guarantees are
74 * made to the contents of this structure after a merge operation has completed.
75 */
76struct vma_merge_struct {
77 struct mm_struct *mm;
78 struct vma_iterator *vmi;
79 /*
80 * Adjacent VMAs, any of which may be NULL if not present:
81 *
82 * |------|--------|------|
83 * | prev | middle | next |
84 * |------|--------|------|
85 *
86 * middle may not yet exist in the case of a proposed new VMA being
87 * merged, or it may be an existing VMA.
88 *
89 * next may be assigned by the caller.
90 */
91 struct vm_area_struct *prev;
92 struct vm_area_struct *middle;
93 struct vm_area_struct *next;
94 /* This is the VMA we ultimately target to become the merged VMA. */
95 struct vm_area_struct *target;
96 /*
97 * Initially, the start, end, pgoff fields are provided by the caller
98 * and describe the proposed new VMA range, whether modifying an
99 * existing VMA (which will be 'middle'), or adding a new one.
100 *
101 * During the merge process these fields are updated to describe the new
102 * range _including those VMAs which will be merged_.
103 */
104 unsigned long start;
105 unsigned long end;
106 pgoff_t pgoff;
107 pgoff_t anon_pgoff;
108
109 union {
110 /* Temporary while VMA flags are being converted. */
111 vm_flags_t vm_flags;
112 vma_flags_t vma_flags;
113 };
114 struct file *file;
115 struct anon_vma *anon_vma;
116 struct mempolicy *policy;
117 struct vm_userfaultfd_ctx uffd_ctx;
118 struct anon_vma_name *anon_name;
119 enum vma_merge_state state;
120
121 /* If copied from (i.e. mremap()'d) the VMA from which we are copying. */
122 struct vm_area_struct *copied_from;
123
124 /* Flags which callers can use to modify merge behaviour: */
125
126 /*
127 * If we can expand, simply do so. We know there is nothing to merge to
128 * the right. Does not reset state upon failure to merge. The VMA
129 * iterator is assumed to be positioned at the previous VMA, rather than
130 * at the gap.
131 */
132 bool just_expand :1;
133
134 /*
135 * If a merge is possible, but an OOM error occurs, give up and don't
136 * execute the merge, returning NULL.
137 */
138 bool give_up_on_oom :1;
139
140 /*
141 * If set, skip uprobe_mmap upon merged vma.
142 */
143 bool skip_vma_uprobe :1;
144
145 /* Internal flags set during merge process: */
146
147 /*
148 * Internal flag indicating the merge increases vmg->middle->vm_start
149 * (and thereby, vmg->prev->vm_end).
150 */
151 bool __adjust_middle_start :1;
152 /*
153 * Internal flag indicating the merge decreases vmg->next->vm_start
154 * (and thereby, vmg->middle->vm_end).
155 */
156 bool __adjust_next_start :1;
157 /*
158 * Internal flag used during the merge operation to indicate we will
159 * remove vmg->middle.
160 */
161 bool __remove_middle :1;
162 /*
163 * Internal flag used during the merge operation to indicate we will
164 * remove vmg->next.
165 */
166 bool __remove_next :1;
167
168};
169
170struct unmap_desc {
171 struct ma_state *mas; /* the maple state point to the first vma */
172 struct vm_area_struct *first; /* The first vma */
173 unsigned long pg_start; /* The first pagetable address to free (floor) */
174 unsigned long pg_end; /* The last pagetable address to free (ceiling) */
175 unsigned long vma_start; /* The min vma address */
176 unsigned long vma_end; /* The max vma address */
177 unsigned long tree_end; /* Maximum for the vma tree search */
178 unsigned long tree_reset; /* Where to reset the vma tree walk */
179 bool mm_wr_locked; /* If the mmap write lock is held */
180};
181
182/*
183 * unmap_all_init() - Initialize unmap_desc to remove all vmas, point the
184 * pg_start and pg_end to a safe location.
185 */
186static inline void unmap_all_init(struct unmap_desc *unmap,
187 struct vma_iterator *vmi, struct vm_area_struct *vma)
188{
189 unmap->mas = &vmi->mas;
190 unmap->first = vma;
191 unmap->pg_start = FIRST_USER_ADDRESS;
192 unmap->pg_end = USER_PGTABLES_CEILING;
193 unmap->vma_start = 0;
194 unmap->vma_end = ULONG_MAX;
195 unmap->tree_end = ULONG_MAX;
196 unmap->tree_reset = vma->vm_end;
197 unmap->mm_wr_locked = false;
198}
199
200/*
201 * unmap_pgtable_init() - Initialize unmap_desc to remove all page tables within
202 * the user range.
203 *
204 * ARM can have mappings outside of vmas.
205 * See: e2cdef8c847b4 ("[PATCH] freepgt: free_pgtables from FIRST_USER_ADDRESS")
206 *
207 * ARM LPAE uses page table mappings beyond the USER_PGTABLES_CEILING
208 * See: CONFIG_ARM_LPAE in arch/arm/include/asm/pgtable.h
209 */
210static inline void unmap_pgtable_init(struct unmap_desc *unmap,
211 struct vma_iterator *vmi)
212{
213 vma_iter_set(vmi, unmap->tree_reset);
214 unmap->vma_start = FIRST_USER_ADDRESS;
215 unmap->vma_end = USER_PGTABLES_CEILING;
216 unmap->tree_end = USER_PGTABLES_CEILING;
217}
218
219#define UNMAP_STATE(name, _vmi, _vma, _vma_start, _vma_end, _prev, _next) \
220 struct unmap_desc name = { \
221 .mas = &(_vmi)->mas, \
222 .first = _vma, \
223 .pg_start = _prev ? ((struct vm_area_struct *)_prev)->vm_end : \
224 FIRST_USER_ADDRESS, \
225 .pg_end = _next ? ((struct vm_area_struct *)_next)->vm_start : \
226 USER_PGTABLES_CEILING, \
227 .vma_start = _vma_start, \
228 .vma_end = _vma_end, \
229 .tree_end = _next ? \
230 ((struct vm_area_struct *)_next)->vm_start : \
231 USER_PGTABLES_CEILING, \
232 .tree_reset = _vma->vm_end, \
233 .mm_wr_locked = true, \
234 }
235
236static inline bool vmg_nomem(struct vma_merge_struct *vmg)
237{
238 return vmg->state == VMA_MERGE_ERROR_NOMEM;
239}
240
241static inline pgoff_t vmg_pages(const struct vma_merge_struct *vmg)
242{
243 const unsigned long size = vmg->end - vmg->start;
244
245 return size >> PAGE_SHIFT;
246}
247
248static inline pgoff_t vmg_start_pgoff(const struct vma_merge_struct *vmg)
249{
250 return vmg->pgoff;
251}
252
253static inline pgoff_t vmg_end_pgoff(const struct vma_merge_struct *vmg)
254{
255 return vmg_start_pgoff(vmg) + vmg_pages(vmg);
256}
257
258static inline void assert_sane_pgoff(struct vm_area_struct *vma, pgoff_t pgoff)
259{
260 /* nommu doesn't set a virtual pgoff for anon VMAs. */
261 if (!IS_ENABLED(CONFIG_MMU))
262 return;
263 /*
264 * File-backed VMAs have arbitrary page offset (either page offset into
265 * file or for pfnmap the PFN of the start of the range or drivers may
266 * set arbitrary page offset).
267 */
268 if (!vma_is_anonymous(vma))
269 return;
270 /* MAP_PRIVATE-/dev/zero is anon, non-NULL vm_file, but has file pgoff. */
271 if (vma->vm_file)
272 return;
273 /* If faulted in, could have been remapped. */
274 if (vma->anon_vma)
275 return;
276 /* OK this is really an anon VMA - expect virtual page offset. */
277 VM_WARN_ON_ONCE(pgoff != vma->vm_start >> PAGE_SHIFT);
278}
279
280static inline void vma_set_pgoff(struct vm_area_struct *vma, pgoff_t pgoff)
281{
282 vma_assert_can_modify(vma);
283 assert_sane_pgoff(vma, pgoff);
284 vma->vm_pgoff = pgoff;
285}
286
287static inline pgoff_t vmg_start_anon_pgoff(const struct vma_merge_struct *vmg)
288{
289 return vmg->anon_pgoff;
290}
291
292static inline pgoff_t vmg_end_anon_pgoff(const struct vma_merge_struct *vmg)
293{
294 return vmg_start_anon_pgoff(vmg) + vmg_pages(vmg);
295}
296
297static inline void __vma_set_anon_pgoff(struct vm_area_struct *vma, pgoff_t pgoff)
298{
299#ifdef CONFIG_64BIT
300 vma->__vm_anon_pgoff_hi = pgoff >> 32;
301#endif
302 vma->__vm_anon_pgoff_lo = pgoff & GENMASK(31, 0);
303}
304
305static inline void vma_set_anon_pgoff(struct vm_area_struct *vma, pgoff_t pgoff)
306{
307 vma_assert_can_modify(vma);
308 __vma_set_anon_pgoff(vma, pgoff);
309}
310
311static inline void vma_add_pgoff(struct vm_area_struct *vma, pgoff_t delta)
312{
313 vma_assert_can_modify(vma);
314 vma_set_pgoff(vma, vma_start_pgoff(vma) + delta);
315 vma_set_anon_pgoff(vma, vma_start_anon_pgoff(vma) + delta);
316}
317
318static inline void vma_sub_pgoff(struct vm_area_struct *vma, pgoff_t delta)
319{
320 vma_assert_can_modify(vma);
321 vma_set_pgoff(vma, vma_start_pgoff(vma) - delta);
322 vma_set_anon_pgoff(vma, vma_start_anon_pgoff(vma) - delta);
323}
324
325#define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_, anon_pgoff_) \
326 struct vma_merge_struct name = { \
327 .mm = mm_, \
328 .vmi = vmi_, \
329 .start = start_, \
330 .end = end_, \
331 .vma_flags = vma_flags_, \
332 .pgoff = pgoff_, \
333 .anon_pgoff = anon_pgoff_, \
334 .state = VMA_MERGE_START, \
335 }
336
337#define VMG_VMA_STATE(name, vmi_, prev_, vma_, start_, end_) \
338 struct vma_merge_struct name = { \
339 .mm = vma_->vm_mm, \
340 .vmi = vmi_, \
341 .prev = prev_, \
342 .middle = vma_, \
343 .next = NULL, \
344 .start = start_, \
345 .end = end_, \
346 .vm_flags = vma_->vm_flags, \
347 .pgoff = linear_page_index(vma_, start_), \
348 .anon_pgoff = __linear_anon_page_index(vma_, start_), \
349 .file = vma_->vm_file, \
350 .anon_vma = vma_->anon_vma, \
351 .policy = vma_policy(vma_), \
352 .uffd_ctx = vma_->vm_userfaultfd_ctx, \
353 .anon_name = anon_vma_name(vma_), \
354 .state = VMA_MERGE_START, \
355 }
356
357#ifdef CONFIG_DEBUG_VM_MAPLE_TREE
358void validate_mm(struct mm_struct *mm);
359#else
360#define validate_mm(mm) do { } while (0)
361#endif
362
363__must_check int vma_expand(struct vma_merge_struct *vmg);
364__must_check int vma_shrink(struct vma_iterator *vmi,
365 struct vm_area_struct *vma, unsigned long end);
366
367static inline int vma_iter_store_gfp(struct vma_iterator *vmi,
368 struct vm_area_struct *vma, gfp_t gfp)
369
370{
371 if (vmi->mas.status != ma_start &&
372 ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
373 vma_iter_invalidate(vmi);
374
375 __mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
376 mas_store_gfp(&vmi->mas, vma, gfp);
377 if (unlikely(mas_is_err(&vmi->mas)))
378 return -ENOMEM;
379
380 vma_mark_attached(vma);
381 return 0;
382}
383
384/*
385 * Temporary helper function for stacked mmap handlers which specify
386 * f_op->mmap() but which might have an underlying file system which implements
387 * f_op->mmap_prepare().
388 */
389static inline void compat_set_vma_from_desc(struct vm_area_struct *vma,
390 struct vm_area_desc *desc)
391{
392 /*
393 * Since we're invoking .mmap_prepare() despite having a partially
394 * established VMA, we must take care to handle setting fields
395 * correctly.
396 */
397
398 /* Mutable fields. Populated with initial state. */
399 vma_set_pgoff(vma, desc->pgoff);
400 if (desc->vm_file != vma->vm_file)
401 vma_set_file(vma, desc->vm_file);
402 vma->flags = desc->vma_flags;
403 vma->vm_page_prot = desc->page_prot;
404
405 /* User-defined fields. */
406 vma->vm_ops = desc->vm_ops;
407 vma->vm_private_data = desc->private_data;
408}
409
410int
411do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
412 struct mm_struct *mm, unsigned long start,
413 unsigned long end, struct list_head *uf, bool unlock);
414
415int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,
416 unsigned long start, size_t len, struct list_head *uf,
417 bool unlock);
418
419void remove_vma(struct vm_area_struct *vma);
420void unmap_region(struct unmap_desc *unmap);
421
422/**
423 * vma_modify_flags() - Perform any necessary split/merge in preparation for
424 * setting VMA flags to *@vm_flags in the range @start to @end contained within
425 * @vma.
426 * @vmi: Valid VMA iterator positioned at @vma.
427 * @prev: The VMA immediately prior to @vma or NULL if @vma is the first.
428 * @vma: The VMA containing the range @start to @end to be updated.
429 * @start: The start of the range to update. May be offset within @vma.
430 * @end: The exclusive end of the range to update, may be offset within @vma.
431 * @vma_flags_ptr: A pointer to the VMA flags that the @start to @end range is
432 * about to be set to. On merge, this will be updated to include sticky flags.
433 *
434 * IMPORTANT: The actual modification being requested here is NOT applied,
435 * rather the VMA is perhaps split, perhaps merged to accommodate the change,
436 * and the caller is expected to perform the actual modification.
437 *
438 * In order to account for sticky VMA flags, the @vma_flags_ptr parameter points
439 * to the requested flags which are then updated so the caller, should they
440 * overwrite any existing flags, correctly retains these.
441 *
442 * Returns: A VMA which contains the range @start to @end ready to have its
443 * flags altered to *@vma_flags.
444 */
445__must_check struct vm_area_struct *vma_modify_flags(struct vma_iterator *vmi,
446 struct vm_area_struct *prev, struct vm_area_struct *vma,
447 unsigned long start, unsigned long end, vma_flags_t *vma_flags_ptr);
448
449/**
450 * vma_modify_name() - Perform any necessary split/merge in preparation for
451 * setting anonymous VMA name to @new_name in the range @start to @end contained
452 * within @vma.
453 * @vmi: Valid VMA iterator positioned at @vma.
454 * @prev: The VMA immediately prior to @vma or NULL if @vma is the first.
455 * @vma: The VMA containing the range @start to @end to be updated.
456 * @start: The start of the range to update. May be offset within @vma.
457 * @end: The exclusive end of the range to update, may be offset within @vma.
458 * @new_name: The anonymous VMA name that the @start to @end range is about to
459 * be set to.
460 *
461 * IMPORTANT: The actual modification being requested here is NOT applied,
462 * rather the VMA is perhaps split, perhaps merged to accommodate the change,
463 * and the caller is expected to perform the actual modification.
464 *
465 * Returns: A VMA which contains the range @start to @end ready to have its
466 * anonymous VMA name changed to @new_name.
467 */
468__must_check struct vm_area_struct *vma_modify_name(struct vma_iterator *vmi,
469 struct vm_area_struct *prev, struct vm_area_struct *vma,
470 unsigned long start, unsigned long end,
471 struct anon_vma_name *new_name);
472
473/**
474 * vma_modify_policy() - Perform any necessary split/merge in preparation for
475 * setting NUMA policy to @new_pol in the range @start to @end contained
476 * within @vma.
477 * @vmi: Valid VMA iterator positioned at @vma.
478 * @prev: The VMA immediately prior to @vma or NULL if @vma is the first.
479 * @vma: The VMA containing the range @start to @end to be updated.
480 * @start: The start of the range to update. May be offset within @vma.
481 * @end: The exclusive end of the range to update, may be offset within @vma.
482 * @new_pol: The NUMA policy that the @start to @end range is about to be set
483 * to.
484 *
485 * IMPORTANT: The actual modification being requested here is NOT applied,
486 * rather the VMA is perhaps split, perhaps merged to accommodate the change,
487 * and the caller is expected to perform the actual modification.
488 *
489 * Returns: A VMA which contains the range @start to @end ready to have its
490 * NUMA policy changed to @new_pol.
491 */
492__must_check struct vm_area_struct *vma_modify_policy(struct vma_iterator *vmi,
493 struct vm_area_struct *prev, struct vm_area_struct *vma,
494 unsigned long start, unsigned long end,
495 struct mempolicy *new_pol);
496
497/**
498 * vma_modify_flags_uffd() - Perform any necessary split/merge in preparation for
499 * setting VMA flags to @vm_flags and UFFD context to @new_ctx in the range
500 * @start to @end contained within @vma.
501 * @vmi: Valid VMA iterator positioned at @vma.
502 * @prev: The VMA immediately prior to @vma or NULL if @vma is the first.
503 * @vma: The VMA containing the range @start to @end to be updated.
504 * @start: The start of the range to update. May be offset within @vma.
505 * @end: The exclusive end of the range to update, may be offset within @vma.
506 * @vma_flags: The VMA flags that the @start to @end range is about to be set to.
507 * @new_ctx: The userfaultfd context that the @start to @end range is about to
508 * be set to.
509 * @give_up_on_oom: If an out of memory condition occurs on merge, simply give
510 * up on it and treat the merge as best-effort.
511 *
512 * IMPORTANT: The actual modification being requested here is NOT applied,
513 * rather the VMA is perhaps split, perhaps merged to accommodate the change,
514 * and the caller is expected to perform the actual modification.
515 *
516 * Returns: A VMA which contains the range @start to @end ready to have its VMA
517 * flags changed to @vma_flags and its userfaultfd context changed to @new_ctx.
518 */
519__must_check struct vm_area_struct *vma_modify_flags_uffd(struct vma_iterator *vmi,
520 struct vm_area_struct *prev, struct vm_area_struct *vma,
521 unsigned long start, unsigned long end, const vma_flags_t *vma_flags,
522 struct vm_userfaultfd_ctx new_ctx, bool give_up_on_oom);
523
524__must_check struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg);
525
526__must_check struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi,
527 struct vm_area_struct *vma, unsigned long delta);
528
529void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb);
530
531void unlink_file_vma_batch_final(struct unlink_vma_file_batch *vb);
532
533void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb,
534 struct vm_area_struct *vma);
535
536struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
537 unsigned long addr, unsigned long len, pgoff_t pgoff,
538 pgoff_t anon_pgoff, bool *need_rmap_locks, bool keep_source);
539
540struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma);
541
542bool vma_needs_dirty_tracking(struct vm_area_struct *vma);
543bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot);
544
545int mm_take_all_locks(struct mm_struct *mm);
546void mm_drop_all_locks(struct mm_struct *mm);
547
548unsigned long mmap_region(struct file *file, unsigned long addr,
549 unsigned long len, vma_flags_t vma_flags, unsigned long pgoff,
550 struct list_head *uf);
551
552int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma,
553 unsigned long addr, unsigned long request,
554 vma_flags_t vma_flags);
555
556unsigned long unmapped_area(struct vm_unmapped_area_info *info);
557unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info);
558
559#ifdef CONFIG_MMU
560int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
561 unsigned long addr, int new_below);
562#endif
563
564static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma)
565{
566 /*
567 * We want to check manually if we can change individual PTEs writable
568 * if we can't do that automatically for all PTEs in a mapping. For
569 * private mappings, that's always the case when we have write
570 * permissions as we properly have to handle COW.
571 */
572 if (vma->vm_flags & VM_SHARED)
573 return vma_wants_writenotify(vma, vma->vm_page_prot);
574 return !!(vma->vm_flags & VM_WRITE);
575}
576
577#ifdef CONFIG_MMU
578static inline pgprot_t vma_pgprot_modify(pgprot_t oldprot, vma_flags_t vma_flags)
579{
580 const pgprot_t prot = vma_flags_to_page_prot(vma_flags);
581
582 return pgprot_modify(oldprot, prot);
583}
584#endif
585
586static inline struct vm_area_struct *vma_prev_limit(struct vma_iterator *vmi,
587 unsigned long min)
588{
589 return mas_prev(&vmi->mas, min);
590}
591
592/*
593 * These three helpers classifies VMAs for virtual memory accounting.
594 */
595
596/*
597 * Executable code area - executable, not writable, not stack
598 */
599static inline bool is_exec_mapping(vm_flags_t flags)
600{
601 return (flags & (VM_EXEC | VM_WRITE | VM_STACK)) == VM_EXEC;
602}
603
604/*
605 * Stack area (including shadow stacks)
606 *
607 * VM_GROWSUP / VM_GROWSDOWN VMAs are always private anonymous:
608 * do_mmap() forbids all other combinations.
609 */
610static inline bool is_stack_mapping(vm_flags_t flags)
611{
612 return ((flags & VM_STACK) == VM_STACK) || (flags & VM_SHADOW_STACK);
613}
614
615/*
616 * Data area - private, writable, not stack
617 */
618static inline bool is_data_mapping(vm_flags_t flags)
619{
620 return (flags & (VM_WRITE | VM_SHARED | VM_STACK)) == VM_WRITE;
621}
622
623static inline bool is_data_mapping_vma_flags(const vma_flags_t *vma_flags)
624{
625 return vma_flags_test(vma_flags, VMA_WRITE_BIT) &&
626 !vma_flags_test_any(vma_flags, VMA_SHARED_BIT, VMA_STACK_BIT);
627}
628
629static inline void vma_iter_config(struct vma_iterator *vmi,
630 unsigned long index, unsigned long last)
631{
632 __mas_set_range(&vmi->mas, index, last - 1);
633}
634
635static inline void vma_iter_reset(struct vma_iterator *vmi)
636{
637 mas_reset(&vmi->mas);
638}
639
640static inline
641struct vm_area_struct *vma_iter_prev_range_limit(struct vma_iterator *vmi, unsigned long min)
642{
643 return mas_prev_range(&vmi->mas, min);
644}
645
646static inline
647struct vm_area_struct *vma_iter_next_range_limit(struct vma_iterator *vmi, unsigned long max)
648{
649 return mas_next_range(&vmi->mas, max);
650}
651
652static inline int vma_iter_area_lowest(struct vma_iterator *vmi, unsigned long min,
653 unsigned long max, unsigned long size)
654{
655 return mas_empty_area(&vmi->mas, min, max - 1, size);
656}
657
658static inline int vma_iter_area_highest(struct vma_iterator *vmi, unsigned long min,
659 unsigned long max, unsigned long size)
660{
661 return mas_empty_area_rev(&vmi->mas, min, max - 1, size);
662}
663
664/*
665 * VMA Iterator functions shared between nommu and mmap
666 */
667static inline int vma_iter_prealloc(struct vma_iterator *vmi,
668 struct vm_area_struct *vma)
669{
670 return mas_preallocate(&vmi->mas, vma, GFP_KERNEL);
671}
672
673static inline void vma_iter_clear(struct vma_iterator *vmi)
674{
675 mas_store_prealloc(&vmi->mas, NULL);
676}
677
678static inline struct vm_area_struct *vma_iter_load(struct vma_iterator *vmi)
679{
680 return mas_walk(&vmi->mas);
681}
682
683/* Store a VMA with preallocated memory */
684static inline void vma_iter_store_overwrite(struct vma_iterator *vmi,
685 struct vm_area_struct *vma)
686{
687 vma_assert_attached(vma);
688
689#if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
690 if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
691 vmi->mas.index > vma->vm_start)) {
692 pr_warn("%lx > %lx\n store vma %lx-%lx\n into slot %lx-%lx\n",
693 vmi->mas.index, vma->vm_start, vma->vm_start,
694 vma->vm_end, vmi->mas.index, vmi->mas.last);
695 }
696 if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
697 vmi->mas.last < vma->vm_start)) {
698 pr_warn("%lx < %lx\nstore vma %lx-%lx\ninto slot %lx-%lx\n",
699 vmi->mas.last, vma->vm_start, vma->vm_start, vma->vm_end,
700 vmi->mas.index, vmi->mas.last);
701 }
702#endif
703
704 if (vmi->mas.status != ma_start &&
705 ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
706 vma_iter_invalidate(vmi);
707
708 __mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
709 mas_store_prealloc(&vmi->mas, vma);
710}
711
712static inline void vma_iter_store_new(struct vma_iterator *vmi,
713 struct vm_area_struct *vma)
714{
715 vma_mark_attached(vma);
716 vma_iter_store_overwrite(vmi, vma);
717}
718
719static inline unsigned long vma_iter_addr(struct vma_iterator *vmi)
720{
721 return vmi->mas.index;
722}
723
724static inline unsigned long vma_iter_end(struct vma_iterator *vmi)
725{
726 return vmi->mas.last + 1;
727}
728
729static inline
730struct vm_area_struct *vma_iter_prev_range(struct vma_iterator *vmi)
731{
732 return mas_prev_range(&vmi->mas, 0);
733}
734
735/*
736 * Retrieve the next VMA and rewind the iterator to end of the previous VMA, or
737 * if no previous VMA, to index 0.
738 */
739static inline
740struct vm_area_struct *vma_iter_next_rewind(struct vma_iterator *vmi,
741 struct vm_area_struct **pprev)
742{
743 struct vm_area_struct *next = vma_next(vmi);
744 struct vm_area_struct *prev = vma_prev(vmi);
745
746 /*
747 * Consider the case where no previous VMA exists. We advance to the
748 * next VMA, skipping any gap, then rewind to the start of the range.
749 *
750 * If we were to unconditionally advance to the next range we'd wind up
751 * at the next VMA again, so we check to ensure there is a previous VMA
752 * to skip over.
753 */
754 if (prev)
755 vma_iter_next_range(vmi);
756
757 if (pprev)
758 *pprev = prev;
759
760 return next;
761}
762
763#ifdef CONFIG_64BIT
764static inline bool vma_is_sealed(struct vm_area_struct *vma)
765{
766 return (vma->vm_flags & VM_SEALED);
767}
768#else
769static inline bool vma_is_sealed(struct vm_area_struct *vma)
770{
771 return false;
772}
773#endif
774
775#if defined(CONFIG_STACK_GROWSUP)
776int expand_upwards(struct vm_area_struct *vma, unsigned long address);
777#endif
778
779int expand_downwards(struct vm_area_struct *vma, unsigned long address);
780
781int __vm_munmap(unsigned long start, size_t len, bool unlock);
782
783int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma);
784
785/* vma_init.h, shared between CONFIG_MMU and nommu. */
786void __init vma_state_init(void);
787struct vm_area_struct *vm_area_alloc(struct mm_struct *mm);
788struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig);
789void vm_area_free(struct vm_area_struct *vma);
790
791/* vma_exec.c */
792#ifdef CONFIG_MMU
793int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap,
794 unsigned long *top_mem_p);
795int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift);
796#endif
797
798#ifdef CONFIG_MMU
799/*
800 * Denies creating a writable executable mapping or gaining executable permissions.
801 *
802 * This denies the following:
803 *
804 * a) mmap(PROT_WRITE | PROT_EXEC)
805 *
806 * b) mmap(PROT_WRITE)
807 * mprotect(PROT_EXEC)
808 *
809 * c) mmap(PROT_WRITE)
810 * mprotect(PROT_READ)
811 * mprotect(PROT_EXEC)
812 *
813 * But allows the following:
814 *
815 * d) mmap(PROT_READ | PROT_EXEC)
816 * mmap(PROT_READ | PROT_EXEC | PROT_BTI)
817 *
818 * This is only applicable if the user has set the Memory-Deny-Write-Execute
819 * (MDWE) protection mask for the current process.
820 *
821 * @old specifies the VMA flags the VMA originally possessed, and @new the ones
822 * we propose to set.
823 *
824 * Return: false if proposed change is OK, true if not ok and should be denied.
825 */
826static inline bool map_deny_write_exec(const vma_flags_t *old,
827 const vma_flags_t *new)
828{
829 /* If MDWE is disabled, we have nothing to deny. */
830 if (!mm_flags_test(MMF_HAS_MDWE, current->mm))
831 return false;
832
833 /* If the new VMA is not executable, we have nothing to deny. */
834 if (!vma_flags_test(new, VMA_EXEC_BIT))
835 return false;
836
837 /* Under MDWE we do not accept newly writably executable VMAs... */
838 if (vma_flags_test(new, VMA_WRITE_BIT))
839 return true;
840
841 /* ...nor previously non-executable VMAs becoming executable. */
842 if (!vma_flags_test(old, VMA_EXEC_BIT))
843 return true;
844
845 return false;
846}
847#endif
848
849struct vm_area_struct *__install_special_mapping(struct mm_struct *mm,
850 unsigned long addr, unsigned long len,
851 vm_flags_t vm_flags, void *priv,
852 const struct vm_operations_struct *ops);
853
854#endif /* __MM_VMA_H */