projects/linux·blob master · refs

1900 lines · 50.1 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * mm/mmap.c
4 *
5 * Written by obz.
6 *
7 * Address space accounting code <[email protected]>
8 */
9
10#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11
12#include <linux/kernel.h>
13#include <linux/slab.h>
14#include <linux/backing-dev.h>
15#include <linux/mm.h>
16#include <linux/mm_inline.h>
17#include <linux/shm.h>
18#include <linux/mman.h>
19#include <linux/pagemap.h>
20#include <linux/swap.h>
21#include <linux/syscalls.h>
22#include <linux/capability.h>
23#include <linux/init.h>
24#include <linux/file.h>
25#include <linux/fs.h>
26#include <linux/personality.h>
27#include <linux/security.h>
28#include <linux/hugetlb.h>
29#include <linux/shmem_fs.h>
30#include <linux/profile.h>
31#include <linux/export.h>
32#include <linux/mount.h>
33#include <linux/mempolicy.h>
34#include <linux/rmap.h>
35#include <linux/mmu_notifier.h>
36#include <linux/mmdebug.h>
37#include <linux/perf_event.h>
38#include <linux/audit.h>
39#include <linux/khugepaged.h>
40#include <linux/uprobes.h>
41#include <linux/notifier.h>
42#include <linux/memory.h>
43#include <linux/printk.h>
44#include <linux/userfaultfd_k.h>
45#include <linux/moduleparam.h>
46#include <linux/pkeys.h>
47#include <linux/oom.h>
48#include <linux/sched/mm.h>
49#include <linux/ksm.h>
50#include <linux/memfd.h>
51
52#include <linux/uaccess.h>
53#include <asm/cacheflush.h>
54#include <asm/tlb.h>
55#include <asm/mmu_context.h>
56
57#define CREATE_TRACE_POINTS
58#include <trace/events/mmap.h>
59
60#include "internal.h"
61
62#ifndef arch_mmap_check
63#define arch_mmap_check(addr, len, flags) (0)
64#endif
65
66#ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS
67const int mmap_rnd_bits_min = CONFIG_ARCH_MMAP_RND_BITS_MIN;
68int mmap_rnd_bits_max __ro_after_init = CONFIG_ARCH_MMAP_RND_BITS_MAX;
69int mmap_rnd_bits __read_mostly = CONFIG_ARCH_MMAP_RND_BITS;
70#endif
71#ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
72const int mmap_rnd_compat_bits_min = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MIN;
73const int mmap_rnd_compat_bits_max = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MAX;
74int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS;
75#endif
76
77static bool ignore_rlimit_data;
78core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644);
79
80/* Update vma->vm_page_prot to reflect vma->vm_flags. */
81void vma_set_page_prot(struct vm_area_struct *vma)
82{
83 vma_flags_t vma_flags = vma->flags;
84 pgprot_t vm_page_prot;
85
86 vm_page_prot = vma_pgprot_modify(vma->vm_page_prot, vma_flags);
87 if (vma_wants_writenotify(vma, vm_page_prot)) {
88 vma_flags_clear(&vma_flags, VMA_SHARED_BIT);
89 vm_page_prot = vma_pgprot_modify(vm_page_prot, vma_flags);
90 }
91 /* remove_protection_ptes reads vma->vm_page_prot without mmap_lock */
92 WRITE_ONCE(vma->vm_page_prot, vm_page_prot);
93}
94
95/*
96 * check_brk_limits() - Use platform specific check of range & verify mlock
97 * limits.
98 * @addr: The address to check
99 * @len: The size of increase.
100 *
101 * Return: 0 on success.
102 */
103static int check_brk_limits(unsigned long addr, unsigned long len)
104{
105 const struct mm_struct *mm = current->mm;
106 const bool is_def_locked =
107 vma_flags_test(&mm->def_vma_flags, VMA_LOCKED_BIT);
108 unsigned long mapped_addr;
109
110 mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
111 if (IS_ERR_VALUE(mapped_addr))
112 return mapped_addr;
113
114 return mlock_future_ok(mm, is_def_locked, len) ? 0 : -EAGAIN;
115}
116
117SYSCALL_DEFINE1(brk, unsigned long, brk)
118{
119 unsigned long newbrk, oldbrk, origbrk;
120 struct mm_struct *mm = current->mm;
121 struct vm_area_struct *brkvma, *next = NULL;
122 unsigned long min_brk;
123 bool populate = false;
124 LIST_HEAD(uf);
125 struct vma_iterator vmi;
126
127 if (mmap_write_lock_killable(mm))
128 return -EINTR;
129
130 origbrk = mm->brk;
131
132 min_brk = mm->start_brk;
133#ifdef CONFIG_COMPAT_BRK
134 /*
135 * CONFIG_COMPAT_BRK can still be overridden by setting
136 * randomize_va_space to 2, which will still cause mm->start_brk
137 * to be arbitrarily shifted
138 */
139 if (!current->brk_randomized)
140 min_brk = mm->end_data;
141#endif
142 if (brk < min_brk)
143 goto out;
144
145 /*
146 * Check against rlimit here. If this check is done later after the test
147 * of oldbrk with newbrk then it can escape the test and let the data
148 * segment grow beyond its set limit the in case where the limit is
149 * not page aligned -Ram Gupta
150 */
151 if (check_data_rlimit(rlimit(RLIMIT_DATA), brk, mm->start_brk,
152 mm->end_data, mm->start_data))
153 goto out;
154
155 newbrk = PAGE_ALIGN(brk);
156 oldbrk = PAGE_ALIGN(mm->brk);
157 if (oldbrk == newbrk) {
158 mm->brk = brk;
159 goto success;
160 }
161
162 /* Always allow shrinking brk. */
163 if (brk <= mm->brk) {
164 /* Search one past newbrk */
165 vma_iter_init(&vmi, mm, newbrk);
166 brkvma = vma_find(&vmi, oldbrk);
167 if (!brkvma || brkvma->vm_start >= oldbrk)
168 goto out; /* mapping intersects with an existing non-brk vma. */
169 /*
170 * mm->brk must be protected by write mmap_lock.
171 * do_vmi_align_munmap() will drop the lock on success, so
172 * update it before calling do_vma_munmap().
173 */
174 mm->brk = brk;
175 if (do_vmi_align_munmap(&vmi, brkvma, mm, newbrk, oldbrk, &uf,
176 /* unlock = */ true))
177 goto out;
178
179 goto success_unlocked;
180 }
181
182 if (check_brk_limits(oldbrk, newbrk - oldbrk))
183 goto out;
184
185 /*
186 * Only check if the next VMA is within the stack_guard_gap of the
187 * expansion area
188 */
189 vma_iter_init(&vmi, mm, oldbrk);
190 next = vma_find(&vmi, newbrk + PAGE_SIZE + stack_guard_gap);
191 if (next && newbrk + PAGE_SIZE > vm_start_gap(next))
192 goto out;
193
194 brkvma = vma_prev_limit(&vmi, mm->start_brk);
195 /* Ok, looks good - let it rip. */
196 if (do_brk_flags(&vmi, brkvma, oldbrk, newbrk - oldbrk,
197 EMPTY_VMA_FLAGS) < 0)
198 goto out;
199
200 mm->brk = brk;
201 if (vma_flags_test(&mm->def_vma_flags, VMA_LOCKED_BIT))
202 populate = true;
203
204success:
205 mmap_write_unlock(mm);
206success_unlocked:
207 userfaultfd_unmap_complete(mm, &uf);
208 if (populate)
209 mm_populate(oldbrk, newbrk - oldbrk);
210 return brk;
211
212out:
213 mm->brk = origbrk;
214 mmap_write_unlock(mm);
215 return origbrk;
216}
217
218/*
219 * If a hint addr is less than mmap_min_addr change hint to be as
220 * low as possible but still greater than mmap_min_addr
221 */
222static inline unsigned long round_hint_to_min(unsigned long hint)
223{
224 hint &= PAGE_MASK;
225 if (((void *)hint != NULL) &&
226 (hint < mmap_min_addr))
227 return PAGE_ALIGN(mmap_min_addr);
228 return hint;
229}
230
231bool mlock_future_ok(const struct mm_struct *mm, bool is_vma_locked,
232 unsigned long bytes)
233{
234 unsigned long locked_pages, limit_pages;
235
236 if (!is_vma_locked || capable(CAP_IPC_LOCK))
237 return true;
238
239 locked_pages = bytes >> PAGE_SHIFT;
240 locked_pages += mm->locked_vm;
241
242 limit_pages = rlimit(RLIMIT_MEMLOCK);
243 limit_pages >>= PAGE_SHIFT;
244
245 return locked_pages <= limit_pages;
246}
247
248static inline u64 file_mmap_size_max(struct file *file, struct inode *inode)
249{
250 if (S_ISREG(inode->i_mode))
251 return MAX_LFS_FILESIZE;
252
253 if (S_ISBLK(inode->i_mode))
254 return MAX_LFS_FILESIZE;
255
256 if (S_ISSOCK(inode->i_mode))
257 return MAX_LFS_FILESIZE;
258
259 /* Special "we do even unsigned file positions" case */
260 if (file->f_op->fop_flags & FOP_UNSIGNED_OFFSET)
261 return 0;
262
263 /* Yes, random drivers might want more. But I'm tired of buggy drivers */
264 return ULONG_MAX;
265}
266
267static inline bool file_mmap_ok(struct file *file, struct inode *inode,
268 unsigned long pgoff, unsigned long len)
269{
270 u64 maxsize = file_mmap_size_max(file, inode);
271
272 if (maxsize && len > maxsize)
273 return false;
274 maxsize -= len;
275 if (pgoff > maxsize >> PAGE_SHIFT)
276 return false;
277 return true;
278}
279
280/**
281 * do_mmap() - Perform a userland memory mapping into the current process
282 * address space of length @len with protection bits @prot, mmap flags @flags
283 * (from which VMA flags will be inferred), and any additional VMA flags to
284 * apply @vma_flags. If this is a file-backed mapping then the file is specified
285 * in @file and page offset into the file via @pgoff.
286 *
287 * This function does not perform security checks on the file and assumes, if
288 * @uf is non-NULL, the caller has provided a list head to track unmap events
289 * for userfaultfd @uf.
290 *
291 * It also simply indicates whether memory population is required by setting
292 * @populate, which must be non-NULL, expecting the caller to actually perform
293 * this task itself if appropriate.
294 *
295 * This function will invoke architecture-specific (and if provided and
296 * relevant, file system-specific) logic to determine the most appropriate
297 * unmapped area in which to place the mapping if not MAP_FIXED.
298 *
299 * Callers which require userland mmap() behaviour should invoke vm_mmap(),
300 * which is also exported for module use.
301 *
302 * Those which require this behaviour less security checks, userfaultfd and
303 * populate behaviour, and who handle the mmap write lock themselves, should
304 * call this function.
305 *
306 * Note that the returned address may reside within a merged VMA if an
307 * appropriate merge were to take place, so it doesn't necessarily specify the
308 * start of a VMA, rather only the start of a valid mapped range of length
309 * @len bytes, rounded down to the nearest page size.
310 *
311 * The caller must write-lock current->mm->mmap_lock.
312 *
313 * @file: An optional struct file pointer describing the file which is to be
314 * mapped, if a file-backed mapping.
315 * @addr: If non-zero, hints at (or if @flags has MAP_FIXED set, specifies) the
316 * address at which to perform this mapping. See mmap (2) for details. Must be
317 * page-aligned.
318 * @len: The length of the mapping. Will be page-aligned and must be at least 1
319 * page in size.
320 * @prot: Protection bits describing access required to the mapping. See mmap
321 * (2) for details.
322 * @flags: Flags specifying how the mapping should be performed, see mmap (2)
323 * for details.
324 * @vma_flags: VMA flags which should be set by default, or EMPTY_VMA_FLAGS
325 * otherwise.
326 * @pgoff: Page offset into the @file if file-backed, should be 0 otherwise.
327 * @populate: A pointer to a value which will be set to 0 if no population of
328 * the range is required, or the number of bytes to populate if it is. Must be
329 * non-NULL. See mmap (2) for details as to under what circumstances population
330 * of the range occurs.
331 * @uf: An optional pointer to a list head to track userfaultfd unmap events
332 * should unmapping events arise. If provided, it is up to the caller to manage
333 * this.
334 *
335 * Returns: Either an error, or the address at which the requested mapping has
336 * been performed.
337 */
338unsigned long do_mmap(struct file *file, unsigned long addr,
339 unsigned long len, unsigned long prot,
340 unsigned long flags, vma_flags_t vma_flags,
341 unsigned long pgoff, unsigned long *populate,
342 struct list_head *uf)
343{
344 struct mm_struct *mm = current->mm;
345 int pkey = 0;
346
347 *populate = 0;
348
349 mmap_assert_write_locked(mm);
350
351 if (!len)
352 return -EINVAL;
353
354 /*
355 * Does the application expect PROT_READ to imply PROT_EXEC?
356 *
357 * (the exception is when the underlying filesystem is noexec
358 * mounted, in which case we don't add PROT_EXEC.)
359 */
360 if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
361 if (!(file && path_noexec(&file->f_path)))
362 prot |= PROT_EXEC;
363
364 /* force arch specific MAP_FIXED handling in get_unmapped_area */
365 if (flags & MAP_FIXED_NOREPLACE)
366 flags |= MAP_FIXED;
367
368 if (!(flags & MAP_FIXED))
369 addr = round_hint_to_min(addr);
370
371 /* Careful about overflows.. */
372 len = PAGE_ALIGN(len);
373 if (!len)
374 return -ENOMEM;
375
376 /* offset overflow? */
377 if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
378 return -EOVERFLOW;
379
380 /* Too many mappings? */
381 if (mm->map_count > get_sysctl_max_map_count())
382 return -ENOMEM;
383
384 /*
385 * addr is returned from get_unmapped_area,
386 * There are two cases:
387 * 1> MAP_FIXED == false
388 * unallocated memory, no need to check sealing.
389 * 1> MAP_FIXED == true
390 * sealing is checked inside mmap_region when
391 * do_vmi_munmap is called.
392 */
393
394 if (prot == PROT_EXEC) {
395 pkey = execute_only_pkey(mm);
396 if (pkey < 0)
397 pkey = 0;
398 }
399
400 /* Do simple checking here so the lower-level routines won't have
401 * to. we assume access permissions have been handled by the open
402 * of the memory object, so we don't do any here.
403 */
404 vma_flags_set_mask(&vma_flags,
405 legacy_to_vma_flags(calc_vm_prot_bits(prot, pkey)));
406 vma_flags_set_mask(&vma_flags,
407 legacy_to_vma_flags(calc_vm_flag_bits(file, flags)));
408 vma_flags_set_mask(&vma_flags, mm->def_vma_flags);
409 vma_flags_set(&vma_flags, VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT,
410 VMA_MAYEXEC_BIT);
411
412 /* Obtain the address to map to. we verify (or select) it and ensure
413 * that it represents a valid section of the address space.
414 */
415 addr = __get_unmapped_area(file, addr, len, pgoff, flags, vma_flags);
416 if (IS_ERR_VALUE(addr))
417 return addr;
418
419 if (flags & MAP_FIXED_NOREPLACE) {
420 if (find_vma_intersection(mm, addr, addr + len))
421 return -EEXIST;
422 }
423
424 if (flags & MAP_LOCKED)
425 if (!can_do_mlock())
426 return -EPERM;
427
428 if (!mlock_future_ok(mm, vma_flags_test(&vma_flags, VMA_LOCKED_BIT), len))
429 return -EAGAIN;
430
431 if (file) {
432 struct inode *inode = file_inode(file);
433 unsigned long flags_mask;
434 int err;
435
436 if (!file_mmap_ok(file, inode, pgoff, len))
437 return -EOVERFLOW;
438
439 flags_mask = LEGACY_MAP_MASK;
440 if (file->f_op->fop_flags & FOP_MMAP_SYNC)
441 flags_mask |= MAP_SYNC;
442
443 switch (flags & MAP_TYPE) {
444 case MAP_SHARED:
445 /*
446 * Force use of MAP_SHARED_VALIDATE with non-legacy
447 * flags. E.g. MAP_SYNC is dangerous to use with
448 * MAP_SHARED as you don't know which consistency model
449 * you will get. We silently ignore unsupported flags
450 * with MAP_SHARED to preserve backward compatibility.
451 */
452 flags &= LEGACY_MAP_MASK;
453 fallthrough;
454 case MAP_SHARED_VALIDATE:
455 if (flags & ~flags_mask)
456 return -EOPNOTSUPP;
457 if (prot & PROT_WRITE) {
458 if (!(file->f_mode & FMODE_WRITE))
459 return -EACCES;
460 if (IS_SWAPFILE(file->f_mapping->host))
461 return -ETXTBSY;
462 }
463
464 /*
465 * Make sure we don't allow writing to an append-only
466 * file..
467 */
468 if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
469 return -EACCES;
470
471 vma_flags_set(&vma_flags, VMA_SHARED_BIT, VMA_MAYSHARE_BIT);
472 if (!(file->f_mode & FMODE_WRITE))
473 vma_flags_clear(&vma_flags, VMA_MAYWRITE_BIT,
474 VMA_SHARED_BIT);
475 fallthrough;
476 case MAP_PRIVATE:
477 if (!(file->f_mode & FMODE_READ))
478 return -EACCES;
479 if (path_noexec(&file->f_path)) {
480 if (vma_flags_test(&vma_flags, VMA_EXEC_BIT))
481 return -EPERM;
482 vma_flags_clear(&vma_flags, VMA_MAYEXEC_BIT);
483 }
484
485 if (!can_mmap_file(file))
486 return -ENODEV;
487 if (vma_flags_can_grow(&vma_flags))
488 return -EINVAL;
489 break;
490
491 default:
492 return -EINVAL;
493 }
494
495 /*
496 * Check to see if we are violating any seals and update VMA
497 * flags if necessary to avoid future seal violations.
498 */
499 err = memfd_check_seals_mmap(file, &vma_flags);
500 if (err)
501 return (unsigned long)err;
502 } else {
503 switch (flags & MAP_TYPE) {
504 case MAP_SHARED:
505 if (vma_flags_can_grow(&vma_flags))
506 return -EINVAL;
507 /*
508 * Ignore pgoff.
509 */
510 pgoff = 0;
511 vma_flags_set(&vma_flags, VMA_SHARED_BIT, VMA_MAYSHARE_BIT);
512 break;
513 case MAP_DROPPABLE: {
514 vma_flags_t droppable = VMA_DROPPABLE;
515
516 if (vma_flags_empty(&droppable))
517 return -EOPNOTSUPP;
518 vma_flags_set_mask(&vma_flags, droppable);
519
520 /*
521 * A locked or stack area makes no sense to be droppable.
522 *
523 * Also, since droppable pages can just go away at any time
524 * it makes no sense to copy them on fork or dump them.
525 *
526 * And don't attempt to combine with hugetlb for now.
527 */
528 if (flags & (MAP_LOCKED | MAP_HUGETLB))
529 return -EINVAL;
530 if (vma_flags_can_grow(&vma_flags))
531 return -EINVAL;
532
533 /*
534 * If the pages can be dropped, then it doesn't make
535 * sense to reserve them.
536 */
537 vma_flags_set(&vma_flags, VMA_NORESERVE_BIT);
538
539 /*
540 * Likewise, they're volatile enough that they
541 * shouldn't survive forks or coredumps.
542 */
543 vma_flags_set(&vma_flags, VMA_WIPEONFORK_BIT,
544 VMA_DONTDUMP_BIT);
545
546 fallthrough;
547 }
548 case MAP_PRIVATE:
549 /*
550 * Set pgoff according to addr for anon_vma.
551 */
552 pgoff = addr >> PAGE_SHIFT;
553 break;
554 default:
555 return -EINVAL;
556 }
557 }
558
559 /*
560 * Set VMA_NORESERVE_BIT if we should not account for the memory use
561 * of this mapping.
562 */
563 if (flags & MAP_NORESERVE) {
564 /* We honor MAP_NORESERVE if allowed to overcommit */
565 if (sysctl_overcommit_memory != OVERCOMMIT_NEVER)
566 vma_flags_set(&vma_flags, VMA_NORESERVE_BIT);
567
568 /* hugetlb applies strict overcommit unless MAP_NORESERVE */
569 if (file && is_file_hugepages(file))
570 vma_flags_set(&vma_flags, VMA_NORESERVE_BIT);
571 }
572
573 addr = mmap_region(file, addr, len, vma_flags, pgoff, uf);
574 if (!IS_ERR_VALUE(addr) &&
575 (vma_flags_test(&vma_flags, VMA_LOCKED_BIT) ||
576 (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE))
577 *populate = len;
578 return addr;
579}
580
581unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len,
582 unsigned long prot, unsigned long flags,
583 unsigned long fd, unsigned long pgoff)
584{
585 struct file *file = NULL;
586 unsigned long retval;
587
588 if (!(flags & MAP_ANONYMOUS)) {
589 audit_mmap_fd(fd, flags);
590 file = fget(fd);
591 if (!file)
592 return -EBADF;
593 if (is_file_hugepages(file)) {
594 len = ALIGN(len, huge_page_size(hstate_file(file)));
595 } else if (unlikely(flags & MAP_HUGETLB)) {
596 retval = -EINVAL;
597 goto out_fput;
598 }
599 } else if (flags & MAP_HUGETLB) {
600 struct hstate *hs;
601
602 hs = hstate_sizelog((flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
603 if (!hs)
604 return -EINVAL;
605
606 len = ALIGN(len, huge_page_size(hs));
607 /*
608 * VM_NORESERVE is used because the reservations will be
609 * taken when vm_ops->mmap() is called
610 */
611 file = hugetlb_file_setup(HUGETLB_ANON_FILE, len,
612 mk_vma_flags(VMA_NORESERVE_BIT),
613 HUGETLB_ANONHUGE_INODE,
614 (flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
615 if (IS_ERR(file))
616 return PTR_ERR(file);
617 }
618
619 retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
620out_fput:
621 if (file)
622 fput(file);
623 return retval;
624}
625
626SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
627 unsigned long, prot, unsigned long, flags,
628 unsigned long, fd, unsigned long, pgoff)
629{
630 return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff);
631}
632
633#ifdef __ARCH_WANT_SYS_OLD_MMAP
634struct mmap_arg_struct {
635 unsigned long addr;
636 unsigned long len;
637 unsigned long prot;
638 unsigned long flags;
639 unsigned long fd;
640 unsigned long offset;
641};
642
643SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
644{
645 struct mmap_arg_struct a;
646
647 if (copy_from_user(&a, arg, sizeof(a)))
648 return -EFAULT;
649 if (offset_in_page(a.offset))
650 return -EINVAL;
651
652 return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
653 a.offset >> PAGE_SHIFT);
654}
655#endif /* __ARCH_WANT_SYS_OLD_MMAP */
656
657/*
658 * Determine if the allocation needs to ensure that there is no
659 * existing mapping within it's guard gaps, for use as start_gap.
660 */
661static inline unsigned long stack_guard_placement(vma_flags_t vma_flags)
662{
663 if (vma_flags_test_single_mask(&vma_flags, VMA_SHADOW_STACK))
664 return PAGE_SIZE;
665
666 return 0;
667}
668
669/*
670 * Search for an unmapped address range.
671 *
672 * We are looking for a range that:
673 * - does not intersect with any VMA;
674 * - is contained within the [low_limit, high_limit) interval;
675 * - is at least the desired size.
676 * - satisfies (begin_addr & align_mask) == (align_offset & align_mask)
677 */
678unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info)
679{
680 unsigned long addr;
681
682 if (info->flags & VM_UNMAPPED_AREA_TOPDOWN)
683 addr = unmapped_area_topdown(info);
684 else
685 addr = unmapped_area(info);
686
687 trace_vm_unmapped_area(addr, info);
688 return addr;
689}
690
691/* Get an address range which is currently unmapped.
692 * For shmat() with addr=0.
693 *
694 * Ugly calling convention alert:
695 * Return value with the low bits set means error value,
696 * ie
697 * if (ret & ~PAGE_MASK)
698 * error = ret;
699 *
700 * This function "knows" that -ENOMEM has the bits set.
701 */
702unsigned long
703generic_get_unmapped_area(struct file *filp, unsigned long addr,
704 unsigned long len, unsigned long pgoff,
705 unsigned long flags, vma_flags_t vma_flags)
706{
707 struct mm_struct *mm = current->mm;
708 struct vm_area_struct *vma, *prev;
709 struct vm_unmapped_area_info info = {};
710 const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags);
711
712 if (len > mmap_end - mmap_min_addr)
713 return -ENOMEM;
714
715 if (flags & MAP_FIXED)
716 return addr;
717
718 if (addr) {
719 addr = PAGE_ALIGN(addr);
720 vma = find_vma_prev(mm, addr, &prev);
721 if (mmap_end - len >= addr && addr >= mmap_min_addr &&
722 (!vma || addr + len <= vm_start_gap(vma)) &&
723 (!prev || addr >= vm_end_gap(prev)))
724 return addr;
725 }
726
727 info.length = len;
728 info.low_limit = mm->mmap_base;
729 info.high_limit = mmap_end;
730 info.start_gap = stack_guard_placement(vma_flags);
731 if (filp && is_file_hugepages(filp))
732 info.align_mask = huge_page_mask_align(filp);
733 return vm_unmapped_area(&info);
734}
735
736#ifndef HAVE_ARCH_UNMAPPED_AREA
737unsigned long
738arch_get_unmapped_area(struct file *filp, unsigned long addr,
739 unsigned long len, unsigned long pgoff,
740 unsigned long flags, vm_flags_t vm_flags)
741{
742 return generic_get_unmapped_area(filp, addr, len, pgoff, flags,
743 legacy_to_vma_flags(vm_flags));
744}
745#endif
746
747/*
748 * This mmap-allocator allocates new areas top-down from below the
749 * stack's low limit (the base):
750 */
751unsigned long
752generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
753 unsigned long len, unsigned long pgoff,
754 unsigned long flags, vma_flags_t vma_flags)
755{
756 struct vm_area_struct *vma, *prev;
757 struct mm_struct *mm = current->mm;
758 struct vm_unmapped_area_info info = {};
759 const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags);
760
761 /* requested length too big for entire address space */
762 if (len > mmap_end - mmap_min_addr)
763 return -ENOMEM;
764
765 if (flags & MAP_FIXED)
766 return addr;
767
768 /* requesting a specific address */
769 if (addr) {
770 addr = PAGE_ALIGN(addr);
771 vma = find_vma_prev(mm, addr, &prev);
772 if (mmap_end - len >= addr && addr >= mmap_min_addr &&
773 (!vma || addr + len <= vm_start_gap(vma)) &&
774 (!prev || addr >= vm_end_gap(prev)))
775 return addr;
776 }
777
778 info.flags = VM_UNMAPPED_AREA_TOPDOWN;
779 info.length = len;
780 info.low_limit = PAGE_SIZE;
781 info.high_limit = arch_get_mmap_base(addr, mm->mmap_base);
782 info.start_gap = stack_guard_placement(vma_flags);
783 if (filp && is_file_hugepages(filp))
784 info.align_mask = huge_page_mask_align(filp);
785 addr = vm_unmapped_area(&info);
786
787 /*
788 * A failed mmap() very likely causes application failure,
789 * so fall back to the bottom-up function here. This scenario
790 * can happen with large stack limits and large mmap()
791 * allocations.
792 */
793 if (offset_in_page(addr)) {
794 VM_BUG_ON(addr != -ENOMEM);
795 info.flags = 0;
796 info.low_limit = TASK_UNMAPPED_BASE;
797 info.high_limit = mmap_end;
798 addr = vm_unmapped_area(&info);
799 }
800
801 return addr;
802}
803
804#ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
805unsigned long
806arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
807 unsigned long len, unsigned long pgoff,
808 unsigned long flags, vm_flags_t vm_flags)
809{
810 return generic_get_unmapped_area_topdown(filp, addr, len, pgoff, flags,
811 legacy_to_vma_flags(vm_flags));
812}
813#endif
814
815unsigned long mm_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr,
816 unsigned long len, unsigned long pgoff, unsigned long flags,
817 vma_flags_t vma_flags)
818{
819 if (mm_flags_test(MMF_TOPDOWN, current->mm))
820 return arch_get_unmapped_area_topdown(filp, addr, len, pgoff,
821 flags, vma_flags_to_legacy(vma_flags));
822 return arch_get_unmapped_area(filp, addr, len, pgoff, flags,
823 vma_flags_to_legacy(vma_flags));
824}
825
826unsigned long
827__get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
828 unsigned long pgoff, unsigned long flags, vma_flags_t vma_flags)
829{
830 unsigned long (*get_area)(struct file *, unsigned long,
831 unsigned long, unsigned long, unsigned long)
832 = NULL;
833
834 unsigned long error = arch_mmap_check(addr, len, flags);
835 if (error)
836 return error;
837
838 /* Careful about overflows.. */
839 if (len > TASK_SIZE)
840 return -ENOMEM;
841
842 if (file) {
843 if (file->f_op->get_unmapped_area)
844 get_area = file->f_op->get_unmapped_area;
845 } else if (flags & MAP_SHARED) {
846 /*
847 * mmap_region() will call shmem_zero_setup() to create a file,
848 * so use shmem's get_unmapped_area in case it can be huge.
849 */
850 get_area = shmem_get_unmapped_area;
851 }
852
853 /* Always treat pgoff as zero for anonymous memory. */
854 if (!file)
855 pgoff = 0;
856
857 if (get_area) {
858 addr = get_area(file, addr, len, pgoff, flags);
859 } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && !file
860 && !addr /* no hint */
861 && IS_ALIGNED(len, PMD_SIZE)) {
862 /* Ensures that larger anonymous mappings are THP aligned. */
863 addr = thp_get_unmapped_area_vmaflags(file, addr, len,
864 pgoff, flags, vma_flags);
865 } else {
866 addr = mm_get_unmapped_area_vmaflags(file, addr, len,
867 pgoff, flags, vma_flags);
868 }
869 if (IS_ERR_VALUE(addr))
870 return addr;
871
872 if (addr > TASK_SIZE - len)
873 return -ENOMEM;
874 if (offset_in_page(addr))
875 return -EINVAL;
876
877 error = security_mmap_addr(addr);
878 return error ? error : addr;
879}
880
881unsigned long
882mm_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
883 unsigned long pgoff, unsigned long flags)
884{
885 return mm_get_unmapped_area_vmaflags(file, addr, len, pgoff, flags,
886 EMPTY_VMA_FLAGS);
887}
888EXPORT_SYMBOL(mm_get_unmapped_area);
889
890/**
891 * find_vma_intersection() - Look up the first VMA which intersects the interval
892 * @mm: The process address space.
893 * @start_addr: The inclusive start user address.
894 * @end_addr: The exclusive end user address.
895 *
896 * Returns: The first VMA within the provided range, %NULL otherwise. Assumes
897 * start_addr < end_addr.
898 */
899struct vm_area_struct *find_vma_intersection(struct mm_struct *mm,
900 unsigned long start_addr,
901 unsigned long end_addr)
902{
903 unsigned long index = start_addr;
904
905 mmap_assert_locked(mm);
906 return mt_find(&mm->mm_mt, &index, end_addr - 1);
907}
908EXPORT_SYMBOL(find_vma_intersection);
909
910/**
911 * find_vma() - Find the VMA for a given address, or the next VMA.
912 * @mm: The mm_struct to check
913 * @addr: The address
914 *
915 * Returns: The VMA associated with addr, or the next VMA.
916 * May return %NULL in the case of no VMA at addr or above.
917 */
918struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
919{
920 unsigned long index = addr;
921
922 mmap_assert_locked(mm);
923 return mt_find(&mm->mm_mt, &index, ULONG_MAX);
924}
925EXPORT_SYMBOL(find_vma);
926
927/**
928 * find_vma_prev() - Find the VMA for a given address, or the next vma and
929 * set %pprev to the previous VMA, if any.
930 * @mm: The mm_struct to check
931 * @addr: The address
932 * @pprev: The pointer to set to the previous VMA
933 *
934 * Note that RCU lock is missing here since the external mmap_lock() is used
935 * instead.
936 *
937 * Returns: The VMA associated with @addr, or the next vma.
938 * May return %NULL in the case of no vma at addr or above.
939 */
940struct vm_area_struct *
941find_vma_prev(struct mm_struct *mm, unsigned long addr,
942 struct vm_area_struct **pprev)
943{
944 struct vm_area_struct *vma;
945 VMA_ITERATOR(vmi, mm, addr);
946
947 vma = vma_iter_load(&vmi);
948 *pprev = vma_prev(&vmi);
949 if (!vma)
950 vma = vma_next(&vmi);
951 return vma;
952}
953
954/* enforced gap between the expanding stack and other mappings. */
955unsigned long stack_guard_gap = 256UL<<PAGE_SHIFT;
956
957static int __init cmdline_parse_stack_guard_gap(char *p)
958{
959 unsigned long val;
960 char *endptr;
961
962 val = simple_strtoul(p, &endptr, 10);
963 if (!*endptr)
964 stack_guard_gap = val << PAGE_SHIFT;
965
966 return 1;
967}
968__setup("stack_guard_gap=", cmdline_parse_stack_guard_gap);
969
970#ifdef CONFIG_STACK_GROWSUP
971int expand_stack_locked(struct vm_area_struct *vma, unsigned long address)
972{
973 return expand_upwards(vma, address);
974}
975
976struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr)
977{
978 struct vm_area_struct *vma, *prev;
979
980 addr &= PAGE_MASK;
981 vma = find_vma_prev(mm, addr, &prev);
982 if (vma && (vma->vm_start <= addr))
983 return vma;
984 if (!prev)
985 return NULL;
986 if (expand_stack_locked(prev, addr))
987 return NULL;
988 if (vma_test(prev, VMA_LOCKED_BIT))
989 populate_vma_page_range(prev, addr, prev->vm_end, NULL);
990 return prev;
991}
992#else
993int expand_stack_locked(struct vm_area_struct *vma, unsigned long address)
994{
995 return expand_downwards(vma, address);
996}
997
998struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr)
999{
1000 struct vm_area_struct *vma;
1001 unsigned long start;
1002
1003 addr &= PAGE_MASK;
1004 vma = find_vma(mm, addr);
1005 if (!vma)
1006 return NULL;
1007 if (vma->vm_start <= addr)
1008 return vma;
1009 start = vma->vm_start;
1010 if (expand_stack_locked(vma, addr))
1011 return NULL;
1012 if (vma_test(vma, VMA_LOCKED_BIT))
1013 populate_vma_page_range(vma, addr, start, NULL);
1014 return vma;
1015}
1016#endif
1017
1018#if defined(CONFIG_STACK_GROWSUP)
1019
1020#define vma_expand_up(vma,addr) expand_upwards(vma, addr)
1021#define vma_expand_down(vma, addr) (-EFAULT)
1022
1023#else
1024
1025#define vma_expand_up(vma,addr) (-EFAULT)
1026#define vma_expand_down(vma, addr) expand_downwards(vma, addr)
1027
1028#endif
1029
1030/*
1031 * expand_stack(): legacy interface for page faulting. Don't use unless
1032 * you have to.
1033 *
1034 * This is called with the mm locked for reading, drops the lock, takes
1035 * the lock for writing, tries to look up a vma again, expands it if
1036 * necessary, and downgrades the lock to reading again.
1037 *
1038 * If no vma is found or it can't be expanded, it returns NULL and has
1039 * dropped the lock.
1040 */
1041struct vm_area_struct *expand_stack(struct mm_struct *mm, unsigned long addr)
1042{
1043 struct vm_area_struct *vma, *prev;
1044
1045 mmap_read_unlock(mm);
1046 if (mmap_write_lock_killable(mm))
1047 return NULL;
1048
1049 vma = find_vma_prev(mm, addr, &prev);
1050 if (vma && vma->vm_start <= addr)
1051 goto success;
1052
1053 if (prev && !vma_expand_up(prev, addr)) {
1054 vma = prev;
1055 goto success;
1056 }
1057
1058 if (vma && !vma_expand_down(vma, addr))
1059 goto success;
1060
1061 mmap_write_unlock(mm);
1062 return NULL;
1063
1064success:
1065 mmap_write_downgrade(mm);
1066 return vma;
1067}
1068
1069/* do_munmap() - Wrapper function for non-maple tree aware do_munmap() calls.
1070 * @mm: The mm_struct
1071 * @start: The start address to munmap
1072 * @len: The length to be munmapped.
1073 * @uf: The userfaultfd list_head
1074 *
1075 * Return: 0 on success, error otherwise.
1076 */
1077int do_munmap(struct mm_struct *mm, unsigned long start, size_t len,
1078 struct list_head *uf)
1079{
1080 VMA_ITERATOR(vmi, mm, start);
1081
1082 return do_vmi_munmap(&vmi, mm, start, len, uf, false);
1083}
1084
1085int vm_munmap(unsigned long start, size_t len)
1086{
1087 return __vm_munmap(start, len, false);
1088}
1089EXPORT_SYMBOL(vm_munmap);
1090
1091SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
1092{
1093 addr = untagged_addr(addr);
1094 return __vm_munmap(addr, len, true);
1095}
1096
1097
1098/*
1099 * Emulation of deprecated remap_file_pages() syscall.
1100 */
1101SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size,
1102 unsigned long, prot, unsigned long, pgoff, unsigned long, flags)
1103{
1104
1105 struct mm_struct *mm = current->mm;
1106 struct vm_area_struct *vma;
1107 unsigned long populate = 0;
1108 unsigned long ret = -EINVAL;
1109 struct file *file;
1110 vm_flags_t vm_flags;
1111
1112 pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/mm/remap_file_pages.rst.\n",
1113 current->comm, current->pid);
1114
1115 if (prot)
1116 return ret;
1117 start = start & PAGE_MASK;
1118 size = size & PAGE_MASK;
1119
1120 if (start + size <= start)
1121 return ret;
1122
1123 /* Does pgoff wrap? */
1124 if (pgoff + (size >> PAGE_SHIFT) < pgoff)
1125 return ret;
1126
1127 if (mmap_read_lock_killable(mm))
1128 return -EINTR;
1129
1130 /*
1131 * Look up VMA under read lock first so we can perform the security
1132 * without holding locks (which can be problematic). We reacquire a
1133 * write lock later and check nothing changed underneath us.
1134 */
1135 vma = vma_lookup(mm, start);
1136
1137 if (!vma || !vma_test(vma, VMA_SHARED_BIT)) {
1138 mmap_read_unlock(mm);
1139 return -EINVAL;
1140 }
1141
1142 prot |= vma_test(vma, VMA_READ_BIT) ? PROT_READ : 0;
1143 prot |= vma_test(vma, VMA_WRITE_BIT) ? PROT_WRITE : 0;
1144 prot |= vma_test(vma, VMA_EXEC_BIT) ? PROT_EXEC : 0;
1145
1146 flags &= MAP_NONBLOCK;
1147 flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE;
1148 if (vma_test(vma, VMA_LOCKED_BIT))
1149 flags |= MAP_LOCKED;
1150
1151 /* Save vm_flags used to calculate prot and flags, and recheck later. */
1152 vm_flags = vma->vm_flags;
1153 file = get_file(vma->vm_file);
1154
1155 mmap_read_unlock(mm);
1156
1157 /* Call outside mmap_lock to be consistent with other callers. */
1158 ret = security_mmap_file(file, prot, flags);
1159 if (ret) {
1160 fput(file);
1161 return ret;
1162 }
1163
1164 ret = -EINVAL;
1165
1166 /* OK security check passed, take write lock + let it rip. */
1167 if (mmap_write_lock_killable(mm)) {
1168 fput(file);
1169 return -EINTR;
1170 }
1171
1172 vma = vma_lookup(mm, start);
1173
1174 if (!vma)
1175 goto out;
1176
1177 /* Make sure things didn't change under us. */
1178 if (vma->vm_flags != vm_flags)
1179 goto out;
1180 if (vma->vm_file != file)
1181 goto out;
1182
1183 if (start + size > vma->vm_end) {
1184 VMA_ITERATOR(vmi, mm, vma->vm_end);
1185 struct vm_area_struct *next, *prev = vma;
1186
1187 for_each_vma_range(vmi, next, start + size) {
1188 /* hole between vmas ? */
1189 if (next->vm_start != prev->vm_end)
1190 goto out;
1191
1192 if (next->vm_file != vma->vm_file)
1193 goto out;
1194
1195 if (next->vm_flags != vma->vm_flags)
1196 goto out;
1197
1198 if (start + size <= next->vm_end)
1199 break;
1200
1201 prev = next;
1202 }
1203
1204 if (!next)
1205 goto out;
1206 }
1207
1208 ret = do_mmap(vma->vm_file, start, size,
1209 prot, flags, EMPTY_VMA_FLAGS, pgoff, &populate, NULL);
1210out:
1211 mmap_write_unlock(mm);
1212 fput(file);
1213 if (populate)
1214 mm_populate(ret, populate);
1215 if (!IS_ERR_VALUE(ret))
1216 ret = 0;
1217 return ret;
1218}
1219
1220int vm_brk_flags(unsigned long addr, unsigned long request, bool is_exec)
1221{
1222 const vma_flags_t vma_flags = is_exec ?
1223 mk_vma_flags(VMA_EXEC_BIT) : EMPTY_VMA_FLAGS;
1224 struct mm_struct *mm = current->mm;
1225 struct vm_area_struct *vma = NULL;
1226 unsigned long len;
1227 int ret;
1228 bool populate;
1229 LIST_HEAD(uf);
1230 VMA_ITERATOR(vmi, mm, addr);
1231
1232 len = PAGE_ALIGN(request);
1233 if (len < request)
1234 return -ENOMEM;
1235 if (!len)
1236 return 0;
1237
1238 if (mmap_write_lock_killable(mm))
1239 return -EINTR;
1240
1241 ret = check_brk_limits(addr, len);
1242 if (ret)
1243 goto limits_failed;
1244
1245 ret = do_vmi_munmap(&vmi, mm, addr, len, &uf, 0);
1246 if (ret)
1247 goto munmap_failed;
1248
1249 vma = vma_prev(&vmi);
1250 ret = do_brk_flags(&vmi, vma, addr, len, vma_flags);
1251 populate = vma_flags_test(&mm->def_vma_flags, VMA_LOCKED_BIT);
1252 mmap_write_unlock(mm);
1253 userfaultfd_unmap_complete(mm, &uf);
1254 if (populate && !ret)
1255 mm_populate(addr, len);
1256 return ret;
1257
1258munmap_failed:
1259limits_failed:
1260 mmap_write_unlock(mm);
1261 return ret;
1262}
1263
1264static
1265unsigned long tear_down_vmas(struct mm_struct *mm, struct vma_iterator *vmi,
1266 struct vm_area_struct *vma, unsigned long end)
1267{
1268 unsigned long nr_accounted = 0;
1269 int count = 0;
1270
1271 mmap_assert_write_locked(mm);
1272 vma_iter_set(vmi, vma->vm_end);
1273 do {
1274 if (vma_test(vma, VMA_ACCOUNT_BIT))
1275 nr_accounted += vma_pages(vma);
1276 vma_mark_detached(vma);
1277 remove_vma(vma);
1278 count++;
1279 cond_resched();
1280 vma = vma_next(vmi);
1281 } while (vma && vma->vm_end <= end);
1282
1283 VM_WARN_ON_ONCE(count != mm->map_count);
1284 return nr_accounted;
1285}
1286
1287/* Release all mmaps. */
1288void exit_mmap(struct mm_struct *mm)
1289{
1290 struct mmu_gather tlb;
1291 struct vm_area_struct *vma;
1292 unsigned long nr_accounted = 0;
1293 VMA_ITERATOR(vmi, mm, 0);
1294 struct unmap_desc unmap;
1295
1296 /* mm's last user has gone, and its about to be pulled down */
1297 mmu_notifier_release(mm);
1298
1299 mmap_read_lock(mm);
1300 arch_exit_mmap(mm);
1301
1302 vma = vma_next(&vmi);
1303 if (!vma) {
1304 /* Can happen if dup_mmap() received an OOM */
1305 mmap_read_unlock(mm);
1306 mmap_write_lock(mm);
1307 goto destroy;
1308 }
1309
1310 unmap_all_init(&unmap, &vmi, vma);
1311 flush_cache_mm(mm);
1312 tlb_gather_mmu_fullmm(&tlb, mm);
1313 /* update_hiwater_rss(mm) here? but nobody should be looking */
1314 /* Use ULONG_MAX here to ensure all VMAs in the mm are unmapped */
1315 unmap_vmas(&tlb, &unmap);
1316 mmap_read_unlock(mm);
1317
1318 /*
1319 * Set MMF_OOM_SKIP to hide this task from the oom killer/reaper
1320 * because the memory has been already freed.
1321 */
1322 mm_flags_set(MMF_OOM_SKIP, mm);
1323 mmap_write_lock(mm);
1324 unmap.mm_wr_locked = true;
1325 mt_clear_in_rcu(&mm->mm_mt);
1326 unmap_pgtable_init(&unmap, &vmi);
1327 free_pgtables(&tlb, &unmap);
1328 tlb_finish_mmu(&tlb);
1329
1330 /*
1331 * Walk the list again, actually closing and freeing it, with preemption
1332 * enabled, without holding any MM locks besides the unreachable
1333 * mmap_write_lock.
1334 */
1335 nr_accounted = tear_down_vmas(mm, &vmi, vma, ULONG_MAX);
1336
1337destroy:
1338 __mt_destroy(&mm->mm_mt);
1339 trace_exit_mmap(mm);
1340 mmap_write_unlock(mm);
1341 vm_unacct_memory(nr_accounted);
1342}
1343
1344/*
1345 * Return true if the calling process may expand its vm space by the passed
1346 * number of pages
1347 */
1348bool may_expand_vm(struct mm_struct *mm, const vma_flags_t *vma_flags,
1349 unsigned long npages)
1350{
1351 if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> PAGE_SHIFT)
1352 return false;
1353
1354 if (is_data_mapping_vma_flags(vma_flags) &&
1355 mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) {
1356 /* Workaround for Valgrind */
1357 if (rlimit(RLIMIT_DATA) == 0 &&
1358 mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> PAGE_SHIFT)
1359 return true;
1360
1361 pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Update limits%s.\n",
1362 current->comm, current->pid,
1363 (mm->data_vm + npages) << PAGE_SHIFT,
1364 rlimit(RLIMIT_DATA),
1365 ignore_rlimit_data ? "" : " or use boot option ignore_rlimit_data");
1366
1367 if (!ignore_rlimit_data)
1368 return false;
1369 }
1370
1371 return true;
1372}
1373
1374void vm_stat_account(struct mm_struct *mm, vm_flags_t flags, long npages)
1375{
1376 WRITE_ONCE(mm->total_vm, READ_ONCE(mm->total_vm)+npages);
1377
1378 if (is_exec_mapping(flags))
1379 mm->exec_vm += npages;
1380 else if (is_stack_mapping(flags))
1381 mm->stack_vm += npages;
1382 else if (is_data_mapping(flags))
1383 mm->data_vm += npages;
1384}
1385
1386static vm_fault_t special_mapping_fault(struct vm_fault *vmf);
1387
1388/*
1389 * Close hook, called for unmap() and on the old vma for mremap().
1390 *
1391 * Having a close hook prevents vma merging regardless of flags.
1392 */
1393static void special_mapping_close(struct vm_area_struct *vma)
1394{
1395 const struct vm_special_mapping *sm = vma->vm_private_data;
1396
1397 if (sm->close)
1398 sm->close(sm, vma);
1399}
1400
1401static const char *special_mapping_name(struct vm_area_struct *vma)
1402{
1403 return ((struct vm_special_mapping *)vma->vm_private_data)->name;
1404}
1405
1406static int special_mapping_mremap(struct vm_area_struct *new_vma)
1407{
1408 struct vm_special_mapping *sm = new_vma->vm_private_data;
1409
1410 if (WARN_ON_ONCE(current->mm != new_vma->vm_mm))
1411 return -EFAULT;
1412
1413 if (sm->mremap)
1414 return sm->mremap(sm, new_vma);
1415
1416 return 0;
1417}
1418
1419static int special_mapping_split(struct vm_area_struct *vma, unsigned long addr)
1420{
1421 /*
1422 * Forbid splitting special mappings - kernel has expectations over
1423 * the number of pages in mapping. Together with VMA_DONTEXPAND_BIT
1424 * the size of vma should stay the same over the special mapping's
1425 * lifetime.
1426 */
1427 return -EINVAL;
1428}
1429
1430static const struct vm_operations_struct special_mapping_vmops = {
1431 .close = special_mapping_close,
1432 .fault = special_mapping_fault,
1433 .mremap = special_mapping_mremap,
1434 .name = special_mapping_name,
1435 /* vDSO code relies that VVAR can't be accessed remotely */
1436 .access = NULL,
1437 .may_split = special_mapping_split,
1438};
1439
1440static vm_fault_t special_mapping_fault(struct vm_fault *vmf)
1441{
1442 struct vm_area_struct *vma = vmf->vma;
1443 pgoff_t pgoff;
1444 struct page **pages;
1445 struct vm_special_mapping *sm = vma->vm_private_data;
1446
1447 if (sm->fault)
1448 return sm->fault(sm, vmf->vma, vmf);
1449
1450 pages = sm->pages;
1451
1452 for (pgoff = vmf->pgoff; pgoff && *pages; ++pages)
1453 pgoff--;
1454
1455 if (*pages) {
1456 struct page *page = *pages;
1457 get_page(page);
1458 vmf->page = page;
1459 return 0;
1460 }
1461
1462 return VM_FAULT_SIGBUS;
1463}
1464
1465bool vma_is_special_mapping(const struct vm_area_struct *vma,
1466 const struct vm_special_mapping *sm)
1467{
1468 return vma->vm_private_data == sm &&
1469 vma->vm_ops == &special_mapping_vmops;
1470}
1471
1472/*
1473 * Called with mm->mmap_lock held for writing.
1474 * Insert a new vma covering the given region, with the given flags.
1475 * Its pages are supplied by the given array of struct page *.
1476 * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated.
1477 * The region past the last page supplied will always produce SIGBUS.
1478 * The array pointer and the pages it points to are assumed to stay alive
1479 * for as long as this mapping might exist.
1480 */
1481struct vm_area_struct *_install_special_mapping(
1482 struct mm_struct *mm,
1483 unsigned long addr, unsigned long len,
1484 vm_flags_t vm_flags, const struct vm_special_mapping *spec)
1485{
1486 return __install_special_mapping(mm, addr, len, vm_flags, (void *)spec,
1487 &special_mapping_vmops);
1488}
1489
1490#ifdef CONFIG_SYSCTL
1491#if defined(HAVE_ARCH_PICK_MMAP_LAYOUT) || \
1492 defined(CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT)
1493int sysctl_legacy_va_layout;
1494#endif
1495
1496static const struct ctl_table mmap_table[] = {
1497 {
1498 .procname = "max_map_count",
1499 .data = &sysctl_max_map_count,
1500 .maxlen = sizeof(sysctl_max_map_count),
1501 .mode = 0644,
1502 .proc_handler = proc_dointvec_minmax,
1503 .extra1 = SYSCTL_ZERO,
1504 },
1505#if defined(HAVE_ARCH_PICK_MMAP_LAYOUT) || \
1506 defined(CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT)
1507 {
1508 .procname = "legacy_va_layout",
1509 .data = &sysctl_legacy_va_layout,
1510 .maxlen = sizeof(sysctl_legacy_va_layout),
1511 .mode = 0644,
1512 .proc_handler = proc_dointvec_minmax,
1513 .extra1 = SYSCTL_ZERO,
1514 },
1515#endif
1516#ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS
1517 {
1518 .procname = "mmap_rnd_bits",
1519 .data = &mmap_rnd_bits,
1520 .maxlen = sizeof(mmap_rnd_bits),
1521 .mode = 0600,
1522 .proc_handler = proc_dointvec_minmax,
1523 .extra1 = (void *)&mmap_rnd_bits_min,
1524 .extra2 = (void *)&mmap_rnd_bits_max,
1525 },
1526#endif
1527#ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
1528 {
1529 .procname = "mmap_rnd_compat_bits",
1530 .data = &mmap_rnd_compat_bits,
1531 .maxlen = sizeof(mmap_rnd_compat_bits),
1532 .mode = 0600,
1533 .proc_handler = proc_dointvec_minmax,
1534 .extra1 = (void *)&mmap_rnd_compat_bits_min,
1535 .extra2 = (void *)&mmap_rnd_compat_bits_max,
1536 },
1537#endif
1538};
1539#endif /* CONFIG_SYSCTL */
1540
1541/*
1542 * initialise the percpu counter for VM, initialise VMA state.
1543 */
1544void __init mmap_init(void)
1545{
1546 int ret;
1547
1548 ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL);
1549 VM_BUG_ON(ret);
1550#ifdef CONFIG_SYSCTL
1551 register_sysctl_init("vm", mmap_table);
1552#endif
1553 vma_state_init();
1554}
1555
1556/*
1557 * Initialise sysctl_user_reserve_kbytes.
1558 *
1559 * This is intended to prevent a user from starting a single memory hogging
1560 * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
1561 * mode.
1562 *
1563 * The default value is min(3% of free memory, 128MB)
1564 * 128MB is enough to recover with sshd/login, bash, and top/kill.
1565 */
1566static int init_user_reserve(void)
1567{
1568 unsigned long free_kbytes;
1569
1570 free_kbytes = K(global_zone_page_state(NR_FREE_PAGES));
1571
1572 sysctl_user_reserve_kbytes = min(free_kbytes / 32, SZ_128K);
1573 return 0;
1574}
1575subsys_initcall(init_user_reserve);
1576
1577/*
1578 * Initialise sysctl_admin_reserve_kbytes.
1579 *
1580 * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
1581 * to log in and kill a memory hogging process.
1582 *
1583 * Systems with more than 256MB will reserve 8MB, enough to recover
1584 * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
1585 * only reserve 3% of free pages by default.
1586 */
1587static int init_admin_reserve(void)
1588{
1589 unsigned long free_kbytes;
1590
1591 free_kbytes = K(global_zone_page_state(NR_FREE_PAGES));
1592
1593 sysctl_admin_reserve_kbytes = min(free_kbytes / 32, SZ_8K);
1594 return 0;
1595}
1596subsys_initcall(init_admin_reserve);
1597
1598/*
1599 * Reinititalise user and admin reserves if memory is added or removed.
1600 *
1601 * The default user reserve max is 128MB, and the default max for the
1602 * admin reserve is 8MB. These are usually, but not always, enough to
1603 * enable recovery from a memory hogging process using login/sshd, a shell,
1604 * and tools like top. It may make sense to increase or even disable the
1605 * reserve depending on the existence of swap or variations in the recovery
1606 * tools. So, the admin may have changed them.
1607 *
1608 * If memory is added and the reserves have been eliminated or increased above
1609 * the default max, then we'll trust the admin.
1610 *
1611 * If memory is removed and there isn't enough free memory, then we
1612 * need to reset the reserves.
1613 *
1614 * Otherwise keep the reserve set by the admin.
1615 */
1616static int reserve_mem_notifier(struct notifier_block *nb,
1617 unsigned long action, void *data)
1618{
1619 unsigned long tmp, free_kbytes;
1620
1621 switch (action) {
1622 case MEM_ONLINE:
1623 /* Default max is 128MB. Leave alone if modified by operator. */
1624 tmp = sysctl_user_reserve_kbytes;
1625 if (tmp > 0 && tmp < SZ_128K)
1626 init_user_reserve();
1627
1628 /* Default max is 8MB. Leave alone if modified by operator. */
1629 tmp = sysctl_admin_reserve_kbytes;
1630 if (tmp > 0 && tmp < SZ_8K)
1631 init_admin_reserve();
1632
1633 break;
1634 case MEM_OFFLINE:
1635 free_kbytes = K(global_zone_page_state(NR_FREE_PAGES));
1636
1637 if (sysctl_user_reserve_kbytes > free_kbytes) {
1638 init_user_reserve();
1639 pr_info("vm.user_reserve_kbytes reset to %lu\n",
1640 sysctl_user_reserve_kbytes);
1641 }
1642
1643 if (sysctl_admin_reserve_kbytes > free_kbytes) {
1644 init_admin_reserve();
1645 pr_info("vm.admin_reserve_kbytes reset to %lu\n",
1646 sysctl_admin_reserve_kbytes);
1647 }
1648 break;
1649 default:
1650 break;
1651 }
1652 return NOTIFY_OK;
1653}
1654
1655static int __meminit init_reserve_notifier(void)
1656{
1657 if (hotplug_memory_notifier(reserve_mem_notifier, DEFAULT_CALLBACK_PRI))
1658 pr_err("Failed registering memory add/remove notifier for admin reserve\n");
1659
1660 return 0;
1661}
1662subsys_initcall(init_reserve_notifier);
1663
1664/*
1665 * Obtain a read lock on mm->mmap_lock, if the specified address is below the
1666 * start of the VMA, the intent is to perform a write, and it is a
1667 * downward-growing stack, then attempt to expand the stack to contain it.
1668 *
1669 * This function is intended only for obtaining an argument page from an ELF
1670 * image, and is almost certainly NOT what you want to use for any other
1671 * purpose.
1672 *
1673 * IMPORTANT - VMA fields are accessed without an mmap lock being held, so the
1674 * VMA referenced must not be linked in any user-visible tree, i.e. it must be a
1675 * new VMA being mapped.
1676 *
1677 * The function assumes that addr is either contained within the VMA or below
1678 * it, and makes no attempt to validate this value beyond that.
1679 *
1680 * Returns true if the read lock was obtained and a stack was perhaps expanded,
1681 * false if the stack expansion failed.
1682 *
1683 * On stack expansion the function temporarily acquires an mmap write lock
1684 * before downgrading it.
1685 */
1686bool mmap_read_lock_maybe_expand(struct mm_struct *mm,
1687 struct vm_area_struct *new_vma,
1688 unsigned long addr, bool write)
1689{
1690 if (!write || addr >= new_vma->vm_start) {
1691 mmap_read_lock(mm);
1692 return true;
1693 }
1694
1695 if (!vma_test(new_vma, VMA_GROWSDOWN_BIT))
1696 return false;
1697
1698 mmap_write_lock(mm);
1699 if (expand_downwards(new_vma, addr)) {
1700 mmap_write_unlock(mm);
1701 return false;
1702 }
1703
1704 mmap_write_downgrade(mm);
1705 return true;
1706}
1707
1708__latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
1709{
1710 struct vm_area_struct *mpnt, *tmp;
1711 int retval;
1712 unsigned long charge = 0;
1713 LIST_HEAD(uf);
1714 VMA_ITERATOR(vmi, mm, 0);
1715
1716 if (mmap_write_lock_killable(oldmm))
1717 return -EINTR;
1718 flush_cache_dup_mm(oldmm);
1719 uprobe_dup_mmap(oldmm, mm);
1720 /*
1721 * Not linked in yet - no deadlock potential:
1722 */
1723 mmap_write_lock_nested(mm, SINGLE_DEPTH_NESTING);
1724
1725 /* No ordering required: file already has been exposed. */
1726 dup_mm_exe_file(mm, oldmm);
1727
1728 mm->total_vm = oldmm->total_vm;
1729 mm->data_vm = oldmm->data_vm;
1730 mm->exec_vm = oldmm->exec_vm;
1731 mm->stack_vm = oldmm->stack_vm;
1732
1733 /* Use __mt_dup() to efficiently build an identical maple tree. */
1734 retval = __mt_dup(&oldmm->mm_mt, &mm->mm_mt, GFP_KERNEL);
1735 if (unlikely(retval))
1736 goto out;
1737
1738 mt_clear_in_rcu(vmi.mas.tree);
1739 for_each_vma(vmi, mpnt) {
1740 struct file *file;
1741
1742 retval = vma_start_write_killable(mpnt);
1743 if (retval < 0)
1744 goto loop_out;
1745 if (vma_test(mpnt, VMA_DONTCOPY_BIT)) {
1746 retval = vma_iter_clear_gfp(&vmi, mpnt->vm_start,
1747 mpnt->vm_end, GFP_KERNEL);
1748 if (retval)
1749 goto loop_out;
1750
1751 vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt));
1752 continue;
1753 }
1754 charge = 0;
1755 if (vma_test(mpnt, VMA_ACCOUNT_BIT)) {
1756 unsigned long len = vma_pages(mpnt);
1757
1758 if (security_vm_enough_memory_mm(oldmm, len)) /* sic */
1759 goto fail_nomem;
1760 charge = len;
1761 }
1762
1763 tmp = vm_area_dup(mpnt);
1764 if (!tmp)
1765 goto fail_nomem;
1766 retval = vma_dup_policy(mpnt, tmp);
1767 if (retval)
1768 goto fail_nomem_policy;
1769 tmp->vm_mm = mm;
1770 retval = dup_userfaultfd(tmp, &uf);
1771 if (retval)
1772 goto fail_nomem_anon_vma_fork;
1773
1774 if (vma_test(tmp, VMA_WIPEONFORK_BIT)) {
1775 /*
1776 * VMA_WIPEONFORK_BIT gets a clean slate in the child.
1777 * Don't prepare anon_vma until fault since we don't
1778 * copy page for current vma.
1779 */
1780 tmp->anon_vma = NULL;
1781 } else if (anon_vma_fork(tmp, mpnt))
1782 goto fail_nomem_anon_vma_fork;
1783
1784 vma_start_write(tmp);
1785 vma_clear_flags_mask(tmp, VMA_LOCKED_MASK);
1786 /*
1787 * Copy/update hugetlb private vma information.
1788 */
1789 if (is_vm_hugetlb_page(tmp))
1790 hugetlb_dup_vma_private(tmp);
1791
1792 /*
1793 * Link the vma into the MT. After using __mt_dup(), memory
1794 * allocation is not necessary here, so it cannot fail.
1795 */
1796 vma_iter_bulk_store(&vmi, tmp);
1797
1798 mm->map_count++;
1799
1800 if (tmp->vm_ops && tmp->vm_ops->open)
1801 tmp->vm_ops->open(tmp);
1802
1803 file = tmp->vm_file;
1804 if (file) {
1805 struct address_space *mapping = file->f_mapping;
1806
1807 get_file(file);
1808 i_mmap_lock_write(mapping);
1809 if (vma_is_shared_maywrite(tmp))
1810 mapping_allow_writable(mapping);
1811 flush_dcache_mmap_lock(mapping);
1812 /* insert tmp into the share list, just after mpnt */
1813 mapping_rmap_tree_insert_after(tmp, mpnt, mapping);
1814 flush_dcache_mmap_unlock(mapping);
1815 i_mmap_unlock_write(mapping);
1816 }
1817
1818 if (!vma_test(tmp, VMA_WIPEONFORK_BIT))
1819 retval = copy_page_range(tmp, mpnt);
1820
1821 if (retval) {
1822 mpnt = vma_next(&vmi);
1823 goto loop_out;
1824 }
1825 }
1826 /* a new mm has just been created */
1827 retval = arch_dup_mmap(oldmm, mm);
1828loop_out:
1829 vma_iter_free(&vmi);
1830 if (!retval) {
1831 mt_set_in_rcu(vmi.mas.tree);
1832 ksm_fork(mm, oldmm);
1833 khugepaged_fork(mm, oldmm);
1834 } else {
1835 unsigned long end;
1836
1837 /*
1838 * The entire maple tree has already been duplicated, but
1839 * replacing the vmas failed at mpnt (which could be NULL if
1840 * all were allocated but the last vma was not fully set up).
1841 * Use the start address of the failure point to clean up the
1842 * partially initialized tree.
1843 */
1844 if (!mm->map_count) {
1845 /* zero vmas were written to the new tree. */
1846 end = 0;
1847 } else if (mpnt) {
1848 /* partial tree failure */
1849 end = mpnt->vm_start;
1850 } else {
1851 /* All vmas were written to the new tree */
1852 end = ULONG_MAX;
1853 }
1854
1855 /* Hide mm from oom killer because the memory is being freed */
1856 mm_flags_set(MMF_OOM_SKIP, mm);
1857 if (end) {
1858 vma_iter_set(&vmi, 0);
1859 tmp = vma_next(&vmi);
1860 UNMAP_STATE(unmap, &vmi, /* first = */ tmp,
1861 /* vma_start = */ 0, /* vma_end = */ end,
1862 /* prev = */ NULL, /* next = */ NULL);
1863
1864 /*
1865 * Don't iterate over vmas beyond the failure point for
1866 * both unmap_vma() and free_pgtables().
1867 */
1868 unmap.tree_end = end;
1869 flush_cache_mm(mm);
1870 unmap_region(&unmap);
1871 charge = tear_down_vmas(mm, &vmi, tmp, end);
1872 vm_unacct_memory(charge);
1873 }
1874 __mt_destroy(&mm->mm_mt);
1875 /*
1876 * The mm_struct is going to exit, but the locks will be dropped
1877 * first. Set the mm_struct as unstable is advisable as it is
1878 * not fully initialised.
1879 */
1880 mm_flags_set(MMF_UNSTABLE, mm);
1881 }
1882out:
1883 mmap_write_unlock(mm);
1884 flush_tlb_mm(oldmm);
1885 mmap_write_unlock(oldmm);
1886 if (!retval)
1887 dup_userfaultfd_complete(&uf);
1888 else
1889 dup_userfaultfd_fail(&uf);
1890 return retval;
1891
1892fail_nomem_anon_vma_fork:
1893 mpol_put(vma_policy(tmp));
1894fail_nomem_policy:
1895 vm_area_free(tmp);
1896fail_nomem:
1897 retval = -ENOMEM;
1898 vm_unacct_memory(charge);
1899 goto loop_out;
1900}