| 1 | |
| 2 | |
| 3 | * mm/mprotect.c
|
| 4 | *
|
| 5 | * (C) Copyright 1994 Linus Torvalds
|
| 6 | * (C) Copyright 2002 Christoph Hellwig
|
| 7 | *
|
| 8 | * Address space accounting code <[email protected]>
|
| 9 | * (C) Copyright 2002 Red Hat Inc, All Rights Reserved
|
| 10 |
|
| 11 |
|
| 12 | #include <linux/pagewalk.h>
|
| 13 | #include <linux/hugetlb.h>
|
| 14 | #include <linux/shm.h>
|
| 15 | #include <linux/mman.h>
|
| 16 | #include <linux/fs.h>
|
| 17 | #include <linux/highmem.h>
|
| 18 | #include <linux/security.h>
|
| 19 | #include <linux/mempolicy.h>
|
| 20 | #include <linux/personality.h>
|
| 21 | #include <linux/syscalls.h>
|
| 22 | #include <linux/swap.h>
|
| 23 | #include <linux/swapops.h>
|
| 24 | #include <linux/mmu_notifier.h>
|
| 25 | #include <linux/migrate.h>
|
| 26 | #include <linux/perf_event.h>
|
| 27 | #include <linux/pkeys.h>
|
| 28 | #include <linux/ksm.h>
|
| 29 | #include <linux/uaccess.h>
|
| 30 | #include <linux/mm_inline.h>
|
| 31 | #include <linux/pgtable.h>
|
| 32 | #include <linux/userfaultfd_k.h>
|
| 33 | #include <uapi/linux/mman.h>
|
| 34 | #include <asm/cacheflush.h>
|
| 35 | #include <asm/mmu_context.h>
|
| 36 | #include <asm/tlbflush.h>
|
| 37 | #include <asm/tlb.h>
|
| 38 |
|
| 39 | #include "internal.h"
|
| 40 |
|
| 41 | static bool maybe_change_pte_writable(struct vm_area_struct *vma, pte_t pte)
|
| 42 | {
|
| 43 | if (WARN_ON_ONCE(!vma_test(vma, VMA_WRITE_BIT)))
|
| 44 | return false;
|
| 45 |
|
| 46 |
|
| 47 | if (pte_protnone(pte))
|
| 48 | return false;
|
| 49 |
|
| 50 |
|
| 51 | if (pte_needs_soft_dirty_wp(vma, pte))
|
| 52 | return false;
|
| 53 |
|
| 54 |
|
| 55 | if (userfaultfd_pte_wp(vma, pte))
|
| 56 | return false;
|
| 57 |
|
| 58 | return true;
|
| 59 | }
|
| 60 |
|
| 61 | static bool can_change_private_pte_writable(struct vm_area_struct *vma,
|
| 62 | unsigned long addr, pte_t pte)
|
| 63 | {
|
| 64 | struct page *page;
|
| 65 |
|
| 66 | if (!maybe_change_pte_writable(vma, pte))
|
| 67 | return false;
|
| 68 |
|
| 69 | |
| 70 | * Writable MAP_PRIVATE mapping: We can only special-case on
|
| 71 | * exclusive anonymous pages, because we know that our
|
| 72 | * write-fault handler similarly would map them writable without
|
| 73 | * any additional checks while holding the PT lock.
|
| 74 |
|
| 75 | page = vm_normal_page(vma, addr, pte);
|
| 76 | return page && PageAnon(page) && PageAnonExclusive(page);
|
| 77 | }
|
| 78 |
|
| 79 | static bool can_change_shared_pte_writable(struct vm_area_struct *vma,
|
| 80 | pte_t pte)
|
| 81 | {
|
| 82 | if (!maybe_change_pte_writable(vma, pte))
|
| 83 | return false;
|
| 84 |
|
| 85 | VM_WARN_ON_ONCE(is_zero_pfn(pte_pfn(pte)) && pte_dirty(pte));
|
| 86 |
|
| 87 | |
| 88 | * Writable MAP_SHARED mapping: "clean" might indicate that the FS still
|
| 89 | * needs a real write-fault for writenotify
|
| 90 | * (see vma_wants_writenotify()). If "dirty", the assumption is that the
|
| 91 | * FS was already notified and we can simply mark the PTE writable
|
| 92 | * just like the write-fault handler would do.
|
| 93 |
|
| 94 | return pte_dirty(pte);
|
| 95 | }
|
| 96 |
|
| 97 | bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
|
| 98 | pte_t pte)
|
| 99 | {
|
| 100 | if (!vma_test(vma, VMA_SHARED_BIT))
|
| 101 | return can_change_private_pte_writable(vma, addr, pte);
|
| 102 |
|
| 103 | return can_change_shared_pte_writable(vma, pte);
|
| 104 | }
|
| 105 |
|
| 106 | static int mprotect_folio_pte_batch(struct folio *folio, pte_t *ptep,
|
| 107 | pte_t pte, int max_nr_ptes, fpb_t flags)
|
| 108 | {
|
| 109 |
|
| 110 | if (!folio)
|
| 111 | return 1;
|
| 112 |
|
| 113 | if (!folio_test_large(folio))
|
| 114 | return 1;
|
| 115 |
|
| 116 | return folio_pte_batch_flags(folio, NULL, ptep, &pte, max_nr_ptes, flags);
|
| 117 | }
|
| 118 |
|
| 119 |
|
| 120 | static __always_inline void prot_commit_flush_ptes(struct vm_area_struct *vma,
|
| 121 | unsigned long addr, pte_t *ptep, pte_t oldpte, pte_t ptent,
|
| 122 | int nr_ptes, int idx, bool set_write, struct mmu_gather *tlb)
|
| 123 | {
|
| 124 | |
| 125 | * Advance the position in the batch by idx; note that if idx > 0,
|
| 126 | * then the nr_ptes passed here is <= batch size - idx.
|
| 127 |
|
| 128 | addr += idx * PAGE_SIZE;
|
| 129 | ptep += idx;
|
| 130 | oldpte = pte_advance_pfn(oldpte, idx);
|
| 131 | ptent = pte_advance_pfn(ptent, idx);
|
| 132 |
|
| 133 | if (set_write)
|
| 134 | ptent = pte_mkwrite(ptent, vma);
|
| 135 |
|
| 136 | modify_prot_commit_ptes(vma, addr, ptep, oldpte, ptent, nr_ptes);
|
| 137 | if (pte_needs_flush(oldpte, ptent))
|
| 138 | tlb_flush_pte_range(tlb, addr, nr_ptes * PAGE_SIZE);
|
| 139 | }
|
| 140 |
|
| 141 | |
| 142 | * Get max length of consecutive ptes pointing to PageAnonExclusive() pages or
|
| 143 | * !PageAnonExclusive() pages, starting from start_idx. Caller must enforce
|
| 144 | * that the ptes point to consecutive pages of the same anon large folio.
|
| 145 |
|
| 146 | static __always_inline int page_anon_exclusive_batch(int start_idx, int max_len,
|
| 147 | struct page *first_page, bool expected_anon_exclusive)
|
| 148 | {
|
| 149 | int idx;
|
| 150 |
|
| 151 | for (idx = start_idx + 1; idx < start_idx + max_len; ++idx) {
|
| 152 | if (expected_anon_exclusive != PageAnonExclusive(first_page + idx))
|
| 153 | break;
|
| 154 | }
|
| 155 | return idx - start_idx;
|
| 156 | }
|
| 157 |
|
| 158 | |
| 159 | * This function is a result of trying our very best to retain the
|
| 160 | * "avoid the write-fault handler" optimization. In can_change_pte_writable(),
|
| 161 | * if the vma is a private vma, and we cannot determine whether to change
|
| 162 | * the pte to writable just from the vma and the pte, we then need to look
|
| 163 | * at the actual page pointed to by the pte. Unfortunately, if we have a
|
| 164 | * batch of ptes pointing to consecutive pages of the same anon large folio,
|
| 165 | * the anon-exclusivity (or the negation) of the first page does not guarantee
|
| 166 | * the anon-exclusivity (or the negation) of the other pages corresponding to
|
| 167 | * the pte batch; hence in this case it is incorrect to decide to change or
|
| 168 | * not change the ptes to writable just by using information from the first
|
| 169 | * pte of the batch. Therefore, we must individually check all pages and
|
| 170 | * retrieve sub-batches.
|
| 171 |
|
| 172 | static __always_inline void commit_anon_folio_batch(struct vm_area_struct *vma,
|
| 173 | struct folio *folio, struct page *first_page, unsigned long addr, pte_t *ptep,
|
| 174 | pte_t oldpte, pte_t ptent, int nr_ptes, struct mmu_gather *tlb)
|
| 175 | {
|
| 176 | bool expected_anon_exclusive;
|
| 177 | int batch_idx = 0;
|
| 178 | int len;
|
| 179 |
|
| 180 | while (nr_ptes) {
|
| 181 | expected_anon_exclusive = PageAnonExclusive(first_page + batch_idx);
|
| 182 | len = page_anon_exclusive_batch(batch_idx, nr_ptes,
|
| 183 | first_page, expected_anon_exclusive);
|
| 184 | prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, len,
|
| 185 | batch_idx, expected_anon_exclusive, tlb);
|
| 186 | batch_idx += len;
|
| 187 | nr_ptes -= len;
|
| 188 | }
|
| 189 | }
|
| 190 |
|
| 191 | static __always_inline void set_write_prot_commit_flush_ptes(struct vm_area_struct *vma,
|
| 192 | struct folio *folio, struct page *page, unsigned long addr, pte_t *ptep,
|
| 193 | pte_t oldpte, pte_t ptent, int nr_ptes, struct mmu_gather *tlb)
|
| 194 | {
|
| 195 | bool set_write;
|
| 196 |
|
| 197 | if (vma_test(vma, VMA_SHARED_BIT)) {
|
| 198 | set_write = can_change_shared_pte_writable(vma, ptent);
|
| 199 | prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, nr_ptes,
|
| 200 | 0, set_write, tlb);
|
| 201 | return;
|
| 202 | }
|
| 203 |
|
| 204 | set_write = maybe_change_pte_writable(vma, ptent) &&
|
| 205 | (folio && folio_test_anon(folio));
|
| 206 | if (!set_write) {
|
| 207 | prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, nr_ptes,
|
| 208 | 0, set_write, tlb);
|
| 209 | return;
|
| 210 | }
|
| 211 | commit_anon_folio_batch(vma, folio, page, addr, ptep, oldpte, ptent, nr_ptes, tlb);
|
| 212 | }
|
| 213 |
|
| 214 | static long change_softleaf_pte(struct vm_area_struct *vma,
|
| 215 | unsigned long addr, pte_t *pte, pte_t oldpte, unsigned long cp_flags)
|
| 216 | {
|
| 217 | const bool uffd_prot = cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP);
|
| 218 | const bool uffd_prot_resolve = cp_flags &
|
| 219 | (MM_CP_UFFD_WP_RESOLVE | MM_CP_UFFD_RWP_RESOLVE);
|
| 220 | softleaf_t entry = softleaf_from_pte(oldpte);
|
| 221 | pte_t newpte;
|
| 222 |
|
| 223 | if (softleaf_is_migration_write(entry)) {
|
| 224 | const struct folio *folio = softleaf_to_folio(entry);
|
| 225 |
|
| 226 | |
| 227 | * A protection check is difficult so
|
| 228 | * just be safe and disable write
|
| 229 |
|
| 230 | if (folio_test_anon(folio))
|
| 231 | entry = make_readable_exclusive_migration_entry(swp_offset(entry));
|
| 232 | else
|
| 233 | entry = make_readable_migration_entry(swp_offset(entry));
|
| 234 | newpte = swp_entry_to_pte(entry);
|
| 235 | if (pte_swp_soft_dirty(oldpte))
|
| 236 | newpte = pte_swp_mksoft_dirty(newpte);
|
| 237 | } else if (softleaf_is_device_private_write(entry)) {
|
| 238 | |
| 239 | * We do not preserve soft-dirtiness. See
|
| 240 | * copy_nonpresent_pte() for explanation.
|
| 241 |
|
| 242 | entry = make_readable_device_private_entry(swp_offset(entry));
|
| 243 | newpte = swp_entry_to_pte(entry);
|
| 244 | if (pte_swp_uffd(oldpte))
|
| 245 | newpte = pte_swp_mkuffd(newpte);
|
| 246 | } else if (softleaf_is_marker(entry)) {
|
| 247 | |
| 248 | * Ignore error swap entries unconditionally,
|
| 249 | * because any access should sigbus/sigsegv
|
| 250 | * anyway.
|
| 251 |
|
| 252 | if (softleaf_is_poison_marker(entry) ||
|
| 253 | softleaf_is_guard_marker(entry))
|
| 254 | return 0;
|
| 255 | |
| 256 | * If this is uffd-wp pte marker and we'd like
|
| 257 | * to unprotect it, drop it; the next page
|
| 258 | * fault will trigger without uffd trapping.
|
| 259 |
|
| 260 | if (uffd_prot_resolve) {
|
| 261 | pte_clear(vma->vm_mm, addr, pte);
|
| 262 | return 1;
|
| 263 | }
|
| 264 | return 0;
|
| 265 | } else {
|
| 266 | newpte = oldpte;
|
| 267 | }
|
| 268 |
|
| 269 | if (uffd_prot)
|
| 270 | newpte = pte_swp_mkuffd(newpte);
|
| 271 | else if (uffd_prot_resolve)
|
| 272 | newpte = pte_swp_clear_uffd(newpte);
|
| 273 |
|
| 274 | if (!pte_same(oldpte, newpte)) {
|
| 275 | set_pte_at(vma->vm_mm, addr, pte, newpte);
|
| 276 | return 1;
|
| 277 | }
|
| 278 | return 0;
|
| 279 | }
|
| 280 |
|
| 281 | static __always_inline void change_present_ptes(struct mmu_gather *tlb,
|
| 282 | struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
|
| 283 | int nr_ptes, unsigned long end, pgprot_t newprot,
|
| 284 | struct folio *folio, struct page *page, unsigned long cp_flags)
|
| 285 | {
|
| 286 | const bool uffd_prot = cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP);
|
| 287 | const bool uffd_prot_resolve = cp_flags &
|
| 288 | (MM_CP_UFFD_WP_RESOLVE | MM_CP_UFFD_RWP_RESOLVE);
|
| 289 | pte_t ptent, oldpte;
|
| 290 |
|
| 291 | oldpte = modify_prot_start_ptes(vma, addr, ptep, nr_ptes);
|
| 292 | ptent = pte_modify(oldpte, newprot);
|
| 293 |
|
| 294 | if (uffd_prot)
|
| 295 | ptent = pte_mkuffd(ptent);
|
| 296 | else if (uffd_prot_resolve)
|
| 297 | ptent = pte_clear_uffd(ptent);
|
| 298 |
|
| 299 | |
| 300 | * The uffd bit on a VM_UFFD_RWP VMA carries PROT_NONE
|
| 301 | * semantics. If mprotect() or NUMA hinting changed the
|
| 302 | * base protection, restore PAGE_NONE so the PTE still
|
| 303 | * traps on any access. pte_modify() preserves
|
| 304 | * _PAGE_UFFD.
|
| 305 |
|
| 306 | if (userfaultfd_rwp(vma) && pte_uffd(ptent))
|
| 307 | ptent = pte_modify(ptent, PAGE_NONE);
|
| 308 |
|
| 309 | |
| 310 | * In some writable, shared mappings, we might want
|
| 311 | * to catch actual write access -- see
|
| 312 | * vma_wants_writenotify().
|
| 313 | *
|
| 314 | * In all writable, private mappings, we have to
|
| 315 | * properly handle COW.
|
| 316 | *
|
| 317 | * In both cases, we can sometimes still change PTEs
|
| 318 | * writable and avoid the write-fault handler, for
|
| 319 | * example, if a PTE is already dirty and no other
|
| 320 | * COW or special handling is required.
|
| 321 |
|
| 322 | if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
|
| 323 | !pte_write(ptent))
|
| 324 | set_write_prot_commit_flush_ptes(vma, folio, page,
|
| 325 | addr, ptep, oldpte, ptent, nr_ptes, tlb);
|
| 326 | else
|
| 327 | prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent,
|
| 328 | nr_ptes, 0, false, tlb);
|
| 329 | }
|
| 330 |
|
| 331 | static long change_pte_range(struct mmu_gather *tlb,
|
| 332 | struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr,
|
| 333 | unsigned long end, pgprot_t newprot, unsigned long cp_flags)
|
| 334 | {
|
| 335 | pte_t *pte, oldpte;
|
| 336 | spinlock_t *ptl;
|
| 337 | long pages = 0;
|
| 338 | bool is_private_single_threaded;
|
| 339 | bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
|
| 340 | bool uffd_rwp = cp_flags & MM_CP_UFFD_RWP;
|
| 341 | bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
|
| 342 | int nr_ptes;
|
| 343 |
|
| 344 | tlb_change_page_size(tlb, PAGE_SIZE);
|
| 345 | pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
|
| 346 | if (!pte)
|
| 347 | return -EAGAIN;
|
| 348 |
|
| 349 | if (prot_numa)
|
| 350 | is_private_single_threaded = vma_is_single_threaded_private(vma);
|
| 351 |
|
| 352 | flush_tlb_batched_pending(vma->vm_mm);
|
| 353 | lazy_mmu_mode_enable();
|
| 354 | do {
|
| 355 | nr_ptes = 1;
|
| 356 | oldpte = ptep_get(pte);
|
| 357 | if (pte_present(oldpte)) {
|
| 358 | const fpb_t flags = FPB_RESPECT_SOFT_DIRTY | FPB_RESPECT_WRITE;
|
| 359 | int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
|
| 360 | struct folio *folio = NULL;
|
| 361 | struct page *page;
|
| 362 |
|
| 363 |
|
| 364 | if (prot_numa && pte_protnone(oldpte))
|
| 365 | continue;
|
| 366 | |
| 367 | * RWP-protected PTEs carry _PAGE_UFFD as a marker on
|
| 368 | * top of PROT_NONE. Skip only entries already in that
|
| 369 | * exact state; plain PROT_NONE from mprotect() still needs
|
| 370 | * to be promoted so future faults can be distinguished.
|
| 371 |
|
| 372 | if (uffd_rwp && pte_protnone(oldpte) && pte_uffd(oldpte))
|
| 373 | continue;
|
| 374 |
|
| 375 | page = vm_normal_page(vma, addr, oldpte);
|
| 376 | if (page)
|
| 377 | folio = page_folio(page);
|
| 378 |
|
| 379 | |
| 380 | * Avoid trapping faults against the zero or KSM
|
| 381 | * pages. See similar comment in change_huge_pmd.
|
| 382 | * Skip this filter for uffd RWP which
|
| 383 | * must set protnone regardless of NUMA placement.
|
| 384 |
|
| 385 | if (prot_numa &&
|
| 386 | !folio_can_map_prot_numa(folio, vma,
|
| 387 | is_private_single_threaded)) {
|
| 388 |
|
| 389 |
|
| 390 | nr_ptes = mprotect_folio_pte_batch(folio,
|
| 391 | pte, oldpte, max_nr_ptes, 0);
|
| 392 | continue;
|
| 393 | }
|
| 394 |
|
| 395 | nr_ptes = mprotect_folio_pte_batch(folio, pte, oldpte, max_nr_ptes, flags);
|
| 396 |
|
| 397 | |
| 398 | * Optimize for the small-folio common case by
|
| 399 | * special-casing it here. Compiler constant propagation
|
| 400 | * plus copious amounts of __always_inline does wonders.
|
| 401 |
|
| 402 | if (likely(nr_ptes == 1)) {
|
| 403 | change_present_ptes(tlb, vma, addr, pte, 1,
|
| 404 | end, newprot, folio, page, cp_flags);
|
| 405 | } else {
|
| 406 | change_present_ptes(tlb, vma, addr, pte,
|
| 407 | nr_ptes, end, newprot, folio, page,
|
| 408 | cp_flags);
|
| 409 | }
|
| 410 |
|
| 411 | pages += nr_ptes;
|
| 412 | } else if (pte_none(oldpte)) {
|
| 413 | |
| 414 | * Nobody plays with any none ptes besides
|
| 415 | * userfaultfd when applying the protections.
|
| 416 |
|
| 417 | if (likely(!uffd_wp))
|
| 418 | continue;
|
| 419 |
|
| 420 | if (userfaultfd_wp_use_markers(vma)) {
|
| 421 | |
| 422 | * For file-backed mem, we need to be able to
|
| 423 | * wr-protect a none pte, because even if the
|
| 424 | * pte is none, the page/swap cache could
|
| 425 | * exist. Doing that by install a marker.
|
| 426 |
|
| 427 | set_pte_at(vma->vm_mm, addr, pte,
|
| 428 | make_pte_marker(PTE_MARKER_UFFD_WP));
|
| 429 | pages++;
|
| 430 | }
|
| 431 | } else {
|
| 432 | pages += change_softleaf_pte(vma, addr, pte, oldpte, cp_flags);
|
| 433 | }
|
| 434 | } while (pte += nr_ptes, addr += nr_ptes * PAGE_SIZE, addr != end);
|
| 435 | lazy_mmu_mode_disable();
|
| 436 | pte_unmap_unlock(pte - 1, ptl);
|
| 437 |
|
| 438 | return pages;
|
| 439 | }
|
| 440 |
|
| 441 | |
| 442 | * Return true if we want to split THPs into PTE mappings in change
|
| 443 | * protection procedure, false otherwise.
|
| 444 |
|
| 445 | static inline bool
|
| 446 | pgtable_split_needed(struct vm_area_struct *vma, unsigned long cp_flags)
|
| 447 | {
|
| 448 | |
| 449 | * pte markers only resides in pte level, if we need pte markers,
|
| 450 | * we need to split. For example, we cannot wr-protect a file thp
|
| 451 | * (e.g. 2M shmem) because file thp is handled differently when
|
| 452 | * split by erasing the pmd so far.
|
| 453 |
|
| 454 | return (cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP)) && !vma_is_anonymous(vma);
|
| 455 | }
|
| 456 |
|
| 457 | |
| 458 | * Return true if we want to populate pgtables in change protection
|
| 459 | * procedure, false otherwise
|
| 460 |
|
| 461 | static inline bool
|
| 462 | pgtable_populate_needed(struct vm_area_struct *vma, unsigned long cp_flags)
|
| 463 | {
|
| 464 |
|
| 465 | if (!(cp_flags & MM_CP_UFFD_WP))
|
| 466 | return false;
|
| 467 |
|
| 468 |
|
| 469 | return userfaultfd_wp_use_markers(vma);
|
| 470 | }
|
| 471 |
|
| 472 | |
| 473 | * Populate the pgtable underneath for whatever reason if requested.
|
| 474 | * When {pte|pmd|...}_alloc() failed we treat it the same way as pgtable
|
| 475 | * allocation failures during page faults by kicking OOM and returning
|
| 476 | * error.
|
| 477 |
|
| 478 | #define change_pmd_prepare(vma, pmd, cp_flags) \
|
| 479 | ({ \
|
| 480 | long err = 0; \
|
| 481 | if (unlikely(pgtable_populate_needed(vma, cp_flags))) { \
|
| 482 | if (pte_alloc(vma->vm_mm, pmd)) \
|
| 483 | err = -ENOMEM; \
|
| 484 | } \
|
| 485 | err; \
|
| 486 | })
|
| 487 |
|
| 488 | |
| 489 | * This is the general pud/p4d/pgd version of change_pmd_prepare(). We need to
|
| 490 | * have separate change_pmd_prepare() because pte_alloc() returns 0 on success,
|
| 491 | * while {pmd|pud|p4d}_alloc() returns the valid pointer on success.
|
| 492 |
|
| 493 | #define change_prepare(vma, high, low, addr, cp_flags) \
|
| 494 | ({ \
|
| 495 | long err = 0; \
|
| 496 | if (unlikely(pgtable_populate_needed(vma, cp_flags))) { \
|
| 497 | low##_t *p = low##_alloc(vma->vm_mm, high, addr); \
|
| 498 | if (p == NULL) \
|
| 499 | err = -ENOMEM; \
|
| 500 | } \
|
| 501 | err; \
|
| 502 | })
|
| 503 |
|
| 504 | static inline long change_pmd_range(struct mmu_gather *tlb,
|
| 505 | struct vm_area_struct *vma, pud_t *pud, unsigned long addr,
|
| 506 | unsigned long end, pgprot_t newprot, unsigned long cp_flags)
|
| 507 | {
|
| 508 | pmd_t *pmd;
|
| 509 | unsigned long next;
|
| 510 | long pages = 0;
|
| 511 | unsigned long nr_huge_updates = 0;
|
| 512 |
|
| 513 | pmd = pmd_offset(pud, addr);
|
| 514 | do {
|
| 515 | long ret;
|
| 516 | pmd_t _pmd;
|
| 517 | again:
|
| 518 | next = pmd_addr_end(addr, end);
|
| 519 |
|
| 520 | ret = change_pmd_prepare(vma, pmd, cp_flags);
|
| 521 | if (ret) {
|
| 522 | pages = ret;
|
| 523 | break;
|
| 524 | }
|
| 525 |
|
| 526 | if (pmd_none(*pmd))
|
| 527 | goto next;
|
| 528 |
|
| 529 | _pmd = pmdp_get_lockless(pmd);
|
| 530 | if (pmd_is_huge(_pmd)) {
|
| 531 | if ((next - addr != HPAGE_PMD_SIZE) ||
|
| 532 | pgtable_split_needed(vma, cp_flags)) {
|
| 533 | __split_huge_pmd(vma, pmd, addr, false);
|
| 534 | |
| 535 | * For file-backed, the pmd could have been
|
| 536 | * cleared; make sure pmd populated if
|
| 537 | * necessary, then fall-through to pte level.
|
| 538 |
|
| 539 | ret = change_pmd_prepare(vma, pmd, cp_flags);
|
| 540 | if (ret) {
|
| 541 | pages = ret;
|
| 542 | break;
|
| 543 | }
|
| 544 | } else {
|
| 545 | ret = change_huge_pmd(tlb, vma, pmd,
|
| 546 | addr, newprot, cp_flags);
|
| 547 | if (ret) {
|
| 548 | if (ret == HPAGE_PMD_NR) {
|
| 549 | pages += HPAGE_PMD_NR;
|
| 550 | nr_huge_updates++;
|
| 551 | }
|
| 552 |
|
| 553 |
|
| 554 | goto next;
|
| 555 | }
|
| 556 | }
|
| 557 |
|
| 558 | }
|
| 559 |
|
| 560 | ret = change_pte_range(tlb, vma, pmd, addr, next, newprot,
|
| 561 | cp_flags);
|
| 562 | if (ret < 0)
|
| 563 | goto again;
|
| 564 | pages += ret;
|
| 565 | next:
|
| 566 | cond_resched();
|
| 567 | } while (pmd++, addr = next, addr != end);
|
| 568 |
|
| 569 | if (nr_huge_updates)
|
| 570 | count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates);
|
| 571 | return pages;
|
| 572 | }
|
| 573 |
|
| 574 | static inline long change_pud_range(struct mmu_gather *tlb,
|
| 575 | struct vm_area_struct *vma, p4d_t *p4d, unsigned long addr,
|
| 576 | unsigned long end, pgprot_t newprot, unsigned long cp_flags)
|
| 577 | {
|
| 578 | struct mmu_notifier_range range;
|
| 579 | pud_t *pudp, pud;
|
| 580 | unsigned long next;
|
| 581 | long pages = 0, ret;
|
| 582 |
|
| 583 | range.start = 0;
|
| 584 |
|
| 585 | pudp = pud_offset(p4d, addr);
|
| 586 | do {
|
| 587 | again:
|
| 588 | next = pud_addr_end(addr, end);
|
| 589 | ret = change_prepare(vma, pudp, pmd, addr, cp_flags);
|
| 590 | if (ret) {
|
| 591 | pages = ret;
|
| 592 | break;
|
| 593 | }
|
| 594 |
|
| 595 | pud = pudp_get(pudp);
|
| 596 | if (pud_none(pud))
|
| 597 | continue;
|
| 598 |
|
| 599 | if (!range.start) {
|
| 600 | mmu_notifier_range_init(&range,
|
| 601 | MMU_NOTIFY_PROTECTION_VMA, 0,
|
| 602 | vma->vm_mm, addr, end);
|
| 603 | mmu_notifier_invalidate_range_start(&range);
|
| 604 | }
|
| 605 |
|
| 606 | if (pud_leaf(pud)) {
|
| 607 | if ((next - addr != PUD_SIZE) ||
|
| 608 | pgtable_split_needed(vma, cp_flags)) {
|
| 609 | __split_huge_pud(vma, pudp, addr);
|
| 610 | goto again;
|
| 611 | } else {
|
| 612 | ret = change_huge_pud(tlb, vma, pudp,
|
| 613 | addr, newprot, cp_flags);
|
| 614 | if (ret == 0)
|
| 615 | goto again;
|
| 616 |
|
| 617 | if (ret == HPAGE_PUD_NR)
|
| 618 | pages += HPAGE_PUD_NR;
|
| 619 | continue;
|
| 620 | }
|
| 621 | }
|
| 622 |
|
| 623 | pages += change_pmd_range(tlb, vma, pudp, addr, next, newprot,
|
| 624 | cp_flags);
|
| 625 | } while (pudp++, addr = next, addr != end);
|
| 626 |
|
| 627 | if (range.start)
|
| 628 | mmu_notifier_invalidate_range_end(&range);
|
| 629 |
|
| 630 | return pages;
|
| 631 | }
|
| 632 |
|
| 633 | static inline long change_p4d_range(struct mmu_gather *tlb,
|
| 634 | struct vm_area_struct *vma, pgd_t *pgd, unsigned long addr,
|
| 635 | unsigned long end, pgprot_t newprot, unsigned long cp_flags)
|
| 636 | {
|
| 637 | p4d_t *p4d;
|
| 638 | unsigned long next;
|
| 639 | long pages = 0, ret;
|
| 640 |
|
| 641 | p4d = p4d_offset(pgd, addr);
|
| 642 | do {
|
| 643 | next = p4d_addr_end(addr, end);
|
| 644 | ret = change_prepare(vma, p4d, pud, addr, cp_flags);
|
| 645 | if (ret)
|
| 646 | return ret;
|
| 647 | if (p4d_none_or_clear_bad(p4d))
|
| 648 | continue;
|
| 649 | pages += change_pud_range(tlb, vma, p4d, addr, next, newprot,
|
| 650 | cp_flags);
|
| 651 | } while (p4d++, addr = next, addr != end);
|
| 652 |
|
| 653 | return pages;
|
| 654 | }
|
| 655 |
|
| 656 | static long change_protection_range(struct mmu_gather *tlb,
|
| 657 | struct vm_area_struct *vma, unsigned long addr,
|
| 658 | unsigned long end, pgprot_t newprot, unsigned long cp_flags)
|
| 659 | {
|
| 660 | struct mm_struct *mm = vma->vm_mm;
|
| 661 | pgd_t *pgd;
|
| 662 | unsigned long next;
|
| 663 | long pages = 0, ret;
|
| 664 |
|
| 665 | BUG_ON(addr >= end);
|
| 666 | pgd = pgd_offset(mm, addr);
|
| 667 | tlb_start_vma(tlb, vma);
|
| 668 | do {
|
| 669 | next = pgd_addr_end(addr, end);
|
| 670 | ret = change_prepare(vma, pgd, p4d, addr, cp_flags);
|
| 671 | if (ret) {
|
| 672 | pages = ret;
|
| 673 | break;
|
| 674 | }
|
| 675 | if (pgd_none_or_clear_bad(pgd))
|
| 676 | continue;
|
| 677 | pages += change_p4d_range(tlb, vma, pgd, addr, next, newprot,
|
| 678 | cp_flags);
|
| 679 | } while (pgd++, addr = next, addr != end);
|
| 680 |
|
| 681 | tlb_end_vma(tlb, vma);
|
| 682 |
|
| 683 | return pages;
|
| 684 | }
|
| 685 |
|
| 686 | long change_protection(struct mmu_gather *tlb,
|
| 687 | struct vm_area_struct *vma, unsigned long start,
|
| 688 | unsigned long end, unsigned long cp_flags)
|
| 689 | {
|
| 690 | pgprot_t newprot = vma->vm_page_prot;
|
| 691 | long pages;
|
| 692 |
|
| 693 | |
| 694 | * MM_CP_UFFD_{WP,RWP} and _RESOLVE are mutually exclusive within one
|
| 695 | * change, and WP and RWP cannot mix. Miswired callers get a warn and
|
| 696 | * a no-op; userspace cannot reach this state.
|
| 697 |
|
| 698 | if (WARN_ON_ONCE((cp_flags & MM_CP_UFFD_WP_ALL) == MM_CP_UFFD_WP_ALL ||
|
| 699 | (cp_flags & MM_CP_UFFD_RWP_ALL) == MM_CP_UFFD_RWP_ALL ||
|
| 700 | ((cp_flags & MM_CP_UFFD_WP_ALL) &&
|
| 701 | (cp_flags & MM_CP_UFFD_RWP_ALL))))
|
| 702 | return 0;
|
| 703 |
|
| 704 | #ifdef CONFIG_NUMA_BALANCING
|
| 705 | |
| 706 | * Ordinary protection updates (mprotect, uffd-wp, softdirty tracking)
|
| 707 | * are expected to reflect their requirements via VMA flags such that
|
| 708 | * vma_set_page_prot() will adjust vma->vm_page_prot accordingly.
|
| 709 |
|
| 710 | if (cp_flags & MM_CP_PROT_NUMA)
|
| 711 | newprot = PAGE_NONE;
|
| 712 | #else
|
| 713 | WARN_ON_ONCE(cp_flags & MM_CP_PROT_NUMA);
|
| 714 | #endif
|
| 715 |
|
| 716 | if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE) &&
|
| 717 | (cp_flags & MM_CP_UFFD_RWP))
|
| 718 | newprot = PAGE_NONE;
|
| 719 |
|
| 720 | if (is_vm_hugetlb_page(vma))
|
| 721 | pages = hugetlb_change_protection(vma, start, end, newprot,
|
| 722 | cp_flags);
|
| 723 | else
|
| 724 | pages = change_protection_range(tlb, vma, start, end, newprot,
|
| 725 | cp_flags);
|
| 726 |
|
| 727 | return pages;
|
| 728 | }
|
| 729 |
|
| 730 | static int prot_none_pte_entry(pte_t *pte, unsigned long addr,
|
| 731 | unsigned long next, struct mm_walk *walk)
|
| 732 | {
|
| 733 | return pfn_modify_allowed(pte_pfn(ptep_get(pte)),
|
| 734 | *(pgprot_t *)(walk->private)) ?
|
| 735 | 0 : -EACCES;
|
| 736 | }
|
| 737 |
|
| 738 | #ifdef CONFIG_HUGETLB_PAGE
|
| 739 | static int prot_none_hugetlb_entry(pte_t *pte, unsigned long hmask,
|
| 740 | unsigned long addr, unsigned long next,
|
| 741 | struct mm_walk *walk)
|
| 742 | {
|
| 743 | const pte_t entry = huge_ptep_get(walk->mm, addr, pte);
|
| 744 |
|
| 745 | if (pfn_modify_allowed(pte_pfn(entry), *(pgprot_t *)(walk->private)))
|
| 746 | return 0;
|
| 747 | return -EACCES;
|
| 748 | }
|
| 749 | #else
|
| 750 | #define prot_none_hugetlb_entry NULL
|
| 751 | #endif
|
| 752 |
|
| 753 | static const struct mm_walk_ops prot_none_walk_ops = {
|
| 754 | .pte_entry = prot_none_pte_entry,
|
| 755 | .hugetlb_entry = prot_none_hugetlb_entry,
|
| 756 | .walk_lock = PGWALK_WRLOCK,
|
| 757 | };
|
| 758 |
|
| 759 | int
|
| 760 | mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb,
|
| 761 | struct vm_area_struct *vma, struct vm_area_struct **pprev,
|
| 762 | unsigned long start, unsigned long end, vm_flags_t newflags)
|
| 763 | {
|
| 764 | struct mm_struct *mm = vma->vm_mm;
|
| 765 | const vma_flags_t old_vma_flags = READ_ONCE(vma->flags);
|
| 766 | vma_flags_t new_vma_flags = legacy_to_vma_flags(newflags);
|
| 767 | long nrpages = (end - start) >> PAGE_SHIFT;
|
| 768 | unsigned int mm_cp_flags = 0;
|
| 769 | unsigned long charged = 0;
|
| 770 | int error;
|
| 771 |
|
| 772 | if (vma_is_sealed(vma))
|
| 773 | return -EPERM;
|
| 774 |
|
| 775 | if (vma_flags_same_pair(&old_vma_flags, &new_vma_flags)) {
|
| 776 | *pprev = vma;
|
| 777 | return 0;
|
| 778 | }
|
| 779 |
|
| 780 | |
| 781 | * Do PROT_NONE PFN permission checks here when we can still
|
| 782 | * bail out without undoing a lot of state. This is a rather
|
| 783 | * uncommon case, so doesn't need to be very optimized.
|
| 784 |
|
| 785 | if (arch_has_pfn_modify_check() &&
|
| 786 | vma_flags_test_any(&old_vma_flags, VMA_PFNMAP_BIT,
|
| 787 | VMA_MIXEDMAP_BIT) &&
|
| 788 | !vma_flags_test_any_mask(&new_vma_flags, VMA_ACCESS_FLAGS)) {
|
| 789 | pgprot_t new_pgprot = vm_get_page_prot(newflags);
|
| 790 |
|
| 791 | error = walk_page_range_vma(vma, start, end,
|
| 792 | &prot_none_walk_ops, &new_pgprot);
|
| 793 | if (error)
|
| 794 | return error;
|
| 795 | }
|
| 796 |
|
| 797 | |
| 798 | * If we make a private mapping writable we increase our commit;
|
| 799 | * but (without finer accounting) cannot reduce our commit if we
|
| 800 | * make it unwritable again except in the anonymous case where no
|
| 801 | * anon_vma has yet to be assigned.
|
| 802 | *
|
| 803 | * hugetlb mapping were accounted for even if read-only so there is
|
| 804 | * no need to account for them here.
|
| 805 |
|
| 806 | if (vma_flags_test(&new_vma_flags, VMA_WRITE_BIT)) {
|
| 807 |
|
| 808 | if (!may_expand_vm(mm, &new_vma_flags, nrpages) &&
|
| 809 | may_expand_vm(mm, &old_vma_flags, nrpages))
|
| 810 | return -ENOMEM;
|
| 811 | if (!vma_flags_test_any(&old_vma_flags,
|
| 812 | VMA_ACCOUNT_BIT, VMA_WRITE_BIT, VMA_HUGETLB_BIT,
|
| 813 | VMA_SHARED_BIT, VMA_NORESERVE_BIT)) {
|
| 814 | charged = nrpages;
|
| 815 | if (security_vm_enough_memory_mm(mm, charged))
|
| 816 | return -ENOMEM;
|
| 817 | vma_flags_set(&new_vma_flags, VMA_ACCOUNT_BIT);
|
| 818 | }
|
| 819 | } else if (vma_flags_test(&old_vma_flags, VMA_ACCOUNT_BIT) &&
|
| 820 | vma_is_anonymous(vma) && !vma->anon_vma) {
|
| 821 | vma_flags_clear(&new_vma_flags, VMA_ACCOUNT_BIT);
|
| 822 | }
|
| 823 |
|
| 824 | vma = vma_modify_flags(vmi, *pprev, vma, start, end, &new_vma_flags);
|
| 825 | if (IS_ERR(vma)) {
|
| 826 | error = PTR_ERR(vma);
|
| 827 | goto fail;
|
| 828 | }
|
| 829 |
|
| 830 | *pprev = vma;
|
| 831 |
|
| 832 | |
| 833 | * vm_flags and vm_page_prot are protected by the mmap_lock
|
| 834 | * held in write mode.
|
| 835 |
|
| 836 | vma_start_write(vma);
|
| 837 | vma_flags_reset_once(vma, &new_vma_flags);
|
| 838 | if (vma_wants_manual_pte_write_upgrade(vma))
|
| 839 | mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE;
|
| 840 | vma_set_page_prot(vma);
|
| 841 |
|
| 842 | change_protection(tlb, vma, start, end, mm_cp_flags);
|
| 843 |
|
| 844 | if (vma_flags_test(&old_vma_flags, VMA_ACCOUNT_BIT) &&
|
| 845 | !vma_flags_test(&new_vma_flags, VMA_ACCOUNT_BIT))
|
| 846 | vm_unacct_memory(nrpages);
|
| 847 |
|
| 848 | |
| 849 | * Private VMA_LOCKED_BIT VMA becoming writable: trigger COW to avoid
|
| 850 | * major fault on access.
|
| 851 |
|
| 852 | if (vma_flags_test(&new_vma_flags, VMA_WRITE_BIT) &&
|
| 853 | vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT) &&
|
| 854 | !vma_flags_test_any(&old_vma_flags, VMA_WRITE_BIT, VMA_SHARED_BIT))
|
| 855 | populate_vma_page_range(vma, start, end, NULL);
|
| 856 |
|
| 857 | vm_stat_account(mm, vma_flags_to_legacy(old_vma_flags), -nrpages);
|
| 858 | newflags = vma_flags_to_legacy(new_vma_flags);
|
| 859 | vm_stat_account(mm, newflags, nrpages);
|
| 860 | perf_event_mmap(vma);
|
| 861 | return 0;
|
| 862 |
|
| 863 | fail:
|
| 864 | vm_unacct_memory(charged);
|
| 865 | return error;
|
| 866 | }
|
| 867 |
|
| 868 | |
| 869 | * pkey==-1 when doing a legacy mprotect()
|
| 870 |
|
| 871 | static int do_mprotect_pkey(unsigned long start, size_t len,
|
| 872 | unsigned long prot, int pkey)
|
| 873 | {
|
| 874 | unsigned long nstart, end, tmp, reqprot;
|
| 875 | struct vm_area_struct *vma, *prev;
|
| 876 | int error;
|
| 877 | const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP);
|
| 878 | const bool rier = (current->personality & READ_IMPLIES_EXEC) &&
|
| 879 | (prot & PROT_READ);
|
| 880 | struct mmu_gather tlb;
|
| 881 | struct vma_iterator vmi;
|
| 882 |
|
| 883 | start = untagged_addr(start);
|
| 884 |
|
| 885 | prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP);
|
| 886 | if (grows == (PROT_GROWSDOWN|PROT_GROWSUP))
|
| 887 | return -EINVAL;
|
| 888 |
|
| 889 | if (start & ~PAGE_MASK)
|
| 890 | return -EINVAL;
|
| 891 | if (!len)
|
| 892 | return 0;
|
| 893 | len = PAGE_ALIGN(len);
|
| 894 | end = start + len;
|
| 895 | if (end <= start)
|
| 896 | return -ENOMEM;
|
| 897 | if (!arch_validate_prot(prot, start))
|
| 898 | return -EINVAL;
|
| 899 |
|
| 900 | reqprot = prot;
|
| 901 |
|
| 902 | if (mmap_write_lock_killable(current->mm))
|
| 903 | return -EINTR;
|
| 904 |
|
| 905 | |
| 906 | * If userspace did not allocate the pkey, do not let
|
| 907 | * them use it here.
|
| 908 |
|
| 909 | error = -EINVAL;
|
| 910 | if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey))
|
| 911 | goto out;
|
| 912 |
|
| 913 | vma_iter_init(&vmi, current->mm, start);
|
| 914 | vma = vma_find(&vmi, end);
|
| 915 | error = -ENOMEM;
|
| 916 | if (!vma)
|
| 917 | goto out;
|
| 918 |
|
| 919 | if (unlikely(grows & PROT_GROWSDOWN)) {
|
| 920 | if (vma->vm_start >= end)
|
| 921 | goto out;
|
| 922 | start = vma->vm_start;
|
| 923 | error = -EINVAL;
|
| 924 | if (!vma_test(vma, VMA_GROWSDOWN_BIT))
|
| 925 | goto out;
|
| 926 | } else {
|
| 927 | if (vma->vm_start > start)
|
| 928 | goto out;
|
| 929 | if (unlikely(grows & PROT_GROWSUP)) {
|
| 930 | end = vma->vm_end;
|
| 931 | error = -EINVAL;
|
| 932 | if (!vma_test_single_mask(vma, VMA_GROWSUP))
|
| 933 | goto out;
|
| 934 | }
|
| 935 | }
|
| 936 |
|
| 937 | prev = vma_prev(&vmi);
|
| 938 | if (start > vma->vm_start)
|
| 939 | prev = vma;
|
| 940 |
|
| 941 | tlb_gather_mmu(&tlb, current->mm);
|
| 942 | nstart = start;
|
| 943 | tmp = vma->vm_start;
|
| 944 | for_each_vma_range(vmi, vma, end) {
|
| 945 | vm_flags_t mask_off_old_flags;
|
| 946 | vma_flags_t new_vma_flags;
|
| 947 | vm_flags_t newflags;
|
| 948 | int new_vma_pkey;
|
| 949 |
|
| 950 | if (vma->vm_start != tmp) {
|
| 951 | error = -ENOMEM;
|
| 952 | break;
|
| 953 | }
|
| 954 |
|
| 955 |
|
| 956 | if (rier && vma_test(vma, VMA_MAYEXEC_BIT))
|
| 957 | prot |= PROT_EXEC;
|
| 958 |
|
| 959 | |
| 960 | * Each mprotect() call explicitly passes r/w/x permissions.
|
| 961 | * If a permission is not passed to mprotect(), it must be
|
| 962 | * cleared from the VMA.
|
| 963 |
|
| 964 | mask_off_old_flags = VM_ACCESS_FLAGS | VM_FLAGS_CLEAR;
|
| 965 |
|
| 966 | new_vma_pkey = arch_override_mprotect_pkey(vma, prot, pkey);
|
| 967 | newflags = calc_vm_prot_bits(prot, new_vma_pkey);
|
| 968 | newflags |= (vma->vm_flags & ~mask_off_old_flags);
|
| 969 | new_vma_flags = legacy_to_vma_flags(newflags);
|
| 970 |
|
| 971 |
|
| 972 | if ((newflags & ~(newflags >> 4)) & VM_ACCESS_FLAGS) {
|
| 973 | error = -EACCES;
|
| 974 | break;
|
| 975 | }
|
| 976 |
|
| 977 | if (map_deny_write_exec(&vma->flags, &new_vma_flags)) {
|
| 978 | error = -EACCES;
|
| 979 | break;
|
| 980 | }
|
| 981 |
|
| 982 |
|
| 983 | if (!arch_validate_flags(newflags)) {
|
| 984 | error = -EINVAL;
|
| 985 | break;
|
| 986 | }
|
| 987 |
|
| 988 | error = security_file_mprotect(vma, reqprot, prot);
|
| 989 | if (error)
|
| 990 | break;
|
| 991 |
|
| 992 | tmp = vma->vm_end;
|
| 993 | if (tmp > end)
|
| 994 | tmp = end;
|
| 995 |
|
| 996 | if (vma->vm_ops && vma->vm_ops->mprotect) {
|
| 997 | error = vma->vm_ops->mprotect(vma, nstart, tmp, newflags);
|
| 998 | if (error)
|
| 999 | break;
|
| 1000 | }
|
| 1001 |
|
| 1002 | error = mprotect_fixup(&vmi, &tlb, vma, &prev, nstart, tmp, newflags);
|
| 1003 | if (error)
|
| 1004 | break;
|
| 1005 |
|
| 1006 | tmp = vma_iter_end(&vmi);
|
| 1007 | nstart = tmp;
|
| 1008 | prot = reqprot;
|
| 1009 | }
|
| 1010 | tlb_finish_mmu(&tlb);
|
| 1011 |
|
| 1012 | if (!error && tmp < end)
|
| 1013 | error = -ENOMEM;
|
| 1014 |
|
| 1015 | out:
|
| 1016 | mmap_write_unlock(current->mm);
|
| 1017 | return error;
|
| 1018 | }
|
| 1019 |
|
| 1020 | SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len,
|
| 1021 | unsigned long, prot)
|
| 1022 | {
|
| 1023 | return do_mprotect_pkey(start, len, prot, -1);
|
| 1024 | }
|
| 1025 |
|
| 1026 | #ifdef CONFIG_ARCH_HAS_PKEYS
|
| 1027 |
|
| 1028 | SYSCALL_DEFINE4(pkey_mprotect, unsigned long, start, size_t, len,
|
| 1029 | unsigned long, prot, int, pkey)
|
| 1030 | {
|
| 1031 | return do_mprotect_pkey(start, len, prot, pkey);
|
| 1032 | }
|
| 1033 |
|
| 1034 | SYSCALL_DEFINE2(pkey_alloc, unsigned long, flags, unsigned long, init_val)
|
| 1035 | {
|
| 1036 | int pkey;
|
| 1037 | int ret;
|
| 1038 |
|
| 1039 |
|
| 1040 | if (flags)
|
| 1041 | return -EINVAL;
|
| 1042 |
|
| 1043 | if (init_val & ~PKEY_ACCESS_MASK)
|
| 1044 | return -EINVAL;
|
| 1045 |
|
| 1046 | mmap_write_lock(current->mm);
|
| 1047 | pkey = mm_pkey_alloc(current->mm);
|
| 1048 |
|
| 1049 | ret = -ENOSPC;
|
| 1050 | if (pkey == -1)
|
| 1051 | goto out;
|
| 1052 |
|
| 1053 | ret = arch_set_user_pkey_access(pkey, init_val);
|
| 1054 | if (ret) {
|
| 1055 | mm_pkey_free(current->mm, pkey);
|
| 1056 | goto out;
|
| 1057 | }
|
| 1058 | ret = pkey;
|
| 1059 | out:
|
| 1060 | mmap_write_unlock(current->mm);
|
| 1061 | return ret;
|
| 1062 | }
|
| 1063 |
|
| 1064 | SYSCALL_DEFINE1(pkey_free, int, pkey)
|
| 1065 | {
|
| 1066 | int ret;
|
| 1067 |
|
| 1068 | mmap_write_lock(current->mm);
|
| 1069 | ret = mm_pkey_free(current->mm, pkey);
|
| 1070 | mmap_write_unlock(current->mm);
|
| 1071 |
|
| 1072 | |
| 1073 | * We could provide warnings or errors if any VMA still
|
| 1074 | * has the pkey set here.
|
| 1075 |
|
| 1076 | return ret;
|
| 1077 | }
|
| 1078 |
|
| 1079 | #endif
|