projects/linux·blob master · refs

1750 lines · 42.0 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_X86_PGTABLE_H
3#define _ASM_X86_PGTABLE_H
4
5#include <linux/mem_encrypt.h>
6#include <asm/page.h>
7#include <asm/pgtable_types.h>
8
9/*
10 * Macro to mark a page protection value as UC-
11 */
12#define pgprot_noncached(prot) \
13 ((boot_cpu_data.x86 > 3) \
14 ? (__pgprot(pgprot_val(prot) | \
15 cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS))) \
16 : (prot))
17
18#ifndef __ASSEMBLER__
19#include <linux/spinlock.h>
20#include <asm/x86_init.h>
21#include <asm/pkru.h>
22#include <asm/fpu/api.h>
23#include <asm/coco.h>
24#include <asm-generic/pgtable_uffd.h>
25#include <linux/page_table_check.h>
26
27extern pgd_t early_top_pgt[PTRS_PER_PGD];
28bool __init __early_make_pgtable(unsigned long address, pmdval_t pmd);
29
30struct seq_file;
31void ptdump_walk_pgd_level(struct seq_file *m, struct mm_struct *mm);
32void ptdump_walk_pgd_level_debugfs(struct seq_file *m, struct mm_struct *mm,
33 bool user);
34bool ptdump_walk_pgd_level_checkwx(void);
35#define ptdump_check_wx ptdump_walk_pgd_level_checkwx
36void ptdump_walk_user_pgd_level_checkwx(void);
37
38/*
39 * Macros to add or remove encryption attribute
40 */
41#define pgprot_encrypted(prot) __pgprot(cc_mkenc(pgprot_val(prot)))
42#define pgprot_decrypted(prot) __pgprot(cc_mkdec(pgprot_val(prot)))
43
44#ifdef CONFIG_DEBUG_WX
45#define debug_checkwx_user() ptdump_walk_user_pgd_level_checkwx()
46#else
47#define debug_checkwx_user() do { } while (0)
48#endif
49
50extern spinlock_t pgd_lock;
51extern struct list_head pgd_list;
52
53struct mm_struct *pgd_page_get_mm(struct ptdesc *pt);
54
55extern pmdval_t early_pmd_flags;
56
57#ifdef CONFIG_PARAVIRT_XXL
58#include <asm/paravirt.h>
59#else /* !CONFIG_PARAVIRT_XXL */
60#define set_pte(ptep, pte) native_set_pte(ptep, pte)
61
62#define set_pte_atomic(ptep, pte) \
63 native_set_pte_atomic(ptep, pte)
64
65#define set_pmd(pmdp, pmd) native_set_pmd(pmdp, pmd)
66
67#ifndef __PAGETABLE_P4D_FOLDED
68#define set_pgd(pgdp, pgd) native_set_pgd(pgdp, pgd)
69#define pgd_clear(pgdp) \
70do { \
71 if (pgtable_l5_enabled()) \
72 native_pgd_clear(pgdp); \
73} while (0)
74#endif
75
76#ifndef set_p4d
77# define set_p4d(p4dp, p4d) native_set_p4d(p4dp, p4d)
78#endif
79
80#ifndef __PAGETABLE_PUD_FOLDED
81#define p4d_clear(p4d) native_p4d_clear(p4d)
82#endif
83
84#ifndef set_pud
85# define set_pud(pudp, pud) native_set_pud(pudp, pud)
86#endif
87
88#ifndef __PAGETABLE_PUD_FOLDED
89#define pud_clear(pud) native_pud_clear(pud)
90#endif
91
92#define pte_clear(mm, addr, ptep) native_pte_clear(mm, addr, ptep)
93#define pmd_clear(pmd) native_pmd_clear(pmd)
94
95#define pgd_val(x) native_pgd_val(x)
96#define __pgd(x) native_make_pgd(x)
97
98#ifndef __PAGETABLE_P4D_FOLDED
99#define p4d_val(x) native_p4d_val(x)
100#define __p4d(x) native_make_p4d(x)
101#endif
102
103#ifndef __PAGETABLE_PUD_FOLDED
104#define pud_val(x) native_pud_val(x)
105#define __pud(x) native_make_pud(x)
106#endif
107
108#ifndef __PAGETABLE_PMD_FOLDED
109#define pmd_val(x) native_pmd_val(x)
110#define __pmd(x) native_make_pmd(x)
111#endif
112
113#define pte_val(x) native_pte_val(x)
114#define __pte(x) native_make_pte(x)
115
116#define arch_end_context_switch(prev) do {} while(0)
117static inline void arch_flush_lazy_mmu_mode(void) {}
118#endif /* CONFIG_PARAVIRT_XXL */
119
120static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set)
121{
122 pmdval_t v = native_pmd_val(pmd);
123
124 return native_make_pmd(v | set);
125}
126
127static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear)
128{
129 pmdval_t v = native_pmd_val(pmd);
130
131 return native_make_pmd(v & ~clear);
132}
133
134static inline pud_t pud_set_flags(pud_t pud, pudval_t set)
135{
136 pudval_t v = native_pud_val(pud);
137
138 return native_make_pud(v | set);
139}
140
141static inline pud_t pud_clear_flags(pud_t pud, pudval_t clear)
142{
143 pudval_t v = native_pud_val(pud);
144
145 return native_make_pud(v & ~clear);
146}
147
148/*
149 * The following only work if pte_present() is true.
150 * Undefined behaviour if not..
151 */
152static inline bool pte_dirty(pte_t pte)
153{
154 return pte_flags(pte) & _PAGE_DIRTY_BITS;
155}
156
157static inline bool pte_shstk(pte_t pte)
158{
159 return cpu_feature_enabled(X86_FEATURE_SHSTK) &&
160 (pte_flags(pte) & (_PAGE_RW | _PAGE_DIRTY)) == _PAGE_DIRTY;
161}
162
163static inline int pte_young(pte_t pte)
164{
165 return pte_flags(pte) & _PAGE_ACCESSED;
166}
167
168static inline bool pte_decrypted(pte_t pte)
169{
170 return cc_mkdec(pte_val(pte)) == pte_val(pte);
171}
172
173#define pmd_dirty pmd_dirty
174static inline bool pmd_dirty(pmd_t pmd)
175{
176 return pmd_flags(pmd) & _PAGE_DIRTY_BITS;
177}
178
179static inline bool pmd_shstk(pmd_t pmd)
180{
181 return cpu_feature_enabled(X86_FEATURE_SHSTK) &&
182 (pmd_flags(pmd) & (_PAGE_RW | _PAGE_DIRTY | _PAGE_PSE)) ==
183 (_PAGE_DIRTY | _PAGE_PSE);
184}
185
186#define pmd_young pmd_young
187static inline int pmd_young(pmd_t pmd)
188{
189 return pmd_flags(pmd) & _PAGE_ACCESSED;
190}
191
192static inline bool pud_dirty(pud_t pud)
193{
194 return pud_flags(pud) & _PAGE_DIRTY_BITS;
195}
196
197static inline int pud_young(pud_t pud)
198{
199 return pud_flags(pud) & _PAGE_ACCESSED;
200}
201
202static inline bool pud_shstk(pud_t pud)
203{
204 return cpu_feature_enabled(X86_FEATURE_SHSTK) &&
205 (pud_flags(pud) & (_PAGE_RW | _PAGE_DIRTY | _PAGE_PSE)) ==
206 (_PAGE_DIRTY | _PAGE_PSE);
207}
208
209static inline int pte_write(pte_t pte)
210{
211 /*
212 * Shadow stack pages are logically writable, but do not have
213 * _PAGE_RW. Check for them separately from _PAGE_RW itself.
214 */
215 return (pte_flags(pte) & _PAGE_RW) || pte_shstk(pte);
216}
217
218#define pmd_write pmd_write
219static inline int pmd_write(pmd_t pmd)
220{
221 /*
222 * Shadow stack pages are logically writable, but do not have
223 * _PAGE_RW. Check for them separately from _PAGE_RW itself.
224 */
225 return (pmd_flags(pmd) & _PAGE_RW) || pmd_shstk(pmd);
226}
227
228#define pud_write pud_write
229static inline int pud_write(pud_t pud)
230{
231 return pud_flags(pud) & _PAGE_RW;
232}
233
234static inline int pte_huge(pte_t pte)
235{
236 return pte_flags(pte) & _PAGE_PSE;
237}
238
239static inline int pte_global(pte_t pte)
240{
241 return pte_flags(pte) & _PAGE_GLOBAL;
242}
243
244static inline int pte_exec(pte_t pte)
245{
246 return !(pte_flags(pte) & _PAGE_NX);
247}
248
249static inline int pte_special(pte_t pte)
250{
251 return pte_flags(pte) & _PAGE_SPECIAL;
252}
253
254/* Entries that were set to PROT_NONE are inverted */
255
256static inline u64 protnone_mask(u64 val);
257
258#define PFN_PTE_SHIFT PAGE_SHIFT
259
260static inline unsigned long pte_pfn(pte_t pte)
261{
262 phys_addr_t pfn = pte_val(pte);
263 pfn ^= protnone_mask(pfn);
264 return (pfn & PTE_PFN_MASK) >> PAGE_SHIFT;
265}
266
267static inline unsigned long pmd_pfn(pmd_t pmd)
268{
269 phys_addr_t pfn = pmd_val(pmd);
270 pfn ^= protnone_mask(pfn);
271 return (pfn & pmd_pfn_mask(pmd)) >> PAGE_SHIFT;
272}
273
274#define pud_pfn pud_pfn
275static inline unsigned long pud_pfn(pud_t pud)
276{
277 phys_addr_t pfn = pud_val(pud);
278 pfn ^= protnone_mask(pfn);
279 return (pfn & pud_pfn_mask(pud)) >> PAGE_SHIFT;
280}
281
282static inline unsigned long p4d_pfn(p4d_t p4d)
283{
284 return (p4d_val(p4d) & p4d_pfn_mask(p4d)) >> PAGE_SHIFT;
285}
286
287static inline unsigned long pgd_pfn(pgd_t pgd)
288{
289 return (pgd_val(pgd) & PTE_PFN_MASK) >> PAGE_SHIFT;
290}
291
292#define pte_page(pte) pfn_to_page(pte_pfn(pte))
293
294#define pmd_leaf pmd_leaf
295static inline bool pmd_leaf(pmd_t pte)
296{
297 return pmd_flags(pte) & _PAGE_PSE;
298}
299
300#ifdef CONFIG_TRANSPARENT_HUGEPAGE
301static inline int pmd_trans_huge(pmd_t pmd)
302{
303 return (pmd_val(pmd) & _PAGE_PSE) == _PAGE_PSE;
304}
305
306#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
307static inline int pud_trans_huge(pud_t pud)
308{
309 return (pud_val(pud) & _PAGE_PSE) == _PAGE_PSE;
310}
311#endif
312
313#define has_transparent_hugepage has_transparent_hugepage
314static inline int has_transparent_hugepage(void)
315{
316 return boot_cpu_has(X86_FEATURE_PSE);
317}
318
319#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP
320static inline bool pmd_special(pmd_t pmd)
321{
322 return pmd_flags(pmd) & _PAGE_SPECIAL;
323}
324
325static inline pmd_t pmd_mkspecial(pmd_t pmd)
326{
327 return pmd_set_flags(pmd, _PAGE_SPECIAL);
328}
329#endif /* CONFIG_ARCH_SUPPORTS_PMD_PFNMAP */
330
331#ifdef CONFIG_ARCH_SUPPORTS_PUD_PFNMAP
332static inline bool pud_special(pud_t pud)
333{
334 return pud_flags(pud) & _PAGE_SPECIAL;
335}
336
337static inline pud_t pud_mkspecial(pud_t pud)
338{
339 return pud_set_flags(pud, _PAGE_SPECIAL);
340}
341#endif /* CONFIG_ARCH_SUPPORTS_PUD_PFNMAP */
342#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
343
344static inline pte_t pte_set_flags(pte_t pte, pteval_t set)
345{
346 pteval_t v = native_pte_val(pte);
347
348 return native_make_pte(v | set);
349}
350
351static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear)
352{
353 pteval_t v = native_pte_val(pte);
354
355 return native_make_pte(v & ~clear);
356}
357
358/*
359 * Write protection operations can result in Dirty=1,Write=0 PTEs. But in the
360 * case of X86_FEATURE_USER_SHSTK, these PTEs denote shadow stack memory. So
361 * when creating dirty, write-protected memory, a software bit is used:
362 * _PAGE_BIT_SAVED_DIRTY. The following functions take a PTE and transition the
363 * Dirty bit to SavedDirty, and vice-vesra.
364 *
365 * This shifting is only done if needed. In the case of shifting
366 * Dirty->SavedDirty, the condition is if the PTE is Write=0. In the case of
367 * shifting SavedDirty->Dirty, the condition is Write=1.
368 */
369static inline pgprotval_t mksaveddirty_shift(pgprotval_t v)
370{
371 pgprotval_t cond = (~v >> _PAGE_BIT_RW) & 1;
372
373 v |= ((v >> _PAGE_BIT_DIRTY) & cond) << _PAGE_BIT_SAVED_DIRTY;
374 v &= ~(cond << _PAGE_BIT_DIRTY);
375
376 return v;
377}
378
379static inline pgprotval_t clear_saveddirty_shift(pgprotval_t v)
380{
381 pgprotval_t cond = (v >> _PAGE_BIT_RW) & 1;
382
383 v |= ((v >> _PAGE_BIT_SAVED_DIRTY) & cond) << _PAGE_BIT_DIRTY;
384 v &= ~(cond << _PAGE_BIT_SAVED_DIRTY);
385
386 return v;
387}
388
389static inline pte_t pte_mksaveddirty(pte_t pte)
390{
391 pteval_t v = native_pte_val(pte);
392
393 v = mksaveddirty_shift(v);
394 return native_make_pte(v);
395}
396
397static inline pte_t pte_clear_saveddirty(pte_t pte)
398{
399 pteval_t v = native_pte_val(pte);
400
401 v = clear_saveddirty_shift(v);
402 return native_make_pte(v);
403}
404
405static inline pte_t pte_wrprotect(pte_t pte)
406{
407 pte = pte_clear_flags(pte, _PAGE_RW);
408
409 /*
410 * Blindly clearing _PAGE_RW might accidentally create
411 * a shadow stack PTE (Write=0,Dirty=1). Move the hardware
412 * dirty value to the software bit, if present.
413 */
414 return pte_mksaveddirty(pte);
415}
416
417#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
418static inline int pte_uffd(pte_t pte)
419{
420 return pte_flags(pte) & _PAGE_UFFD;
421}
422
423static inline pte_t pte_mkuffd(pte_t pte)
424{
425 return pte_wrprotect(pte_set_flags(pte, _PAGE_UFFD));
426}
427
428static inline pte_t pte_clear_uffd(pte_t pte)
429{
430 return pte_clear_flags(pte, _PAGE_UFFD);
431}
432#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
433
434static inline pte_t pte_mkclean(pte_t pte)
435{
436 return pte_clear_flags(pte, _PAGE_DIRTY_BITS);
437}
438
439static inline pte_t pte_mkold(pte_t pte)
440{
441 return pte_clear_flags(pte, _PAGE_ACCESSED);
442}
443
444static inline pte_t pte_mkexec(pte_t pte)
445{
446 return pte_clear_flags(pte, _PAGE_NX);
447}
448
449static inline pte_t pte_mkdirty(pte_t pte)
450{
451 pte = pte_set_flags(pte, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
452
453 return pte_mksaveddirty(pte);
454}
455
456static inline pte_t pte_mkwrite_shstk(pte_t pte)
457{
458 pte = pte_clear_flags(pte, _PAGE_RW);
459
460 return pte_set_flags(pte, _PAGE_DIRTY);
461}
462
463static inline pte_t pte_mkyoung(pte_t pte)
464{
465 return pte_set_flags(pte, _PAGE_ACCESSED);
466}
467
468static inline pte_t pte_mkwrite_novma(pte_t pte)
469{
470 return pte_set_flags(pte, _PAGE_RW);
471}
472
473struct vm_area_struct;
474pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma);
475#define pte_mkwrite pte_mkwrite
476
477static inline pte_t pte_mkhuge(pte_t pte)
478{
479 return pte_set_flags(pte, _PAGE_PSE);
480}
481
482static inline pte_t pte_clrhuge(pte_t pte)
483{
484 return pte_clear_flags(pte, _PAGE_PSE);
485}
486
487static inline pte_t pte_mkglobal(pte_t pte)
488{
489 return pte_set_flags(pte, _PAGE_GLOBAL);
490}
491
492static inline pte_t pte_clrglobal(pte_t pte)
493{
494 return pte_clear_flags(pte, _PAGE_GLOBAL);
495}
496
497static inline pte_t pte_mkspecial(pte_t pte)
498{
499 return pte_set_flags(pte, _PAGE_SPECIAL);
500}
501
502/* See comments above mksaveddirty_shift() */
503static inline pmd_t pmd_mksaveddirty(pmd_t pmd)
504{
505 pmdval_t v = native_pmd_val(pmd);
506
507 v = mksaveddirty_shift(v);
508 return native_make_pmd(v);
509}
510
511/* See comments above mksaveddirty_shift() */
512static inline pmd_t pmd_clear_saveddirty(pmd_t pmd)
513{
514 pmdval_t v = native_pmd_val(pmd);
515
516 v = clear_saveddirty_shift(v);
517 return native_make_pmd(v);
518}
519
520static inline pmd_t pmd_wrprotect(pmd_t pmd)
521{
522 pmd = pmd_clear_flags(pmd, _PAGE_RW);
523
524 /*
525 * Blindly clearing _PAGE_RW might accidentally create
526 * a shadow stack PMD (RW=0, Dirty=1). Move the hardware
527 * dirty value to the software bit.
528 */
529 return pmd_mksaveddirty(pmd);
530}
531
532#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
533static inline int pmd_uffd(pmd_t pmd)
534{
535 return pmd_flags(pmd) & _PAGE_UFFD;
536}
537
538static inline pmd_t pmd_mkuffd(pmd_t pmd)
539{
540 return pmd_wrprotect(pmd_set_flags(pmd, _PAGE_UFFD));
541}
542
543static inline pmd_t pmd_clear_uffd(pmd_t pmd)
544{
545 return pmd_clear_flags(pmd, _PAGE_UFFD);
546}
547#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
548
549static inline pmd_t pmd_mkold(pmd_t pmd)
550{
551 return pmd_clear_flags(pmd, _PAGE_ACCESSED);
552}
553
554static inline pmd_t pmd_mkclean(pmd_t pmd)
555{
556 return pmd_clear_flags(pmd, _PAGE_DIRTY_BITS);
557}
558
559static inline pmd_t pmd_mkdirty(pmd_t pmd)
560{
561 pmd = pmd_set_flags(pmd, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
562
563 return pmd_mksaveddirty(pmd);
564}
565
566static inline pmd_t pmd_mkwrite_shstk(pmd_t pmd)
567{
568 pmd = pmd_clear_flags(pmd, _PAGE_RW);
569
570 return pmd_set_flags(pmd, _PAGE_DIRTY);
571}
572
573static inline pmd_t pmd_mkhuge(pmd_t pmd)
574{
575 return pmd_set_flags(pmd, _PAGE_PSE);
576}
577
578static inline pmd_t pmd_mkyoung(pmd_t pmd)
579{
580 return pmd_set_flags(pmd, _PAGE_ACCESSED);
581}
582
583static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)
584{
585 return pmd_set_flags(pmd, _PAGE_RW);
586}
587
588pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma);
589#define pmd_mkwrite pmd_mkwrite
590
591/* See comments above mksaveddirty_shift() */
592static inline pud_t pud_mksaveddirty(pud_t pud)
593{
594 pudval_t v = native_pud_val(pud);
595
596 v = mksaveddirty_shift(v);
597 return native_make_pud(v);
598}
599
600/* See comments above mksaveddirty_shift() */
601static inline pud_t pud_clear_saveddirty(pud_t pud)
602{
603 pudval_t v = native_pud_val(pud);
604
605 v = clear_saveddirty_shift(v);
606 return native_make_pud(v);
607}
608
609static inline pud_t pud_mkold(pud_t pud)
610{
611 return pud_clear_flags(pud, _PAGE_ACCESSED);
612}
613
614static inline pud_t pud_mkclean(pud_t pud)
615{
616 return pud_clear_flags(pud, _PAGE_DIRTY_BITS);
617}
618
619static inline pud_t pud_wrprotect(pud_t pud)
620{
621 pud = pud_clear_flags(pud, _PAGE_RW);
622
623 /*
624 * Blindly clearing _PAGE_RW might accidentally create
625 * a shadow stack PUD (RW=0, Dirty=1). Move the hardware
626 * dirty value to the software bit.
627 */
628 return pud_mksaveddirty(pud);
629}
630
631static inline pud_t pud_mkdirty(pud_t pud)
632{
633 pud = pud_set_flags(pud, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
634
635 return pud_mksaveddirty(pud);
636}
637
638static inline pud_t pud_mkhuge(pud_t pud)
639{
640 return pud_set_flags(pud, _PAGE_PSE);
641}
642
643static inline pud_t pud_mkyoung(pud_t pud)
644{
645 return pud_set_flags(pud, _PAGE_ACCESSED);
646}
647
648static inline pud_t pud_mkwrite(pud_t pud)
649{
650 pud = pud_set_flags(pud, _PAGE_RW);
651
652 return pud_clear_saveddirty(pud);
653}
654
655#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
656static inline int pte_soft_dirty(pte_t pte)
657{
658 return pte_flags(pte) & _PAGE_SOFT_DIRTY;
659}
660
661static inline int pmd_soft_dirty(pmd_t pmd)
662{
663 return pmd_flags(pmd) & _PAGE_SOFT_DIRTY;
664}
665
666static inline int pud_soft_dirty(pud_t pud)
667{
668 return pud_flags(pud) & _PAGE_SOFT_DIRTY;
669}
670
671static inline pte_t pte_mksoft_dirty(pte_t pte)
672{
673 return pte_set_flags(pte, _PAGE_SOFT_DIRTY);
674}
675
676static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
677{
678 return pmd_set_flags(pmd, _PAGE_SOFT_DIRTY);
679}
680
681static inline pud_t pud_mksoft_dirty(pud_t pud)
682{
683 return pud_set_flags(pud, _PAGE_SOFT_DIRTY);
684}
685
686static inline pte_t pte_clear_soft_dirty(pte_t pte)
687{
688 return pte_clear_flags(pte, _PAGE_SOFT_DIRTY);
689}
690
691static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
692{
693 return pmd_clear_flags(pmd, _PAGE_SOFT_DIRTY);
694}
695
696static inline pud_t pud_clear_soft_dirty(pud_t pud)
697{
698 return pud_clear_flags(pud, _PAGE_SOFT_DIRTY);
699}
700
701#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */
702
703/*
704 * Mask out unsupported bits in a present pgprot. Non-present pgprots
705 * can use those bits for other purposes, so leave them be.
706 */
707static inline pgprotval_t massage_pgprot(pgprot_t pgprot)
708{
709 pgprotval_t protval = pgprot_val(pgprot);
710
711 if (protval & _PAGE_PRESENT)
712 protval &= __supported_pte_mask;
713
714 return protval;
715}
716
717static inline pgprotval_t check_pgprot(pgprot_t pgprot)
718{
719 pgprotval_t massaged_val = massage_pgprot(pgprot);
720
721 /* mmdebug.h can not be included here because of dependencies */
722#ifdef CONFIG_DEBUG_VM
723 WARN_ONCE(pgprot_val(pgprot) != massaged_val,
724 "attempted to set unsupported pgprot: %016llx "
725 "bits: %016llx supported: %016llx\n",
726 (u64)pgprot_val(pgprot),
727 (u64)pgprot_val(pgprot) ^ massaged_val,
728 (u64)__supported_pte_mask);
729#endif
730
731 return massaged_val;
732}
733
734static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot)
735{
736 phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT;
737 /* This bit combination is used to mark shadow stacks */
738 WARN_ON_ONCE((pgprot_val(pgprot) & (_PAGE_DIRTY | _PAGE_RW)) ==
739 _PAGE_DIRTY);
740 pfn ^= protnone_mask(pgprot_val(pgprot));
741 pfn &= PTE_PFN_MASK;
742 return __pte(pfn | check_pgprot(pgprot));
743}
744
745static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
746{
747 phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT;
748 pfn ^= protnone_mask(pgprot_val(pgprot));
749 pfn &= PHYSICAL_PMD_PAGE_MASK;
750 return __pmd(pfn | check_pgprot(pgprot));
751}
752
753static inline pud_t pfn_pud(unsigned long page_nr, pgprot_t pgprot)
754{
755 phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT;
756 pfn ^= protnone_mask(pgprot_val(pgprot));
757 pfn &= PHYSICAL_PUD_PAGE_MASK;
758 return __pud(pfn | check_pgprot(pgprot));
759}
760
761static inline pmd_t pmd_mkinvalid(pmd_t pmd)
762{
763 return pfn_pmd(pmd_pfn(pmd),
764 __pgprot(pmd_flags(pmd) & ~(_PAGE_PRESENT|_PAGE_PROTNONE)));
765}
766
767static inline pud_t pud_mkinvalid(pud_t pud)
768{
769 return pfn_pud(pud_pfn(pud),
770 __pgprot(pud_flags(pud) & ~(_PAGE_PRESENT|_PAGE_PROTNONE)));
771}
772
773static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask);
774
775static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
776{
777 pteval_t val = pte_val(pte), oldval = val;
778 pte_t pte_result;
779
780 /*
781 * Chop off the NX bit (if present), and add the NX portion of
782 * the newprot (if present):
783 */
784 val &= _PAGE_CHG_MASK;
785 val |= check_pgprot(newprot) & ~_PAGE_CHG_MASK;
786 val = flip_protnone_guard(oldval, val, PTE_PFN_MASK);
787
788 pte_result = __pte(val);
789
790 /*
791 * To avoid creating Write=0,Dirty=1 PTEs, pte_modify() needs to avoid:
792 * 1. Marking Write=0 PTEs Dirty=1
793 * 2. Marking Dirty=1 PTEs Write=0
794 *
795 * The first case cannot happen because the _PAGE_CHG_MASK will filter
796 * out any Dirty bit passed in newprot. Handle the second case by
797 * going through the mksaveddirty exercise. Only do this if the old
798 * value was Write=1 to avoid doing this on Shadow Stack PTEs.
799 */
800 if (oldval & _PAGE_RW)
801 pte_result = pte_mksaveddirty(pte_result);
802 else
803 pte_result = pte_clear_saveddirty(pte_result);
804
805 return pte_result;
806}
807
808static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
809{
810 pmdval_t val = pmd_val(pmd), oldval = val;
811 pmd_t pmd_result;
812
813 val &= _HPAGE_CHG_MASK;
814 val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK;
815 val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK);
816
817 pmd_result = __pmd(val);
818
819 /*
820 * Avoid creating shadow stack PMD by accident. See comment in
821 * pte_modify().
822 */
823 if (oldval & _PAGE_RW)
824 pmd_result = pmd_mksaveddirty(pmd_result);
825 else
826 pmd_result = pmd_clear_saveddirty(pmd_result);
827
828 return pmd_result;
829}
830
831static inline pud_t pud_modify(pud_t pud, pgprot_t newprot)
832{
833 pudval_t val = pud_val(pud), oldval = val;
834 pud_t pud_result;
835
836 val &= _HPAGE_CHG_MASK;
837 val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK;
838 val = flip_protnone_guard(oldval, val, PHYSICAL_PUD_PAGE_MASK);
839
840 pud_result = __pud(val);
841
842 /*
843 * Avoid creating shadow stack PUD by accident. See comment in
844 * pte_modify().
845 */
846 if (oldval & _PAGE_RW)
847 pud_result = pud_mksaveddirty(pud_result);
848 else
849 pud_result = pud_clear_saveddirty(pud_result);
850
851 return pud_result;
852}
853
854/*
855 * mprotect needs to preserve PAT and encryption bits when updating
856 * vm_page_prot
857 */
858#define pgprot_modify pgprot_modify
859static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
860{
861 pgprotval_t preservebits = pgprot_val(oldprot) & _PAGE_CHG_MASK;
862 pgprotval_t addbits = pgprot_val(newprot) & ~_PAGE_CHG_MASK;
863 return __pgprot(preservebits | addbits);
864}
865
866#define pte_pgprot(x) __pgprot(pte_flags(x))
867#define pmd_pgprot(x) __pgprot(pmd_flags(x))
868#define pud_pgprot(x) __pgprot(pud_flags(x))
869#define p4d_pgprot(x) __pgprot(p4d_flags(x))
870
871#define canon_pgprot(p) __pgprot(massage_pgprot(p))
872
873static inline int is_new_memtype_allowed(u64 paddr, unsigned long size,
874 enum page_cache_mode pcm,
875 enum page_cache_mode new_pcm)
876{
877 /*
878 * PAT type is always WB for untracked ranges, so no need to check.
879 */
880 if (x86_platform.is_untracked_pat_range(paddr, paddr + size))
881 return 1;
882
883 /*
884 * Certain new memtypes are not allowed with certain
885 * requested memtype:
886 * - request is uncached, return cannot be write-back
887 * - request is write-combine, return cannot be write-back
888 * - request is write-through, return cannot be write-back
889 * - request is write-through, return cannot be write-combine
890 */
891 if ((pcm == _PAGE_CACHE_MODE_UC_MINUS &&
892 new_pcm == _PAGE_CACHE_MODE_WB) ||
893 (pcm == _PAGE_CACHE_MODE_WC &&
894 new_pcm == _PAGE_CACHE_MODE_WB) ||
895 (pcm == _PAGE_CACHE_MODE_WT &&
896 new_pcm == _PAGE_CACHE_MODE_WB) ||
897 (pcm == _PAGE_CACHE_MODE_WT &&
898 new_pcm == _PAGE_CACHE_MODE_WC)) {
899 return 0;
900 }
901
902 return 1;
903}
904
905pmd_t *populate_extra_pmd(unsigned long vaddr);
906pte_t *populate_extra_pte(unsigned long vaddr);
907
908#ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION
909pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd);
910
911/*
912 * Take a PGD location (pgdp) and a pgd value that needs to be set there.
913 * Populates the user and returns the resulting PGD that must be set in
914 * the kernel copy of the page tables.
915 */
916static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd)
917{
918 if (!cpu_feature_enabled(X86_FEATURE_PTI))
919 return pgd;
920 return __pti_set_user_pgtbl(pgdp, pgd);
921}
922#else /* CONFIG_MITIGATION_PAGE_TABLE_ISOLATION */
923static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd)
924{
925 return pgd;
926}
927#endif /* CONFIG_MITIGATION_PAGE_TABLE_ISOLATION */
928
929#endif /* __ASSEMBLER__ */
930
931
932#ifdef CONFIG_X86_32
933# include <asm/pgtable_32.h>
934#else
935# include <asm/pgtable_64.h>
936#endif
937
938#ifndef __ASSEMBLER__
939#include <linux/mm_types.h>
940#include <linux/mmdebug.h>
941#include <linux/log2.h>
942#include <asm/fixmap.h>
943
944static inline int pte_none(pte_t pte)
945{
946 return !(pte.pte & ~(_PAGE_KNL_ERRATUM_MASK));
947}
948
949#define __HAVE_ARCH_PTE_SAME
950static inline int pte_same(pte_t a, pte_t b)
951{
952 return a.pte == b.pte;
953}
954
955static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr)
956{
957 if (__pte_needs_invert(pte_val(pte)))
958 return __pte(pte_val(pte) - (nr << PFN_PTE_SHIFT));
959 return __pte(pte_val(pte) + (nr << PFN_PTE_SHIFT));
960}
961#define pte_advance_pfn pte_advance_pfn
962
963static inline int pte_present(pte_t a)
964{
965 return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE);
966}
967
968#define pte_accessible pte_accessible
969static inline bool pte_accessible(struct mm_struct *mm, pte_t a)
970{
971 if (pte_flags(a) & _PAGE_PRESENT)
972 return true;
973
974 if ((pte_flags(a) & _PAGE_PROTNONE) &&
975 atomic_read(&mm->tlb_flush_pending))
976 return true;
977
978 return false;
979}
980
981static inline int pmd_present(pmd_t pmd)
982{
983 /*
984 * Checking for _PAGE_PSE is needed too because
985 * split_huge_page will temporarily clear the present bit (but
986 * the _PAGE_PSE flag will remain set at all times while the
987 * _PAGE_PRESENT bit is clear).
988 */
989 return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE);
990}
991
992#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE
993static inline int pte_protnone(pte_t pte)
994{
995 return (pte_flags(pte) & (_PAGE_PROTNONE | _PAGE_PRESENT))
996 == _PAGE_PROTNONE;
997}
998
999static inline int pmd_protnone(pmd_t pmd)
1000{
1001 return (pmd_flags(pmd) & (_PAGE_PROTNONE | _PAGE_PRESENT))
1002 == _PAGE_PROTNONE;
1003}
1004#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */
1005
1006static inline int pmd_none(pmd_t pmd)
1007{
1008 /* Only check low word on 32-bit platforms, since it might be
1009 out of sync with upper half. */
1010 unsigned long val = native_pmd_val(pmd);
1011 return (val & ~_PAGE_KNL_ERRATUM_MASK) == 0;
1012}
1013
1014static inline unsigned long pmd_page_vaddr(pmd_t pmd)
1015{
1016 return (unsigned long)__va(pmd_val(pmd) & pmd_pfn_mask(pmd));
1017}
1018
1019/*
1020 * Currently stuck as a macro due to indirect forward reference to
1021 * linux/mmzone.h's __section_mem_map_addr() definition:
1022 */
1023#define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd))
1024
1025static inline int pmd_bad(pmd_t pmd)
1026{
1027 return (pmd_flags(pmd) & ~(_PAGE_USER | _PAGE_ACCESSED)) !=
1028 (_KERNPG_TABLE & ~_PAGE_ACCESSED);
1029}
1030
1031static inline unsigned long pages_to_mb(unsigned long npg)
1032{
1033 return npg >> (20 - PAGE_SHIFT);
1034}
1035
1036#if CONFIG_PGTABLE_LEVELS > 2
1037static inline int pud_none(pud_t pud)
1038{
1039 return (native_pud_val(pud) & ~(_PAGE_KNL_ERRATUM_MASK)) == 0;
1040}
1041
1042static inline int pud_present(pud_t pud)
1043{
1044 return pud_flags(pud) & _PAGE_PRESENT;
1045}
1046
1047static inline pmd_t *pud_pgtable(pud_t pud)
1048{
1049 return (pmd_t *)__va(pud_val(pud) & pud_pfn_mask(pud));
1050}
1051
1052/*
1053 * Currently stuck as a macro due to indirect forward reference to
1054 * linux/mmzone.h's __section_mem_map_addr() definition:
1055 */
1056#define pud_page(pud) pfn_to_page(pud_pfn(pud))
1057
1058#define pud_leaf pud_leaf
1059static inline bool pud_leaf(pud_t pud)
1060{
1061 return pud_val(pud) & _PAGE_PSE;
1062}
1063
1064static inline int pud_bad(pud_t pud)
1065{
1066 return (pud_flags(pud) & ~(_KERNPG_TABLE | _PAGE_USER)) != 0;
1067}
1068#endif /* CONFIG_PGTABLE_LEVELS > 2 */
1069
1070#if CONFIG_PGTABLE_LEVELS > 3
1071static inline int p4d_none(p4d_t p4d)
1072{
1073 return (native_p4d_val(p4d) & ~(_PAGE_KNL_ERRATUM_MASK)) == 0;
1074}
1075
1076static inline int p4d_present(p4d_t p4d)
1077{
1078 return p4d_flags(p4d) & _PAGE_PRESENT;
1079}
1080
1081static inline pud_t *p4d_pgtable(p4d_t p4d)
1082{
1083 return (pud_t *)__va(p4d_val(p4d) & p4d_pfn_mask(p4d));
1084}
1085
1086/*
1087 * Currently stuck as a macro due to indirect forward reference to
1088 * linux/mmzone.h's __section_mem_map_addr() definition:
1089 */
1090#define p4d_page(p4d) pfn_to_page(p4d_pfn(p4d))
1091
1092static inline int p4d_bad(p4d_t p4d)
1093{
1094 unsigned long ignore_flags = _KERNPG_TABLE | _PAGE_USER;
1095
1096 if (IS_ENABLED(CONFIG_MITIGATION_PAGE_TABLE_ISOLATION))
1097 ignore_flags |= _PAGE_NX;
1098
1099 return (p4d_flags(p4d) & ~ignore_flags) != 0;
1100}
1101#endif /* CONFIG_PGTABLE_LEVELS > 3 */
1102
1103static inline unsigned long p4d_index(unsigned long address)
1104{
1105 return (address >> P4D_SHIFT) & (PTRS_PER_P4D - 1);
1106}
1107
1108#if CONFIG_PGTABLE_LEVELS > 4
1109static inline int pgd_present(pgd_t pgd)
1110{
1111 if (!pgtable_l5_enabled())
1112 return 1;
1113 return pgd_flags(pgd) & _PAGE_PRESENT;
1114}
1115
1116static inline unsigned long pgd_page_vaddr(pgd_t pgd)
1117{
1118 return (unsigned long)__va((unsigned long)pgd_val(pgd) & PTE_PFN_MASK);
1119}
1120
1121/*
1122 * Currently stuck as a macro due to indirect forward reference to
1123 * linux/mmzone.h's __section_mem_map_addr() definition:
1124 */
1125#define pgd_page(pgd) pfn_to_page(pgd_pfn(pgd))
1126
1127/* to find an entry in a page-table-directory. */
1128static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)
1129{
1130 if (!pgtable_l5_enabled())
1131 return (p4d_t *)pgd;
1132 return (p4d_t *)pgd_page_vaddr(*pgd) + p4d_index(address);
1133}
1134
1135static inline int pgd_bad(pgd_t pgd)
1136{
1137 unsigned long ignore_flags = _PAGE_USER;
1138
1139 if (!pgtable_l5_enabled())
1140 return 0;
1141
1142 if (IS_ENABLED(CONFIG_MITIGATION_PAGE_TABLE_ISOLATION))
1143 ignore_flags |= _PAGE_NX;
1144
1145 return (pgd_flags(pgd) & ~ignore_flags) != _KERNPG_TABLE;
1146}
1147
1148static inline int pgd_none(pgd_t pgd)
1149{
1150 if (!pgtable_l5_enabled())
1151 return 0;
1152 /*
1153 * There is no need to do a workaround for the KNL stray
1154 * A/D bit erratum here. PGDs only point to page tables
1155 * except on 32-bit non-PAE which is not supported on
1156 * KNL.
1157 */
1158 return !native_pgd_val(pgd);
1159}
1160#endif /* CONFIG_PGTABLE_LEVELS > 4 */
1161
1162#endif /* __ASSEMBLER__ */
1163
1164#define KERNEL_PGD_BOUNDARY pgd_index(PAGE_OFFSET)
1165#define KERNEL_PGD_PTRS (PTRS_PER_PGD - KERNEL_PGD_BOUNDARY)
1166
1167#ifndef __ASSEMBLER__
1168
1169extern int direct_gbpages;
1170void init_mem_mapping(void);
1171void early_alloc_pgt_buf(void);
1172void __init poking_init(void);
1173unsigned long init_memory_mapping(unsigned long start,
1174 unsigned long end, pgprot_t prot);
1175
1176#ifdef CONFIG_X86_64
1177extern pgd_t trampoline_pgd_entry;
1178#endif
1179
1180/* local pte updates need not use xchg for locking */
1181static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep)
1182{
1183 pte_t res = *ptep;
1184
1185 /* Pure native function needs no input for mm, addr */
1186 native_pte_clear(NULL, 0, ptep);
1187 return res;
1188}
1189
1190static inline pmd_t native_local_pmdp_get_and_clear(pmd_t *pmdp)
1191{
1192 pmd_t res = *pmdp;
1193
1194 native_pmd_clear(pmdp);
1195 return res;
1196}
1197
1198static inline pud_t native_local_pudp_get_and_clear(pud_t *pudp)
1199{
1200 pud_t res = *pudp;
1201
1202 native_pud_clear(pudp);
1203 return res;
1204}
1205
1206static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
1207 pmd_t *pmdp, pmd_t pmd)
1208{
1209 page_table_check_pmd_set(mm, addr, pmdp, pmd);
1210 set_pmd(pmdp, pmd);
1211}
1212
1213static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
1214 pud_t *pudp, pud_t pud)
1215{
1216 page_table_check_pud_set(mm, addr, pudp, pud);
1217 native_set_pud(pudp, pud);
1218}
1219
1220/*
1221 * We only update the dirty/accessed state if we set
1222 * the dirty bit by hand in the kernel, since the hardware
1223 * will do the accessed bit for us, and we don't want to
1224 * race with other CPU's that might be updating the dirty
1225 * bit at the same time.
1226 */
1227struct vm_area_struct;
1228
1229#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
1230extern int ptep_set_access_flags(struct vm_area_struct *vma,
1231 unsigned long address, pte_t *ptep,
1232 pte_t entry, int dirty);
1233
1234#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
1235bool ptep_test_and_clear_young(struct vm_area_struct *vma,
1236 unsigned long addr, pte_t *ptep);
1237
1238#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
1239bool ptep_clear_flush_young(struct vm_area_struct *vma,
1240 unsigned long address, pte_t *ptep);
1241
1242#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
1243static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
1244 pte_t *ptep)
1245{
1246 pte_t pte = native_ptep_get_and_clear(ptep);
1247 page_table_check_pte_clear(mm, addr, pte);
1248 return pte;
1249}
1250
1251#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
1252static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
1253 unsigned long addr, pte_t *ptep,
1254 int full)
1255{
1256 pte_t pte;
1257 if (full) {
1258 /*
1259 * Full address destruction in progress; paravirt does not
1260 * care about updates and native needs no locking
1261 */
1262 pte = native_local_ptep_get_and_clear(ptep);
1263 page_table_check_pte_clear(mm, addr, pte);
1264 } else {
1265 pte = ptep_get_and_clear(mm, addr, ptep);
1266 }
1267 return pte;
1268}
1269
1270#define __HAVE_ARCH_PTEP_SET_WRPROTECT
1271static inline void ptep_set_wrprotect(struct mm_struct *mm,
1272 unsigned long addr, pte_t *ptep)
1273{
1274 /*
1275 * Avoid accidentally creating shadow stack PTEs
1276 * (Write=0,Dirty=1). Use cmpxchg() to prevent races with
1277 * the hardware setting Dirty=1.
1278 */
1279 pte_t old_pte, new_pte;
1280
1281 old_pte = READ_ONCE(*ptep);
1282 do {
1283 new_pte = pte_wrprotect(old_pte);
1284 } while (!try_cmpxchg((long *)&ptep->pte, (long *)&old_pte, *(long *)&new_pte));
1285}
1286
1287/*
1288 * Note: strictly-zero compare is narrower than pte_none(), but the gap is
1289 * harmless: _PAGE_DIRTY and _PAGE_ACCESSED aren't set on untouched kernel PTEs.
1290 */
1291static inline bool ptep_try_set(pte_t *ptep, pte_t new_pte)
1292{
1293 pte_t old_pte = __pte(0);
1294
1295 return try_cmpxchg((long *)&ptep->pte, (long *)&old_pte, *(long *)&new_pte);
1296}
1297#define ptep_try_set ptep_try_set
1298
1299#define flush_tlb_fix_spurious_fault(vma, address, ptep) do { } while (0)
1300
1301#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
1302extern int pmdp_set_access_flags(struct vm_area_struct *vma,
1303 unsigned long address, pmd_t *pmdp,
1304 pmd_t entry, int dirty);
1305extern int pudp_set_access_flags(struct vm_area_struct *vma,
1306 unsigned long address, pud_t *pudp,
1307 pud_t entry, int dirty);
1308
1309#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
1310bool pmdp_test_and_clear_young(struct vm_area_struct *vma,
1311 unsigned long addr, pmd_t *pmdp);
1312bool pudp_test_and_clear_young(struct vm_area_struct *vma,
1313 unsigned long addr, pud_t *pudp);
1314
1315#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
1316bool pmdp_clear_flush_young(struct vm_area_struct *vma,
1317 unsigned long address, pmd_t *pmdp);
1318
1319
1320#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
1321static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr,
1322 pmd_t *pmdp)
1323{
1324 pmd_t pmd = native_pmdp_get_and_clear(pmdp);
1325
1326 page_table_check_pmd_clear(mm, addr, pmd);
1327
1328 return pmd;
1329}
1330
1331#define __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR
1332static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm,
1333 unsigned long addr, pud_t *pudp)
1334{
1335 pud_t pud = native_pudp_get_and_clear(pudp);
1336
1337 page_table_check_pud_clear(mm, addr, pud);
1338
1339 return pud;
1340}
1341
1342#define __HAVE_ARCH_PMDP_SET_WRPROTECT
1343static inline void pmdp_set_wrprotect(struct mm_struct *mm,
1344 unsigned long addr, pmd_t *pmdp)
1345{
1346 /*
1347 * Avoid accidentally creating shadow stack PTEs
1348 * (Write=0,Dirty=1). Use cmpxchg() to prevent races with
1349 * the hardware setting Dirty=1.
1350 */
1351 pmd_t old_pmd, new_pmd;
1352
1353 old_pmd = READ_ONCE(*pmdp);
1354 do {
1355 new_pmd = pmd_wrprotect(old_pmd);
1356 } while (!try_cmpxchg((long *)pmdp, (long *)&old_pmd, *(long *)&new_pmd));
1357}
1358
1359#ifndef pmdp_establish
1360#define pmdp_establish pmdp_establish
1361static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
1362 unsigned long address, pmd_t *pmdp, pmd_t pmd)
1363{
1364 page_table_check_pmd_set(vma->vm_mm, address, pmdp, pmd);
1365 if (IS_ENABLED(CONFIG_SMP)) {
1366 return xchg(pmdp, pmd);
1367 } else {
1368 pmd_t old = *pmdp;
1369 WRITE_ONCE(*pmdp, pmd);
1370 return old;
1371 }
1372}
1373#endif
1374
1375#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
1376static inline pud_t pudp_establish(struct vm_area_struct *vma,
1377 unsigned long address, pud_t *pudp, pud_t pud)
1378{
1379 page_table_check_pud_set(vma->vm_mm, address, pudp, pud);
1380 if (IS_ENABLED(CONFIG_SMP)) {
1381 return xchg(pudp, pud);
1382 } else {
1383 pud_t old = *pudp;
1384 WRITE_ONCE(*pudp, pud);
1385 return old;
1386 }
1387}
1388#endif
1389
1390#define __HAVE_ARCH_PMDP_INVALIDATE_AD
1391extern pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma,
1392 unsigned long address, pmd_t *pmdp);
1393
1394pud_t pudp_invalidate(struct vm_area_struct *vma, unsigned long address,
1395 pud_t *pudp);
1396
1397/*
1398 * Page table pages are page-aligned. The lower half of the top
1399 * level is used for userspace and the top half for the kernel.
1400 *
1401 * Returns true for parts of the PGD that map userspace and
1402 * false for the parts that map the kernel.
1403 */
1404static inline bool pgdp_maps_userspace(void *__ptr)
1405{
1406 unsigned long ptr = (unsigned long)__ptr;
1407
1408 return (((ptr & ~PAGE_MASK) / sizeof(pgd_t)) < PGD_KERNEL_START);
1409}
1410
1411#ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION
1412/*
1413 * All top-level MITIGATION_PAGE_TABLE_ISOLATION page tables are order-1 pages
1414 * (8k-aligned and 8k in size). The kernel one is at the beginning 4k and
1415 * the user one is in the last 4k. To switch between them, you
1416 * just need to flip the 12th bit in their addresses.
1417 */
1418#define PTI_PGTABLE_SWITCH_BIT PAGE_SHIFT
1419
1420/*
1421 * This generates better code than the inline assembly in
1422 * __set_bit().
1423 */
1424static inline void *ptr_set_bit(void *ptr, int bit)
1425{
1426 unsigned long __ptr = (unsigned long)ptr;
1427
1428 __ptr |= BIT(bit);
1429 return (void *)__ptr;
1430}
1431static inline void *ptr_clear_bit(void *ptr, int bit)
1432{
1433 unsigned long __ptr = (unsigned long)ptr;
1434
1435 __ptr &= ~BIT(bit);
1436 return (void *)__ptr;
1437}
1438
1439static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp)
1440{
1441 return ptr_set_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
1442}
1443
1444static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp)
1445{
1446 return ptr_clear_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
1447}
1448
1449static inline p4d_t *kernel_to_user_p4dp(p4d_t *p4dp)
1450{
1451 return ptr_set_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
1452}
1453
1454static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp)
1455{
1456 return ptr_clear_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
1457}
1458#endif /* CONFIG_MITIGATION_PAGE_TABLE_ISOLATION */
1459
1460/*
1461 * clone_pgd_range(pgd_t *dst, pgd_t *src, int count);
1462 *
1463 * dst - pointer to pgd range anywhere on a pgd page
1464 * src - ""
1465 * count - the number of pgds to copy.
1466 *
1467 * dst and src can be on the same page, but the range must not overlap,
1468 * and must not cross a page boundary.
1469 */
1470static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count)
1471{
1472 memcpy(dst, src, count * sizeof(pgd_t));
1473#ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION
1474 if (!cpu_feature_enabled(X86_FEATURE_PTI))
1475 return;
1476 /* Clone the user space pgd as well */
1477 memcpy(kernel_to_user_pgdp(dst), kernel_to_user_pgdp(src),
1478 count * sizeof(pgd_t));
1479#endif
1480}
1481
1482#define PTE_SHIFT ilog2(PTRS_PER_PTE)
1483static inline int page_level_shift(enum pg_level level)
1484{
1485 return (PAGE_SHIFT - PTE_SHIFT) + level * PTE_SHIFT;
1486}
1487static inline unsigned long page_level_size(enum pg_level level)
1488{
1489 return 1UL << page_level_shift(level);
1490}
1491static inline unsigned long page_level_mask(enum pg_level level)
1492{
1493 return ~(page_level_size(level) - 1);
1494}
1495
1496/*
1497 * The x86 doesn't have any external MMU info: the kernel page
1498 * tables contain all the necessary information.
1499 */
1500static inline void update_mmu_cache(struct vm_area_struct *vma,
1501 unsigned long addr, pte_t *ptep)
1502{
1503}
1504static inline void update_mmu_cache_range(struct vm_fault *vmf,
1505 struct vm_area_struct *vma, unsigned long addr,
1506 pte_t *ptep, unsigned int nr)
1507{
1508}
1509static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
1510 unsigned long addr, pmd_t *pmd)
1511{
1512}
1513static inline void update_mmu_cache_pud(struct vm_area_struct *vma,
1514 unsigned long addr, pud_t *pud)
1515{
1516}
1517static inline pte_t pte_swp_mkexclusive(pte_t pte)
1518{
1519 return pte_set_flags(pte, _PAGE_SWP_EXCLUSIVE);
1520}
1521
1522static inline bool pte_swp_exclusive(pte_t pte)
1523{
1524 return pte_flags(pte) & _PAGE_SWP_EXCLUSIVE;
1525}
1526
1527static inline pte_t pte_swp_clear_exclusive(pte_t pte)
1528{
1529 return pte_clear_flags(pte, _PAGE_SWP_EXCLUSIVE);
1530}
1531
1532#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
1533static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
1534{
1535 return pte_set_flags(pte, _PAGE_SWP_SOFT_DIRTY);
1536}
1537
1538static inline int pte_swp_soft_dirty(pte_t pte)
1539{
1540 return pte_flags(pte) & _PAGE_SWP_SOFT_DIRTY;
1541}
1542
1543static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
1544{
1545 return pte_clear_flags(pte, _PAGE_SWP_SOFT_DIRTY);
1546}
1547
1548#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
1549static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)
1550{
1551 return pmd_set_flags(pmd, _PAGE_SWP_SOFT_DIRTY);
1552}
1553
1554static inline int pmd_swp_soft_dirty(pmd_t pmd)
1555{
1556 return pmd_flags(pmd) & _PAGE_SWP_SOFT_DIRTY;
1557}
1558
1559static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)
1560{
1561 return pmd_clear_flags(pmd, _PAGE_SWP_SOFT_DIRTY);
1562}
1563#endif
1564#endif
1565
1566#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
1567static inline pte_t pte_swp_mkuffd(pte_t pte)
1568{
1569 return pte_set_flags(pte, _PAGE_SWP_UFFD);
1570}
1571
1572static inline int pte_swp_uffd(pte_t pte)
1573{
1574 return pte_flags(pte) & _PAGE_SWP_UFFD;
1575}
1576
1577static inline pte_t pte_swp_clear_uffd(pte_t pte)
1578{
1579 return pte_clear_flags(pte, _PAGE_SWP_UFFD);
1580}
1581
1582static inline pmd_t pmd_swp_mkuffd(pmd_t pmd)
1583{
1584 return pmd_set_flags(pmd, _PAGE_SWP_UFFD);
1585}
1586
1587static inline int pmd_swp_uffd(pmd_t pmd)
1588{
1589 return pmd_flags(pmd) & _PAGE_SWP_UFFD;
1590}
1591
1592static inline pmd_t pmd_swp_clear_uffd(pmd_t pmd)
1593{
1594 return pmd_clear_flags(pmd, _PAGE_SWP_UFFD);
1595}
1596#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
1597
1598static inline u16 pte_flags_pkey(unsigned long pte_flags)
1599{
1600#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
1601 /* ifdef to avoid doing 59-bit shift on 32-bit values */
1602 return (pte_flags & _PAGE_PKEY_MASK) >> _PAGE_BIT_PKEY_BIT0;
1603#else
1604 return 0;
1605#endif
1606}
1607
1608static inline bool __pkru_allows_pkey(u16 pkey, bool write)
1609{
1610 u32 pkru = read_pkru();
1611
1612 if (!__pkru_allows_read(pkru, pkey))
1613 return false;
1614 if (write && !__pkru_allows_write(pkru, pkey))
1615 return false;
1616
1617 return true;
1618}
1619
1620/*
1621 * 'pteval' can come from a PTE, PMD or PUD. We only check
1622 * _PAGE_PRESENT, _PAGE_USER, and _PAGE_RW in here which are the
1623 * same value on all 3 types.
1624 */
1625static inline bool __pte_access_permitted(unsigned long pteval, bool write)
1626{
1627 unsigned long need_pte_bits = _PAGE_PRESENT|_PAGE_USER;
1628
1629 /*
1630 * Write=0,Dirty=1 PTEs are shadow stack, which the kernel
1631 * shouldn't generally allow access to, but since they
1632 * are already Write=0, the below logic covers both cases.
1633 */
1634 if (write)
1635 need_pte_bits |= _PAGE_RW;
1636
1637 if ((pteval & need_pte_bits) != need_pte_bits)
1638 return 0;
1639
1640 return __pkru_allows_pkey(pte_flags_pkey(pteval), write);
1641}
1642
1643#define pte_access_permitted pte_access_permitted
1644static inline bool pte_access_permitted(pte_t pte, bool write)
1645{
1646 return __pte_access_permitted(pte_val(pte), write);
1647}
1648
1649#define pmd_access_permitted pmd_access_permitted
1650static inline bool pmd_access_permitted(pmd_t pmd, bool write)
1651{
1652 return __pte_access_permitted(pmd_val(pmd), write);
1653}
1654
1655#define pud_access_permitted pud_access_permitted
1656static inline bool pud_access_permitted(pud_t pud, bool write)
1657{
1658 return __pte_access_permitted(pud_val(pud), write);
1659}
1660
1661#define __HAVE_ARCH_PFN_MODIFY_ALLOWED 1
1662extern bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot);
1663
1664static inline bool arch_has_pfn_modify_check(void)
1665{
1666 return boot_cpu_has_bug(X86_BUG_L1TF);
1667}
1668
1669#define arch_check_zapped_pte arch_check_zapped_pte
1670void arch_check_zapped_pte(struct vm_area_struct *vma, pte_t pte);
1671
1672#define arch_check_zapped_pmd arch_check_zapped_pmd
1673void arch_check_zapped_pmd(struct vm_area_struct *vma, pmd_t pmd);
1674
1675#define arch_check_zapped_pud arch_check_zapped_pud
1676void arch_check_zapped_pud(struct vm_area_struct *vma, pud_t pud);
1677
1678#ifdef CONFIG_XEN_PV
1679#define arch_has_hw_nonleaf_pmd_young arch_has_hw_nonleaf_pmd_young
1680static inline bool arch_has_hw_nonleaf_pmd_young(void)
1681{
1682 return !cpu_feature_enabled(X86_FEATURE_XENPV);
1683}
1684#endif
1685
1686#ifdef CONFIG_PAGE_TABLE_CHECK
1687static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
1688{
1689 return (pte_val(pte) & _PAGE_PRESENT) && (pte_val(pte) & _PAGE_USER);
1690}
1691
1692static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd)
1693{
1694 return pmd_leaf(pmd) && (pmd_val(pmd) & _PAGE_PRESENT) && (pmd_val(pmd) & _PAGE_USER);
1695}
1696
1697static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud)
1698{
1699 return pud_leaf(pud) && (pud_val(pud) & _PAGE_PRESENT) && (pud_val(pud) & _PAGE_USER);
1700}
1701#endif
1702
1703#ifdef CONFIG_X86_SGX
1704int arch_memory_failure(unsigned long pfn, int flags);
1705#define arch_memory_failure arch_memory_failure
1706
1707bool arch_is_platform_page(u64 paddr);
1708#define arch_is_platform_page arch_is_platform_page
1709#endif
1710
1711/*
1712 * Use set_p*_safe(), and elide TLB flushing, when confident that *no*
1713 * TLB flush will be required as a result of the "set". For example, use
1714 * in scenarios where it is known ahead of time that the routine is
1715 * setting non-present entries, or re-setting an existing entry to the
1716 * same value. Otherwise, use the typical "set" helpers and flush the
1717 * TLB.
1718 */
1719#define set_pte_safe(ptep, pte) \
1720({ \
1721 WARN_ON_ONCE(pte_present(*ptep) && !pte_same(*ptep, pte)); \
1722 set_pte(ptep, pte); \
1723})
1724
1725#define set_pmd_safe(pmdp, pmd) \
1726({ \
1727 WARN_ON_ONCE(pmd_present(*pmdp) && !pmd_same(*pmdp, pmd)); \
1728 set_pmd(pmdp, pmd); \
1729})
1730
1731#define set_pud_safe(pudp, pud) \
1732({ \
1733 WARN_ON_ONCE(pud_present(*pudp) && !pud_same(*pudp, pud)); \
1734 set_pud(pudp, pud); \
1735})
1736
1737#define set_p4d_safe(p4dp, p4d) \
1738({ \
1739 WARN_ON_ONCE(p4d_present(*p4dp) && !p4d_same(*p4dp, p4d)); \
1740 set_p4d(p4dp, p4d); \
1741})
1742
1743#define set_pgd_safe(pgdp, pgd) \
1744({ \
1745 WARN_ON_ONCE(pgd_present(*pgdp) && !pgd_same(*pgdp, pgd)); \
1746 set_pgd(pgdp, pgd); \
1747})
1748#endif /* __ASSEMBLER__ */
1749
1750#endif /* _ASM_X86_PGTABLE_H */