projects/linux·blob master · refs

1305 lines · 32.0 KiB

1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2012 Regents of the University of California
4 */
5
6#ifndef _ASM_RISCV_PGTABLE_H
7#define _ASM_RISCV_PGTABLE_H
8
9#include <linux/mmzone.h>
10#include <linux/sizes.h>
11
12#include <asm/pgtable-bits.h>
13
14#ifndef CONFIG_MMU
15#ifdef CONFIG_RELOCATABLE
16#define KERNEL_LINK_ADDR UL(0)
17#else
18#define KERNEL_LINK_ADDR _AC(CONFIG_PHYS_RAM_BASE, UL)
19#endif
20#define KERN_VIRT_SIZE (UL(-1))
21#else
22
23#define ADDRESS_SPACE_END (UL(-1))
24
25#ifdef CONFIG_64BIT
26/* Leave 2GB for kernel and BPF at the end of the address space */
27#define KERNEL_LINK_ADDR (ADDRESS_SPACE_END - SZ_2G + 1)
28#else
29#define KERNEL_LINK_ADDR PAGE_OFFSET
30#endif
31
32/* Number of entries in the page global directory */
33#define PTRS_PER_PGD (PAGE_SIZE / sizeof(pgd_t))
34/* Number of entries in the page table */
35#define PTRS_PER_PTE (PAGE_SIZE / sizeof(pte_t))
36
37/*
38 * Half of the kernel address space (1/4 of the entries of the page global
39 * directory) is for the direct mapping.
40 */
41#define KERN_VIRT_SIZE ((PTRS_PER_PGD / 2 * PGDIR_SIZE) / 2)
42
43#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1)
44#define VMALLOC_END PAGE_OFFSET
45#define VMALLOC_START (PAGE_OFFSET - VMALLOC_SIZE)
46
47#define BPF_JIT_REGION_SIZE (SZ_128M)
48#ifdef CONFIG_64BIT
49#define BPF_JIT_REGION_START (BPF_JIT_REGION_END - BPF_JIT_REGION_SIZE)
50#define BPF_JIT_REGION_END (MODULES_END)
51#else
52#define BPF_JIT_REGION_START (PAGE_OFFSET - BPF_JIT_REGION_SIZE)
53#define BPF_JIT_REGION_END (VMALLOC_END)
54#endif
55
56/* Modules always live before the kernel */
57#ifdef CONFIG_64BIT
58/* This is used to define the end of the KASAN shadow region */
59#define MODULES_LOWEST_VADDR (KERNEL_LINK_ADDR - SZ_2G)
60#define MODULES_VADDR (PFN_ALIGN((unsigned long)&_end) - SZ_2G)
61#define MODULES_END (PFN_ALIGN((unsigned long)&_start))
62#else
63#define MODULES_VADDR VMALLOC_START
64#define MODULES_END VMALLOC_END
65#endif
66
67/*
68 * Roughly size the vmemmap space to be large enough to fit enough
69 * struct pages to map half the virtual address space. Then
70 * position vmemmap directly below the VMALLOC region.
71 */
72#define VA_BITS_SV32 32
73#ifdef CONFIG_64BIT
74#define VA_BITS_SV39 39
75#define VA_BITS_SV48 48
76#define VA_BITS_SV57 57
77
78#define VA_BITS (pgtable_l5_enabled ? \
79 VA_BITS_SV57 : (pgtable_l4_enabled ? VA_BITS_SV48 : VA_BITS_SV39))
80#else
81#define VA_BITS VA_BITS_SV32
82#endif
83
84#define VMEMMAP_SHIFT \
85 (VA_BITS - PAGE_SHIFT - 1 + STRUCT_PAGE_MAX_SHIFT)
86#define VMEMMAP_SIZE BIT(VMEMMAP_SHIFT)
87#define VMEMMAP_END VMALLOC_START
88#define VMEMMAP_START (VMALLOC_START - VMEMMAP_SIZE)
89
90/*
91 * Define vmemmap for pfn_to_page & page_to_pfn calls. Needed if kernel
92 * is configured with CONFIG_SPARSEMEM_VMEMMAP enabled.
93 */
94#define vmemmap ((struct page *)VMEMMAP_START - vmemmap_start_pfn)
95
96/* Needed to limit get_free_mem_region() */
97#if defined(CONFIG_FLATMEM)
98#define DIRECT_MAP_PHYSMEM_END (phys_ram_base + KERN_VIRT_SIZE - 1)
99#elif defined(CONFIG_SPARSEMEM_VMEMMAP)
100#define DIRECT_MAP_PHYSMEM_END \
101 ((vmemmap_start_pfn + VMEMMAP_SIZE / sizeof(struct page)) * PAGE_SIZE - 1)
102#elif defined(CONFIG_SPARSEMEM)
103/* DIRECT_MAP_PHYSMEM_END is not limited by VA space assignment in this case */
104#endif
105
106#define PCI_IO_SIZE SZ_16M
107#define PCI_IO_END VMEMMAP_START
108#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE)
109
110#define FIXADDR_TOP PCI_IO_START
111#ifdef CONFIG_64BIT
112#define MAX_FDT_SIZE PMD_SIZE
113#define FIX_FDT_SIZE (MAX_FDT_SIZE + SZ_2M)
114#define FIXADDR_SIZE (PMD_SIZE + FIX_FDT_SIZE)
115#else
116#define MAX_FDT_SIZE PGDIR_SIZE
117#define FIX_FDT_SIZE MAX_FDT_SIZE
118#define FIXADDR_SIZE (PGDIR_SIZE + FIX_FDT_SIZE)
119#endif
120#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE)
121
122#endif
123
124#ifndef __ASSEMBLER__
125
126#include <asm/page.h>
127#include <asm/tlbflush.h>
128#include <linux/mm_types.h>
129#include <asm/compat.h>
130#include <asm/cpufeature.h>
131
132#define __page_val_to_pfn(_val) (((_val) & _PAGE_PFN_MASK) >> _PAGE_PFN_SHIFT)
133
134#ifdef CONFIG_64BIT
135#include <asm/pgtable-64.h>
136
137#define MMAP_VA_BITS_64 ((VA_BITS >= VA_BITS_SV48) ? VA_BITS_SV48 : VA_BITS)
138#define MMAP_MIN_VA_BITS_64 (VA_BITS_SV39)
139#define MMAP_VA_BITS (is_compat_task() ? VA_BITS_SV32 : MMAP_VA_BITS_64)
140#define MMAP_MIN_VA_BITS (is_compat_task() ? VA_BITS_SV32 : MMAP_MIN_VA_BITS_64)
141#else
142#include <asm/pgtable-32.h>
143#endif /* CONFIG_64BIT */
144
145#include <linux/page_table_check.h>
146
147struct pt_alloc_ops {
148 pte_t *(*get_pte_virt)(phys_addr_t pa);
149 phys_addr_t (*alloc_pte)(uintptr_t va);
150#ifndef __PAGETABLE_PMD_FOLDED
151 pmd_t *(*get_pmd_virt)(phys_addr_t pa);
152 phys_addr_t (*alloc_pmd)(uintptr_t va);
153 pud_t *(*get_pud_virt)(phys_addr_t pa);
154 phys_addr_t (*alloc_pud)(uintptr_t va);
155 p4d_t *(*get_p4d_virt)(phys_addr_t pa);
156 phys_addr_t (*alloc_p4d)(uintptr_t va);
157#endif
158};
159
160extern struct pt_alloc_ops pt_ops __meminitdata;
161
162#ifdef CONFIG_MMU
163/* Number of PGD entries that a user-mode program can use */
164#define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE)
165
166/* Page protection bits */
167#define _PAGE_BASE (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_USER)
168
169#define PAGE_NONE __pgprot(_PAGE_PROT_NONE | _PAGE_READ)
170#define PAGE_READ __pgprot(_PAGE_BASE | _PAGE_READ)
171#define PAGE_WRITE __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_WRITE)
172#define PAGE_EXEC __pgprot(_PAGE_BASE | _PAGE_EXEC)
173#define PAGE_READ_EXEC __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC)
174#define PAGE_WRITE_EXEC __pgprot(_PAGE_BASE | _PAGE_READ | \
175 _PAGE_EXEC | _PAGE_WRITE)
176#define PAGE_SHADOWSTACK __pgprot(_PAGE_BASE | _PAGE_WRITE)
177
178#define PAGE_COPY PAGE_READ
179#define PAGE_COPY_EXEC PAGE_READ_EXEC
180#define PAGE_SHARED PAGE_WRITE
181#define PAGE_SHARED_EXEC PAGE_WRITE_EXEC
182
183#define _PAGE_KERNEL (_PAGE_READ \
184 | _PAGE_WRITE \
185 | _PAGE_PRESENT \
186 | _PAGE_ACCESSED \
187 | _PAGE_DIRTY \
188 | _PAGE_GLOBAL)
189
190#define PAGE_KERNEL __pgprot(_PAGE_KERNEL)
191#define PAGE_KERNEL_READ __pgprot(_PAGE_KERNEL & ~_PAGE_WRITE)
192#define PAGE_KERNEL_EXEC __pgprot(_PAGE_KERNEL | _PAGE_EXEC)
193#define PAGE_KERNEL_READ_EXEC __pgprot((_PAGE_KERNEL & ~_PAGE_WRITE) \
194 | _PAGE_EXEC)
195
196#define PAGE_TABLE __pgprot(_PAGE_TABLE)
197
198#define _PAGE_KERNEL_NC ((_PAGE_KERNEL & ~_PAGE_MTMASK) | _PAGE_NOCACHE)
199#define _PAGE_IOREMAP ((_PAGE_KERNEL & ~_PAGE_MTMASK) | _PAGE_IO)
200#define PAGE_KERNEL_IO __pgprot(_PAGE_IOREMAP)
201
202extern pgd_t swapper_pg_dir[];
203extern pgd_t trampoline_pg_dir[];
204extern pgd_t early_pg_dir[];
205
206#ifdef CONFIG_TRANSPARENT_HUGEPAGE
207static inline int pmd_present(pmd_t pmd)
208{
209 /*
210 * Checking for _PAGE_LEAF is needed too because:
211 * When splitting a THP, split_huge_page() will temporarily clear
212 * the present bit, in this situation, pmd_present() and
213 * pmd_trans_huge() still needs to return true.
214 */
215 return (pmd_val(pmd) & (_PAGE_PRESENT | _PAGE_PROT_NONE | _PAGE_LEAF));
216}
217#else
218static inline int pmd_present(pmd_t pmd)
219{
220 return (pmd_val(pmd) & (_PAGE_PRESENT | _PAGE_PROT_NONE));
221}
222#endif
223
224static inline int pmd_none(pmd_t pmd)
225{
226 return (pmd_val(pmd) == 0);
227}
228
229static inline int pmd_bad(pmd_t pmd)
230{
231 return !pmd_present(pmd) || (pmd_val(pmd) & _PAGE_LEAF);
232}
233
234#define pmd_leaf pmd_leaf
235static inline bool pmd_leaf(pmd_t pmd)
236{
237 return pmd_present(pmd) && (pmd_val(pmd) & _PAGE_LEAF);
238}
239
240static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
241{
242 WRITE_ONCE(*pmdp, pmd);
243}
244
245static inline void pmd_clear(pmd_t *pmdp)
246{
247 set_pmd(pmdp, __pmd(0));
248}
249
250static inline pgd_t pfn_pgd(unsigned long pfn, pgprot_t prot)
251{
252 unsigned long prot_val = pgprot_val(prot);
253
254 ALT_THEAD_PMA(prot_val);
255
256 return __pgd((pfn << _PAGE_PFN_SHIFT) | prot_val);
257}
258
259static inline unsigned long _pgd_pfn(pgd_t pgd)
260{
261 return __page_val_to_pfn(pgd_val(pgd));
262}
263
264static inline struct page *pmd_page(pmd_t pmd)
265{
266 return pfn_to_page(__page_val_to_pfn(pmd_val(pmd)));
267}
268
269static inline unsigned long pmd_page_vaddr(pmd_t pmd)
270{
271 return (unsigned long)pfn_to_virt(__page_val_to_pfn(pmd_val(pmd)));
272}
273
274static inline pte_t pmd_pte(pmd_t pmd)
275{
276 return __pte(pmd_val(pmd));
277}
278
279static inline pte_t pud_pte(pud_t pud)
280{
281 return __pte(pud_val(pud));
282}
283
284#ifdef CONFIG_RISCV_ISA_SVNAPOT
285
286static __always_inline bool has_svnapot(void)
287{
288 return riscv_has_extension_likely(RISCV_ISA_EXT_SVNAPOT);
289}
290
291static inline unsigned long pte_napot(pte_t pte)
292{
293 return pte_val(pte) & _PAGE_NAPOT;
294}
295
296static inline pte_t pte_mknapot(pte_t pte, unsigned int order)
297{
298 int pos = order - 1 + _PAGE_PFN_SHIFT;
299 unsigned long napot_bit = BIT(pos);
300 unsigned long napot_mask = ~GENMASK(pos, _PAGE_PFN_SHIFT);
301
302 return __pte((pte_val(pte) & napot_mask) | napot_bit | _PAGE_NAPOT);
303}
304
305#else
306
307static __always_inline bool has_svnapot(void) { return false; }
308
309static inline unsigned long pte_napot(pte_t pte)
310{
311 return 0;
312}
313
314#endif /* CONFIG_RISCV_ISA_SVNAPOT */
315
316/* Yields the page frame number (PFN) of a page table entry */
317static inline unsigned long pte_pfn(pte_t pte)
318{
319 unsigned long res = __page_val_to_pfn(pte_val(pte));
320
321 if (has_svnapot() && pte_napot(pte))
322 res = res & (res - 1UL);
323
324 return res;
325}
326
327#define pte_page(x) pfn_to_page(pte_pfn(x))
328
329/* Constructs a page table entry */
330static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot)
331{
332 unsigned long prot_val = pgprot_val(prot);
333
334 ALT_THEAD_PMA(prot_val);
335
336 return __pte((pfn << _PAGE_PFN_SHIFT) | prot_val);
337}
338
339#define pte_pgprot pte_pgprot
340static inline pgprot_t pte_pgprot(pte_t pte)
341{
342 unsigned long pfn = pte_pfn(pte);
343
344 return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));
345}
346
347static inline int pte_present(pte_t pte)
348{
349 return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE));
350}
351
352#define pte_accessible pte_accessible
353static inline unsigned long pte_accessible(struct mm_struct *mm, pte_t a)
354{
355 if (pte_val(a) & _PAGE_PRESENT)
356 return true;
357
358 if ((pte_val(a) & _PAGE_PROT_NONE) &&
359 atomic_read(&mm->tlb_flush_pending))
360 return true;
361
362 return false;
363}
364
365static inline int pte_none(pte_t pte)
366{
367 return (pte_val(pte) == 0);
368}
369
370static inline int pte_write(pte_t pte)
371{
372 return pte_val(pte) & _PAGE_WRITE;
373}
374
375static inline int pte_exec(pte_t pte)
376{
377 return pte_val(pte) & _PAGE_EXEC;
378}
379
380static inline int pte_user(pte_t pte)
381{
382 return pte_val(pte) & _PAGE_USER;
383}
384
385static inline int pte_huge(pte_t pte)
386{
387 return pte_present(pte) && (pte_val(pte) & _PAGE_LEAF);
388}
389
390static inline int pte_dirty(pte_t pte)
391{
392 return pte_val(pte) & _PAGE_DIRTY;
393}
394
395static inline int pte_young(pte_t pte)
396{
397 return pte_val(pte) & _PAGE_ACCESSED;
398}
399
400static inline int pte_special(pte_t pte)
401{
402 return pte_val(pte) & _PAGE_SPECIAL;
403}
404
405/* static inline pte_t pte_rdprotect(pte_t pte) */
406
407static inline pte_t pte_wrprotect(pte_t pte)
408{
409 return __pte((pte_val(pte) & ~(_PAGE_WRITE)) | (_PAGE_READ));
410}
411
412#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
413#define pgtable_supports_uffd() \
414 riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B)
415
416static inline bool pte_uffd(pte_t pte)
417{
418 return !!(pte_val(pte) & _PAGE_UFFD);
419}
420
421static inline pte_t pte_mkuffd(pte_t pte)
422{
423 return pte_wrprotect(__pte(pte_val(pte) | _PAGE_UFFD));
424}
425
426static inline pte_t pte_clear_uffd(pte_t pte)
427{
428 return __pte(pte_val(pte) & ~(_PAGE_UFFD));
429}
430
431static inline bool pte_swp_uffd(pte_t pte)
432{
433 return !!(pte_val(pte) & _PAGE_SWP_UFFD);
434}
435
436static inline pte_t pte_swp_mkuffd(pte_t pte)
437{
438 return __pte(pte_val(pte) | _PAGE_SWP_UFFD);
439}
440
441static inline pte_t pte_swp_clear_uffd(pte_t pte)
442{
443 return __pte(pte_val(pte) & ~(_PAGE_SWP_UFFD));
444}
445#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
446
447/* static inline pte_t pte_mkread(pte_t pte) */
448
449struct vm_area_struct;
450pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma);
451#define pte_mkwrite pte_mkwrite
452
453static inline pte_t pte_mkwrite_novma(pte_t pte)
454{
455 return __pte(pte_val(pte) | _PAGE_WRITE);
456}
457
458static inline pte_t pte_mkwrite_shstk(pte_t pte)
459{
460 return __pte((pte_val(pte) & ~(_PAGE_LEAF)) | _PAGE_WRITE);
461}
462
463/* static inline pte_t pte_mkexec(pte_t pte) */
464
465static inline pte_t pte_mkdirty(pte_t pte)
466{
467 return __pte(pte_val(pte) | _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
468}
469
470static inline pte_t pte_mkclean(pte_t pte)
471{
472 return __pte(pte_val(pte) & ~(_PAGE_DIRTY));
473}
474
475static inline pte_t pte_mkyoung(pte_t pte)
476{
477 return __pte(pte_val(pte) | _PAGE_ACCESSED);
478}
479
480static inline pte_t pte_mkold(pte_t pte)
481{
482 return __pte(pte_val(pte) & ~(_PAGE_ACCESSED));
483}
484
485static inline pte_t pte_mkspecial(pte_t pte)
486{
487 return __pte(pte_val(pte) | _PAGE_SPECIAL);
488}
489
490static inline pte_t pte_mkhuge(pte_t pte)
491{
492 return pte;
493}
494
495#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
496#define pgtable_supports_soft_dirty() \
497 (IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) && \
498 riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B))
499
500static inline bool pte_soft_dirty(pte_t pte)
501{
502 return !!(pte_val(pte) & _PAGE_SOFT_DIRTY);
503}
504
505static inline pte_t pte_mksoft_dirty(pte_t pte)
506{
507 return __pte(pte_val(pte) | _PAGE_SOFT_DIRTY);
508}
509
510static inline pte_t pte_clear_soft_dirty(pte_t pte)
511{
512 return __pte(pte_val(pte) & ~(_PAGE_SOFT_DIRTY));
513}
514
515static inline bool pte_swp_soft_dirty(pte_t pte)
516{
517 return !!(pte_val(pte) & _PAGE_SWP_SOFT_DIRTY);
518}
519
520static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
521{
522 return __pte(pte_val(pte) | _PAGE_SWP_SOFT_DIRTY);
523}
524
525static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
526{
527 return __pte(pte_val(pte) & ~(_PAGE_SWP_SOFT_DIRTY));
528}
529#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */
530
531#ifdef CONFIG_RISCV_ISA_SVNAPOT
532#define pte_leaf_size(pte) (pte_napot(pte) ? \
533 napot_cont_size(napot_cont_order(pte)) :\
534 PAGE_SIZE)
535#endif
536
537#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE
538static inline int pte_protnone(pte_t pte)
539{
540 return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE)) == _PAGE_PROT_NONE;
541}
542
543static inline int pmd_protnone(pmd_t pmd)
544{
545 return pte_protnone(pmd_pte(pmd));
546}
547#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */
548
549/* Modify page protection bits */
550static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
551{
552 unsigned long newprot_val = pgprot_val(newprot);
553
554 ALT_THEAD_PMA(newprot_val);
555
556 return __pte((pte_val(pte) & _PAGE_CHG_MASK) | newprot_val);
557}
558
559#define pgd_ERROR(e) \
560 pr_err("%s:%d: bad pgd " PTE_FMT ".\n", __FILE__, __LINE__, pgd_val(e))
561
562
563/* Commit new configuration to MMU hardware */
564static inline void update_mmu_cache_range(struct vm_fault *vmf,
565 struct vm_area_struct *vma, unsigned long address,
566 pte_t *ptep, unsigned int nr)
567{
568 unsigned long asid = get_mm_asid(vma->vm_mm);
569
570 /*
571 * Svvptc guarantees that the new valid pte will be visible within
572 * a bounded timeframe, so when the uarch does not cache invalid
573 * entries, we don't have to do anything.
574 */
575 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC))
576 return;
577
578 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)) {
579 local_sfence_w_inval();
580 while (nr--)
581 local_sinval_vma(address + nr * PAGE_SIZE, asid);
582 local_sfence_inval_ir();
583 return;
584 }
585
586 /*
587 * The kernel assumes that TLBs don't cache invalid entries, but
588 * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a
589 * cache flush; it is necessary even after writing invalid entries.
590 * Relying on flush_tlb_fix_spurious_fault would suffice, but
591 * the extra traps reduce performance. So, eagerly SFENCE.VMA.
592 */
593 while (nr--)
594 local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid);
595
596}
597#define update_mmu_cache(vma, addr, ptep) \
598 update_mmu_cache_range(NULL, vma, addr, ptep, 1)
599
600#define update_mmu_tlb_range(vma, addr, ptep, nr) \
601 update_mmu_cache_range(NULL, vma, addr, ptep, nr)
602
603static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
604 unsigned long address, pmd_t *pmdp)
605{
606 pte_t *ptep = (pte_t *)pmdp;
607
608 update_mmu_cache(vma, address, ptep);
609}
610
611#define __HAVE_ARCH_PTE_SAME
612static inline int pte_same(pte_t pte_a, pte_t pte_b)
613{
614 return pte_val(pte_a) == pte_val(pte_b);
615}
616
617/*
618 * Certain architectures need to do special things when PTEs within
619 * a page table are directly modified. Thus, the following hook is
620 * made available.
621 */
622static inline void set_pte(pte_t *ptep, pte_t pteval)
623{
624 WRITE_ONCE(*ptep, pteval);
625}
626
627void flush_icache_pte(struct mm_struct *mm, pte_t pte);
628
629static inline void __set_pte_at(struct mm_struct *mm, pte_t *ptep, pte_t pteval)
630{
631 if (pte_present(pteval) && pte_exec(pteval))
632 flush_icache_pte(mm, pteval);
633
634 set_pte(ptep, pteval);
635}
636
637#define PFN_PTE_SHIFT _PAGE_PFN_SHIFT
638
639static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
640 pte_t *ptep, pte_t pteval, unsigned int nr)
641{
642 page_table_check_ptes_set(mm, addr, ptep, pteval, nr);
643
644 for (;;) {
645 __set_pte_at(mm, ptep, pteval);
646 if (--nr == 0)
647 break;
648 ptep++;
649 pte_val(pteval) += 1 << _PAGE_PFN_SHIFT;
650 }
651}
652#define set_ptes set_ptes
653
654static inline void pte_clear(struct mm_struct *mm,
655 unsigned long addr, pte_t *ptep)
656{
657 __set_pte_at(mm, ptep, __pte(0));
658}
659
660#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS /* defined in mm/pgtable.c */
661extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address,
662 pte_t *ptep, pte_t entry, int dirty);
663#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG /* defined in mm/pgtable.c */
664bool ptep_test_and_clear_young(struct vm_area_struct *vma,
665 unsigned long address, pte_t *ptep);
666
667#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
668static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
669 unsigned long address, pte_t *ptep)
670{
671#ifdef CONFIG_SMP
672 pte_t pte = __pte(xchg(&ptep->pte, 0));
673#else
674 pte_t pte = *ptep;
675
676 set_pte(ptep, __pte(0));
677#endif
678
679 page_table_check_pte_clear(mm, address, pte);
680
681 return pte;
682}
683
684#define __HAVE_ARCH_PTEP_SET_WRPROTECT
685static inline void ptep_set_wrprotect(struct mm_struct *mm,
686 unsigned long address, pte_t *ptep)
687{
688 pte_t read_pte = READ_ONCE(*ptep);
689 /*
690 * ptep_set_wrprotect can be called for shadow stack ranges too.
691 * shadow stack memory is XWR = 010 and thus clearing _PAGE_WRITE will lead to
692 * encoding 000b which is wrong encoding with V = 1. This should lead to page fault
693 * but we dont want this wrong configuration to be set in page tables.
694 */
695 atomic_long_set((atomic_long_t *)ptep,
696 ((pte_val(read_pte) & ~(unsigned long)_PAGE_WRITE) | _PAGE_READ));
697}
698
699#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
700static inline bool ptep_clear_flush_young(struct vm_area_struct *vma,
701 unsigned long address, pte_t *ptep)
702{
703 /*
704 * This comment is borrowed from x86, but applies equally to RISC-V:
705 *
706 * Clearing the accessed bit without a TLB flush
707 * doesn't cause data corruption. [ It could cause incorrect
708 * page aging and the (mistaken) reclaim of hot pages, but the
709 * chance of that should be relatively low. ]
710 *
711 * So as a performance optimization don't flush the TLB when
712 * clearing the accessed bit, it will eventually be flushed by
713 * a context switch or a VM operation anyway. [ In the rare
714 * event of it not getting flushed for a long time the delay
715 * shouldn't really matter because there's no real memory
716 * pressure for swapout to react to. ]
717 */
718 return ptep_test_and_clear_young(vma, address, ptep);
719}
720
721#define pgprot_nx pgprot_nx
722static inline pgprot_t pgprot_nx(pgprot_t _prot)
723{
724 return __pgprot(pgprot_val(_prot) & ~_PAGE_EXEC);
725}
726
727#define pgprot_noncached pgprot_noncached
728static inline pgprot_t pgprot_noncached(pgprot_t _prot)
729{
730 unsigned long prot = pgprot_val(_prot);
731
732 prot &= ~_PAGE_MTMASK;
733 prot |= _PAGE_IO;
734
735 return __pgprot(prot);
736}
737
738#define pgprot_writecombine pgprot_writecombine
739static inline pgprot_t pgprot_writecombine(pgprot_t _prot)
740{
741 unsigned long prot = pgprot_val(_prot);
742
743 prot &= ~_PAGE_MTMASK;
744 prot |= _PAGE_NOCACHE;
745
746 return __pgprot(prot);
747}
748
749#define pgprot_dmacoherent pgprot_writecombine
750
751/*
752 * Both Svade and Svadu control the hardware behavior when the PTE A/D bits need to be set. By
753 * default the M-mode firmware enables the hardware updating scheme when only Svadu is present in
754 * DT.
755 */
756#define arch_has_hw_pte_young arch_has_hw_pte_young
757static inline bool arch_has_hw_pte_young(void)
758{
759 return riscv_has_extension_unlikely(RISCV_ISA_EXT_SVADU);
760}
761
762/*
763 * THP functions
764 */
765static inline pmd_t pte_pmd(pte_t pte)
766{
767 return __pmd(pte_val(pte));
768}
769
770static inline pud_t pte_pud(pte_t pte)
771{
772 return __pud(pte_val(pte));
773}
774
775static inline pmd_t pmd_mkhuge(pmd_t pmd)
776{
777 return pmd;
778}
779
780static inline pmd_t pmd_mkinvalid(pmd_t pmd)
781{
782 return __pmd(pmd_val(pmd) & ~(_PAGE_PRESENT|_PAGE_PROT_NONE));
783}
784
785#define __pmd_to_phys(pmd) (__page_val_to_pfn(pmd_val(pmd)) << PAGE_SHIFT)
786
787static inline unsigned long pmd_pfn(pmd_t pmd)
788{
789 return ((__pmd_to_phys(pmd) & PMD_MASK) >> PAGE_SHIFT);
790}
791
792#define __pud_to_phys(pud) (__page_val_to_pfn(pud_val(pud)) << PAGE_SHIFT)
793
794#define pud_pfn pud_pfn
795static inline unsigned long pud_pfn(pud_t pud)
796{
797 return ((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT);
798}
799
800#define pmd_pgprot pmd_pgprot
801static inline pgprot_t pmd_pgprot(pmd_t pmd)
802{
803 return pte_pgprot(pmd_pte(pmd));
804}
805
806#define pud_pgprot pud_pgprot
807static inline pgprot_t pud_pgprot(pud_t pud)
808{
809 return pte_pgprot(pud_pte(pud));
810}
811
812static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
813{
814 return pte_pmd(pte_modify(pmd_pte(pmd), newprot));
815}
816
817#define pmd_write pmd_write
818static inline int pmd_write(pmd_t pmd)
819{
820 return pte_write(pmd_pte(pmd));
821}
822
823#define pud_write pud_write
824static inline int pud_write(pud_t pud)
825{
826 return pte_write(pud_pte(pud));
827}
828
829#define pmd_dirty pmd_dirty
830static inline int pmd_dirty(pmd_t pmd)
831{
832 return pte_dirty(pmd_pte(pmd));
833}
834
835#define pmd_young pmd_young
836static inline int pmd_young(pmd_t pmd)
837{
838 return pte_young(pmd_pte(pmd));
839}
840
841static inline int pmd_user(pmd_t pmd)
842{
843 return pte_user(pmd_pte(pmd));
844}
845
846static inline pmd_t pmd_mkold(pmd_t pmd)
847{
848 return pte_pmd(pte_mkold(pmd_pte(pmd)));
849}
850
851static inline pmd_t pmd_mkyoung(pmd_t pmd)
852{
853 return pte_pmd(pte_mkyoung(pmd_pte(pmd)));
854}
855
856pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma);
857#define pmd_mkwrite pmd_mkwrite
858
859static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)
860{
861 return pte_pmd(pte_mkwrite_novma(pmd_pte(pmd)));
862}
863
864static inline pmd_t pmd_mkwrite_shstk(pmd_t pte)
865{
866 return __pmd((pmd_val(pte) & ~(_PAGE_LEAF)) | _PAGE_WRITE);
867}
868
869static inline pmd_t pmd_wrprotect(pmd_t pmd)
870{
871 return pte_pmd(pte_wrprotect(pmd_pte(pmd)));
872}
873
874static inline pmd_t pmd_mkclean(pmd_t pmd)
875{
876 return pte_pmd(pte_mkclean(pmd_pte(pmd)));
877}
878
879static inline pmd_t pmd_mkdirty(pmd_t pmd)
880{
881 return pte_pmd(pte_mkdirty(pmd_pte(pmd)));
882}
883
884#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP
885static inline bool pmd_special(pmd_t pmd)
886{
887 return pte_special(pmd_pte(pmd));
888}
889
890static inline pmd_t pmd_mkspecial(pmd_t pmd)
891{
892 return pte_pmd(pte_mkspecial(pmd_pte(pmd)));
893}
894#endif
895
896#ifdef CONFIG_ARCH_SUPPORTS_PUD_PFNMAP
897static inline bool pud_special(pud_t pud)
898{
899 return pte_special(pud_pte(pud));
900}
901
902static inline pud_t pud_mkspecial(pud_t pud)
903{
904 return pte_pud(pte_mkspecial(pud_pte(pud)));
905}
906#endif
907
908#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
909static inline bool pmd_uffd(pmd_t pmd)
910{
911 return pte_uffd(pmd_pte(pmd));
912}
913
914static inline pmd_t pmd_mkuffd(pmd_t pmd)
915{
916 return pte_pmd(pte_mkuffd(pmd_pte(pmd)));
917}
918
919static inline pmd_t pmd_clear_uffd(pmd_t pmd)
920{
921 return pte_pmd(pte_clear_uffd(pmd_pte(pmd)));
922}
923
924static inline bool pmd_swp_uffd(pmd_t pmd)
925{
926 return pte_swp_uffd(pmd_pte(pmd));
927}
928
929static inline pmd_t pmd_swp_mkuffd(pmd_t pmd)
930{
931 return pte_pmd(pte_swp_mkuffd(pmd_pte(pmd)));
932}
933
934static inline pmd_t pmd_swp_clear_uffd(pmd_t pmd)
935{
936 return pte_pmd(pte_swp_clear_uffd(pmd_pte(pmd)));
937}
938#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
939
940#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
941static inline bool pmd_soft_dirty(pmd_t pmd)
942{
943 return pte_soft_dirty(pmd_pte(pmd));
944}
945
946static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
947{
948 return pte_pmd(pte_mksoft_dirty(pmd_pte(pmd)));
949}
950
951static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
952{
953 return pte_pmd(pte_clear_soft_dirty(pmd_pte(pmd)));
954}
955
956#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
957static inline bool pmd_swp_soft_dirty(pmd_t pmd)
958{
959 return pte_swp_soft_dirty(pmd_pte(pmd));
960}
961
962static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)
963{
964 return pte_pmd(pte_swp_mksoft_dirty(pmd_pte(pmd)));
965}
966
967static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)
968{
969 return pte_pmd(pte_swp_clear_soft_dirty(pmd_pte(pmd)));
970}
971#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */
972#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */
973
974static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
975 pmd_t *pmdp, pmd_t pmd)
976{
977 page_table_check_pmd_set(mm, addr, pmdp, pmd);
978 return __set_pte_at(mm, (pte_t *)pmdp, pmd_pte(pmd));
979}
980
981static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
982 pud_t *pudp, pud_t pud)
983{
984 page_table_check_pud_set(mm, addr, pudp, pud);
985 return __set_pte_at(mm, (pte_t *)pudp, pud_pte(pud));
986}
987
988#ifdef CONFIG_PAGE_TABLE_CHECK
989static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
990{
991 return pte_present(pte) && pte_user(pte);
992}
993
994static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd)
995{
996 return pmd_leaf(pmd) && pmd_user(pmd);
997}
998
999static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud)
1000{
1001 return pud_leaf(pud) && pud_user(pud);
1002}
1003#endif
1004
1005#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1006static inline int pmd_trans_huge(pmd_t pmd)
1007{
1008 return pmd_leaf(pmd);
1009}
1010
1011#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
1012static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
1013 unsigned long address, pmd_t *pmdp,
1014 pmd_t entry, int dirty)
1015{
1016 return ptep_set_access_flags(vma, address, (pte_t *)pmdp, pmd_pte(entry), dirty);
1017}
1018
1019#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
1020static inline bool pmdp_test_and_clear_young(struct vm_area_struct *vma,
1021 unsigned long address, pmd_t *pmdp)
1022{
1023 return ptep_test_and_clear_young(vma, address, (pte_t *)pmdp);
1024}
1025
1026#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
1027static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
1028 unsigned long address, pmd_t *pmdp)
1029{
1030#ifdef CONFIG_SMP
1031 pmd_t pmd = __pmd(xchg(&pmdp->pmd, 0));
1032#else
1033 pmd_t pmd = *pmdp;
1034
1035 pmd_clear(pmdp);
1036#endif
1037
1038 page_table_check_pmd_clear(mm, address, pmd);
1039
1040 return pmd;
1041}
1042
1043#define __HAVE_ARCH_PMDP_SET_WRPROTECT
1044static inline void pmdp_set_wrprotect(struct mm_struct *mm,
1045 unsigned long address, pmd_t *pmdp)
1046{
1047 ptep_set_wrprotect(mm, address, (pte_t *)pmdp);
1048}
1049
1050#define pmdp_establish pmdp_establish
1051static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
1052 unsigned long address, pmd_t *pmdp, pmd_t pmd)
1053{
1054 page_table_check_pmd_set(vma->vm_mm, address, pmdp, pmd);
1055 return __pmd(atomic_long_xchg((atomic_long_t *)pmdp, pmd_val(pmd)));
1056}
1057
1058#define pmdp_collapse_flush pmdp_collapse_flush
1059extern pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
1060 unsigned long address, pmd_t *pmdp);
1061
1062static inline pud_t pud_wrprotect(pud_t pud)
1063{
1064 return pte_pud(pte_wrprotect(pud_pte(pud)));
1065}
1066
1067static inline int pud_trans_huge(pud_t pud)
1068{
1069 return pud_leaf(pud);
1070}
1071
1072static inline int pud_dirty(pud_t pud)
1073{
1074 return pte_dirty(pud_pte(pud));
1075}
1076
1077static inline pud_t pud_mkyoung(pud_t pud)
1078{
1079 return pte_pud(pte_mkyoung(pud_pte(pud)));
1080}
1081
1082static inline pud_t pud_mkold(pud_t pud)
1083{
1084 return pte_pud(pte_mkold(pud_pte(pud)));
1085}
1086
1087static inline pud_t pud_mkdirty(pud_t pud)
1088{
1089 return pte_pud(pte_mkdirty(pud_pte(pud)));
1090}
1091
1092static inline pud_t pud_mkclean(pud_t pud)
1093{
1094 return pte_pud(pte_mkclean(pud_pte(pud)));
1095}
1096
1097static inline pud_t pud_mkwrite(pud_t pud)
1098{
1099 return pte_pud(pte_mkwrite_novma(pud_pte(pud)));
1100}
1101
1102static inline pud_t pud_mkhuge(pud_t pud)
1103{
1104 return pud;
1105}
1106
1107static inline int pudp_set_access_flags(struct vm_area_struct *vma,
1108 unsigned long address, pud_t *pudp,
1109 pud_t entry, int dirty)
1110{
1111 return ptep_set_access_flags(vma, address, (pte_t *)pudp, pud_pte(entry), dirty);
1112}
1113
1114static inline bool pudp_test_and_clear_young(struct vm_area_struct *vma,
1115 unsigned long address, pud_t *pudp)
1116{
1117 return ptep_test_and_clear_young(vma, address, (pte_t *)pudp);
1118}
1119
1120#define __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR
1121static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm,
1122 unsigned long address, pud_t *pudp)
1123{
1124#ifdef CONFIG_SMP
1125 pud_t pud = __pud(xchg(&pudp->pud, 0));
1126#else
1127 pud_t pud = *pudp;
1128
1129 pud_clear(pudp);
1130#endif
1131
1132 page_table_check_pud_clear(mm, address, pud);
1133
1134 return pud;
1135}
1136
1137static inline int pud_young(pud_t pud)
1138{
1139 return pte_young(pud_pte(pud));
1140}
1141
1142static inline void update_mmu_cache_pud(struct vm_area_struct *vma,
1143 unsigned long address, pud_t *pudp)
1144{
1145 pte_t *ptep = (pte_t *)pudp;
1146
1147 update_mmu_cache(vma, address, ptep);
1148}
1149
1150static inline pud_t pudp_establish(struct vm_area_struct *vma,
1151 unsigned long address, pud_t *pudp, pud_t pud)
1152{
1153 page_table_check_pud_set(vma->vm_mm, address, pudp, pud);
1154 return __pud(atomic_long_xchg((atomic_long_t *)pudp, pud_val(pud)));
1155}
1156
1157static inline pud_t pud_mkinvalid(pud_t pud)
1158{
1159 return __pud(pud_val(pud) & ~(_PAGE_PRESENT | _PAGE_PROT_NONE));
1160}
1161
1162extern pud_t pudp_invalidate(struct vm_area_struct *vma, unsigned long address,
1163 pud_t *pudp);
1164
1165static inline pud_t pud_modify(pud_t pud, pgprot_t newprot)
1166{
1167 return pte_pud(pte_modify(pud_pte(pud), newprot));
1168}
1169
1170#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1171
1172/*
1173 * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that
1174 * are !pte_none() && !pte_present().
1175 *
1176 * Format of swap PTE:
1177 * bit 0: _PAGE_PRESENT (zero)
1178 * bit 1 to 2: (zero)
1179 * bit 3: _PAGE_SWP_SOFT_DIRTY
1180 * bit 4: _PAGE_SWP_UFFD
1181 * bit 5: _PAGE_PROT_NONE (zero)
1182 * bit 6: exclusive marker
1183 * bits 7 to 11: swap type
1184 * bits 12 to XLEN-1: swap offset
1185 */
1186#define __SWP_TYPE_SHIFT 7
1187#define __SWP_TYPE_BITS 5
1188#define __SWP_TYPE_MASK ((1UL << __SWP_TYPE_BITS) - 1)
1189#define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT)
1190
1191#define MAX_SWAPFILES_CHECK() \
1192 BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > __SWP_TYPE_BITS)
1193
1194#define __swp_type(x) (((x).val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK)
1195#define __swp_offset(x) ((x).val >> __SWP_OFFSET_SHIFT)
1196#define __swp_entry(type, offset) ((swp_entry_t) \
1197 { (((type) & __SWP_TYPE_MASK) << __SWP_TYPE_SHIFT) | \
1198 ((offset) << __SWP_OFFSET_SHIFT) })
1199
1200#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
1201#define __swp_entry_to_pte(x) ((pte_t) { (x).val })
1202
1203static inline bool pte_swp_exclusive(pte_t pte)
1204{
1205 return pte_val(pte) & _PAGE_SWP_EXCLUSIVE;
1206}
1207
1208static inline pte_t pte_swp_mkexclusive(pte_t pte)
1209{
1210 return __pte(pte_val(pte) | _PAGE_SWP_EXCLUSIVE);
1211}
1212
1213static inline pte_t pte_swp_clear_exclusive(pte_t pte)
1214{
1215 return __pte(pte_val(pte) & ~_PAGE_SWP_EXCLUSIVE);
1216}
1217
1218#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
1219#define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) })
1220#define __swp_entry_to_pmd(swp) __pmd((swp).val)
1221#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */
1222
1223/*
1224 * In the RV64 Linux scheme, we give the user half of the virtual-address space
1225 * and give the kernel the other (upper) half.
1226 */
1227#ifdef CONFIG_64BIT
1228#define KERN_VIRT_START (-(BIT(VA_BITS)) + TASK_SIZE)
1229#else
1230#define KERN_VIRT_START FIXADDR_START
1231#endif
1232
1233/*
1234 * Task size is 0x4000000000 for RV64 or 0x9fc00000 for RV32.
1235 * Note that PGDIR_SIZE must evenly divide TASK_SIZE.
1236 * Task size is:
1237 * - 0x9fc00000 (~2.5GB) for RV32.
1238 * - 0x4000000000 ( 256GB) for RV64 using SV39 mmu
1239 * - 0x800000000000 ( 128TB) for RV64 using SV48 mmu
1240 * - 0x100000000000000 ( 64PB) for RV64 using SV57 mmu
1241 *
1242 * Note that PGDIR_SIZE must evenly divide TASK_SIZE since "RISC-V
1243 * Instruction Set Manual Volume II: Privileged Architecture" states that
1244 * "load and store effective addresses, which are 64bits, must have bits
1245 * 63–48 all equal to bit 47, or else a page-fault exception will occur."
1246 * Similarly for SV57, bits 63–57 must be equal to bit 56.
1247 */
1248#ifdef CONFIG_64BIT
1249#define TASK_SIZE_64 (PGDIR_SIZE * PTRS_PER_PGD / 2)
1250
1251#ifdef CONFIG_COMPAT
1252#define TASK_SIZE_32 (_AC(0x80000000, UL) - PAGE_SIZE)
1253#define TASK_SIZE (is_compat_task() ? \
1254 TASK_SIZE_32 : TASK_SIZE_64)
1255#else
1256#define TASK_SIZE TASK_SIZE_64
1257#endif
1258
1259#else
1260#define TASK_SIZE FIXADDR_START
1261#endif
1262
1263#else /* CONFIG_MMU */
1264
1265#define PAGE_SHARED __pgprot(0)
1266#define PAGE_KERNEL __pgprot(0)
1267#define swapper_pg_dir NULL
1268#define TASK_SIZE _AC(-1, UL)
1269#define VMALLOC_START _AC(0, UL)
1270#define VMALLOC_END TASK_SIZE
1271
1272#endif /* !CONFIG_MMU */
1273
1274extern char _start[];
1275extern void *_dtb_early_va;
1276extern uintptr_t _dtb_early_pa;
1277#define dtb_early_va _dtb_early_va
1278#define dtb_early_pa _dtb_early_pa
1279extern u64 satp_mode;
1280
1281void paging_init(void);
1282void misc_mem_init(void);
1283
1284/*
1285 * Use set_p*_safe(), and elide TLB flushing, when confident that *no*
1286 * TLB flush will be required as a result of the "set". For example, use
1287 * in scenarios where it is known ahead of time that the routine is
1288 * setting non-present entries, or re-setting an existing entry to the
1289 * same value. Otherwise, use the typical "set" helpers and flush the
1290 * TLB.
1291 */
1292#define set_p4d_safe(p4dp, p4d) \
1293({ \
1294 WARN_ON_ONCE(p4d_present(*p4dp) && !p4d_same(*p4dp, p4d)); \
1295 set_p4d(p4dp, p4d); \
1296})
1297
1298#define set_pgd_safe(pgdp, pgd) \
1299({ \
1300 WARN_ON_ONCE(pgd_present(*pgdp) && !pgd_same(*pgdp, pgd)); \
1301 set_pgd(pgdp, pgd); \
1302})
1303#endif /* !__ASSEMBLER__ */
1304
1305#endif /* _ASM_RISCV_PGTABLE_H */