projects/linux·blob master · refs

294 lines · 7.3 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3/*
4 * Copyright (c) 2021, Google LLC.
5 * Pasha Tatashin <[email protected]>
6 */
7#include <linux/kstrtox.h>
8#include <linux/mm.h>
9#include <linux/page_table_check.h>
10#include <linux/swap.h>
11#include <linux/leafops.h>
12
13#undef pr_fmt
14#define pr_fmt(fmt) "page_table_check: " fmt
15
16struct page_table_check {
17 atomic_t anon_map_count;
18 atomic_t file_map_count;
19};
20
21static bool __page_table_check_enabled __initdata =
22 IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED);
23
24DEFINE_STATIC_KEY_TRUE(page_table_check_disabled);
25EXPORT_SYMBOL(page_table_check_disabled);
26
27static int __init early_page_table_check_param(char *buf)
28{
29 return kstrtobool(buf, &__page_table_check_enabled);
30}
31
32early_param("page_table_check", early_page_table_check_param);
33
34static bool __init need_page_table_check(void)
35{
36 return __page_table_check_enabled;
37}
38
39static void __init init_page_table_check(void)
40{
41 if (!__page_table_check_enabled)
42 return;
43 static_branch_disable(&page_table_check_disabled);
44}
45
46struct page_ext_operations page_table_check_ops = {
47 .size = sizeof(struct page_table_check),
48 .need = need_page_table_check,
49 .init = init_page_table_check,
50 .need_shared_flags = false,
51};
52
53static struct page_table_check *get_page_table_check(struct page_ext *page_ext)
54{
55 BUG_ON(!page_ext);
56 return page_ext_data(page_ext, &page_table_check_ops);
57}
58
59/*
60 * An entry is removed from the page table, decrement the counters for that page
61 * verify that it is of correct type and counters do not become negative.
62 */
63static void page_table_check_clear(unsigned long pfn, unsigned long pgcnt)
64{
65 struct page_ext_iter iter;
66 struct page_ext *page_ext;
67 struct page *page;
68 bool anon;
69
70 if (!pfn_valid(pfn))
71 return;
72
73 page = pfn_to_page(pfn);
74 BUG_ON(PageSlab(page));
75 anon = PageAnon(page);
76
77 rcu_read_lock();
78 for_each_page_ext(page, pgcnt, page_ext, iter) {
79 struct page_table_check *ptc = get_page_table_check(page_ext);
80
81 if (anon) {
82 BUG_ON(atomic_read(&ptc->file_map_count));
83 BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0);
84 } else {
85 BUG_ON(atomic_read(&ptc->anon_map_count));
86 BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0);
87 }
88 }
89 rcu_read_unlock();
90}
91
92/*
93 * A new entry is added to the page table, increment the counters for that page
94 * verify that it is of correct type and is not being mapped with a different
95 * type to a different process.
96 */
97static void page_table_check_set(unsigned long pfn, unsigned long pgcnt,
98 bool rw)
99{
100 struct page_ext_iter iter;
101 struct page_ext *page_ext;
102 struct page *page;
103 bool anon;
104
105 if (!pfn_valid(pfn))
106 return;
107
108 page = pfn_to_page(pfn);
109 BUG_ON(PageSlab(page));
110 anon = PageAnon(page);
111
112 rcu_read_lock();
113 for_each_page_ext(page, pgcnt, page_ext, iter) {
114 struct page_table_check *ptc = get_page_table_check(page_ext);
115
116 if (anon) {
117 BUG_ON(atomic_read(&ptc->file_map_count));
118 BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw);
119 } else {
120 BUG_ON(atomic_read(&ptc->anon_map_count));
121 BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0);
122 }
123 }
124 rcu_read_unlock();
125}
126
127/*
128 * page is on free list, or is being allocated, verify that counters are zeroes
129 * crash if they are not.
130 */
131void __page_table_check_zero(struct page *page, unsigned int order)
132{
133 struct page_ext_iter iter;
134 struct page_ext *page_ext;
135
136 BUG_ON(PageSlab(page));
137
138 rcu_read_lock();
139 for_each_page_ext(page, 1 << order, page_ext, iter) {
140 struct page_table_check *ptc = get_page_table_check(page_ext);
141
142 BUG_ON(atomic_read(&ptc->anon_map_count));
143 BUG_ON(atomic_read(&ptc->file_map_count));
144 }
145 rcu_read_unlock();
146}
147
148void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr,
149 pte_t pte)
150{
151 if (&init_mm == mm)
152 return;
153
154 if (pte_user_accessible_page(mm, addr, pte) && !pte_special(pte))
155 page_table_check_clear(pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT);
156}
157EXPORT_SYMBOL(__page_table_check_pte_clear);
158
159static inline bool page_table_check_huge_zero_pmd(pmd_t pmd)
160{
161 unsigned long pfn = pmd_pfn(pmd);
162
163 if (!pfn_valid(pfn))
164 return false;
165
166 return is_huge_zero_folio(page_folio(pfn_to_page(pfn)));
167}
168
169void __page_table_check_pmd_clear(struct mm_struct *mm, unsigned long addr,
170 pmd_t pmd)
171{
172 if (&init_mm == mm)
173 return;
174
175 if (pmd_user_accessible_page(mm, addr, pmd) &&
176 !page_table_check_huge_zero_pmd(pmd))
177 page_table_check_clear(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT);
178}
179EXPORT_SYMBOL(__page_table_check_pmd_clear);
180
181void __page_table_check_pud_clear(struct mm_struct *mm, unsigned long addr,
182 pud_t pud)
183{
184 if (&init_mm == mm)
185 return;
186
187 if (pud_user_accessible_page(mm, addr, pud))
188 page_table_check_clear(pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT);
189}
190EXPORT_SYMBOL(__page_table_check_pud_clear);
191
192/* Whether the swap entry cached writable information */
193static inline bool softleaf_cached_writable(softleaf_t entry)
194{
195 return softleaf_is_device_private_write(entry) ||
196 softleaf_is_migration_write(entry);
197}
198
199static void page_table_check_pte_flags(pte_t pte)
200{
201 if (pte_present(pte)) {
202 WARN_ON_ONCE(pte_uffd(pte) && pte_write(pte));
203 } else if (pte_swp_uffd(pte)) {
204 const softleaf_t entry = softleaf_from_pte(pte);
205
206 WARN_ON_ONCE(softleaf_cached_writable(entry));
207 }
208}
209
210void __page_table_check_ptes_set(struct mm_struct *mm, unsigned long addr,
211 pte_t *ptep, pte_t pte, unsigned int nr)
212{
213 unsigned int i;
214
215 if (&init_mm == mm)
216 return;
217
218 page_table_check_pte_flags(pte);
219
220 for (i = 0; i < nr; i++)
221 __page_table_check_pte_clear(mm, addr + PAGE_SIZE * i, ptep_get(ptep + i));
222 if (pte_user_accessible_page(mm, addr, pte) && !pte_special(pte))
223 page_table_check_set(pte_pfn(pte), nr, pte_write(pte));
224}
225EXPORT_SYMBOL(__page_table_check_ptes_set);
226
227static inline void page_table_check_pmd_flags(pmd_t pmd)
228{
229 if (pmd_present(pmd)) {
230 if (pmd_uffd(pmd))
231 WARN_ON_ONCE(pmd_write(pmd));
232 } else if (pmd_swp_uffd(pmd)) {
233 const softleaf_t entry = softleaf_from_pmd(pmd);
234
235 WARN_ON_ONCE(softleaf_cached_writable(entry));
236 }
237}
238
239void __page_table_check_pmds_set(struct mm_struct *mm, unsigned long addr,
240 pmd_t *pmdp, pmd_t pmd, unsigned int nr)
241{
242 unsigned long stride = PMD_SIZE >> PAGE_SHIFT;
243 unsigned int i;
244
245 if (&init_mm == mm)
246 return;
247
248 page_table_check_pmd_flags(pmd);
249
250 for (i = 0; i < nr; i++)
251 __page_table_check_pmd_clear(mm, addr + PMD_SIZE * i, *(pmdp + i));
252 if (pmd_user_accessible_page(mm, addr, pmd) &&
253 !page_table_check_huge_zero_pmd(pmd))
254 page_table_check_set(pmd_pfn(pmd), stride * nr, pmd_write(pmd));
255}
256EXPORT_SYMBOL(__page_table_check_pmds_set);
257
258void __page_table_check_puds_set(struct mm_struct *mm, unsigned long addr,
259 pud_t *pudp, pud_t pud, unsigned int nr)
260{
261 unsigned long stride = PUD_SIZE >> PAGE_SHIFT;
262 unsigned int i;
263
264 if (&init_mm == mm)
265 return;
266
267 for (i = 0; i < nr; i++)
268 __page_table_check_pud_clear(mm, addr + PUD_SIZE * i, *(pudp + i));
269 if (pud_user_accessible_page(mm, addr, pud))
270 page_table_check_set(pud_pfn(pud), stride * nr, pud_write(pud));
271}
272EXPORT_SYMBOL(__page_table_check_puds_set);
273
274void __page_table_check_pte_clear_range(struct mm_struct *mm,
275 unsigned long addr,
276 pmd_t pmd)
277{
278 if (&init_mm == mm)
279 return;
280
281 if (!pmd_bad(pmd) && !pmd_leaf(pmd)) {
282 pte_t *ptep = pte_offset_map(&pmd, addr);
283 unsigned long i;
284
285 if (WARN_ON(!ptep))
286 return;
287 for (i = 0; i < PTRS_PER_PTE; i++) {
288 __page_table_check_pte_clear(mm, addr, ptep_get(ptep));
289 addr += PAGE_SIZE;
290 ptep++;
291 }
292 pte_unmap(ptep - PTRS_PER_PTE);
293 }
294}