projects/linux·blob master · refs

439 lines · 11.1 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Page table allocation functions
4 *
5 * Copyright IBM Corp. 2016
6 * Author(s): Martin Schwidefsky <[email protected]>
7 */
8
9#include <linux/sysctl.h>
10#include <linux/slab.h>
11#include <linux/mm.h>
12#include <asm/mmu_context.h>
13#include <asm/page-states.h>
14#include <asm/pgalloc.h>
15#include <asm/tlbflush.h>
16
17unsigned long *crst_table_alloc_noprof(struct mm_struct *mm)
18{
19 gfp_t gfp = GFP_KERNEL_ACCOUNT;
20 struct ptdesc *ptdesc;
21 unsigned long *table;
22
23 if (mm == &init_mm)
24 gfp &= ~__GFP_ACCOUNT;
25 ptdesc = pagetable_alloc_noprof(gfp, CRST_ALLOC_ORDER);
26 if (!ptdesc)
27 return NULL;
28 table = ptdesc_address(ptdesc);
29 __arch_set_page_dat(table, 1UL << CRST_ALLOC_ORDER);
30 return table;
31}
32
33void crst_table_free(struct mm_struct *mm, unsigned long *table)
34{
35 if (!table)
36 return;
37 pagetable_free(virt_to_ptdesc(table));
38}
39
40static void __crst_table_upgrade(void *arg)
41{
42 struct mm_struct *mm = arg;
43 struct ctlreg asce;
44
45 /* change all active ASCEs to avoid the creation of new TLBs */
46 if (current->active_mm == mm) {
47 asce.val = mm->context.asce;
48 get_lowcore()->user_asce = asce;
49 local_ctl_load(7, &asce);
50 if (!test_thread_flag(TIF_ASCE_PRIMARY))
51 local_ctl_load(1, &asce);
52 }
53 __tlb_flush_local();
54}
55
56int crst_table_upgrade(struct mm_struct *mm, unsigned long end)
57{
58 unsigned long *table, *pgd;
59 int rc, notify;
60
61 mmap_assert_write_locked(mm);
62 /* upgrade should only happen from 3 to 4, 3 to 5, or 4 to 5 levels */
63 VM_BUG_ON(mm->context.asce_limit < _REGION2_SIZE);
64 rc = 0;
65 notify = 0;
66 while (mm->context.asce_limit < end) {
67 table = crst_table_alloc(mm);
68 if (!table) {
69 rc = -ENOMEM;
70 break;
71 }
72 spin_lock_bh(&mm->page_table_lock);
73 pgd = (unsigned long *)mm->pgd;
74 if (mm->context.asce_limit == _REGION2_SIZE) {
75 crst_table_init(table, _REGION2_ENTRY_EMPTY);
76 p4d_populate(mm, (p4d_t *)table, (pud_t *)pgd);
77 pagetable_p4d_ctor(virt_to_ptdesc(table));
78 mm->pgd = (pgd_t *)table;
79 mm->context.asce_limit = _REGION1_SIZE;
80 mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH |
81 _ASCE_USER_BITS | _ASCE_TYPE_REGION2;
82 mm_inc_nr_puds(mm);
83 } else {
84 crst_table_init(table, _REGION1_ENTRY_EMPTY);
85 pgd_populate(mm, (pgd_t *)table, (p4d_t *)pgd);
86 pagetable_pgd_ctor(virt_to_ptdesc(table));
87 mm->pgd = (pgd_t *)table;
88 mm->context.asce_limit = TASK_SIZE_MAX;
89 mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH |
90 _ASCE_USER_BITS | _ASCE_TYPE_REGION1;
91 }
92 notify = 1;
93 spin_unlock_bh(&mm->page_table_lock);
94 }
95 if (notify)
96 on_each_cpu(__crst_table_upgrade, mm, 0);
97 return rc;
98}
99
100unsigned long *page_table_alloc_noprof(struct mm_struct *mm)
101{
102 gfp_t gfp = GFP_KERNEL_ACCOUNT;
103 struct ptdesc *ptdesc;
104 unsigned long *table;
105
106 if (mm == &init_mm)
107 gfp &= ~__GFP_ACCOUNT;
108 ptdesc = pagetable_alloc_noprof(gfp, 0);
109 if (!ptdesc)
110 return NULL;
111 if (!pagetable_pte_ctor(mm, ptdesc)) {
112 pagetable_free(ptdesc);
113 return NULL;
114 }
115 table = ptdesc_address(ptdesc);
116 __arch_set_page_dat(table, 1);
117 memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE);
118 memset64((u64 *)table + PTRS_PER_PTE, 0, PTRS_PER_PTE);
119 return table;
120}
121
122void page_table_free(struct mm_struct *mm, unsigned long *table)
123{
124 struct ptdesc *ptdesc = virt_to_ptdesc(table);
125
126 if (pagetable_is_reserved(ptdesc))
127 return free_reserved_ptdesc(ptdesc);
128 pagetable_dtor_free(ptdesc);
129}
130
131#ifdef CONFIG_TRANSPARENT_HUGEPAGE
132static void pte_free_now(struct rcu_head *head)
133{
134 struct ptdesc *ptdesc = container_of(head, struct ptdesc, pt_rcu_head);
135
136 pagetable_dtor_free(ptdesc);
137}
138
139void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable)
140{
141 struct ptdesc *ptdesc = virt_to_ptdesc(pgtable);
142
143 call_rcu(&ptdesc->pt_rcu_head, pte_free_now);
144}
145#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
146
147/*
148 * Base infrastructure required to generate basic asces, region, segment,
149 * and page tables that do not make use of enhanced features like EDAT1.
150 */
151
152static struct kmem_cache *base_pgt_cache;
153
154static unsigned long *base_pgt_alloc(void)
155{
156 unsigned long *table;
157
158 table = kmem_cache_alloc(base_pgt_cache, GFP_KERNEL);
159 if (table)
160 memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE);
161 return table;
162}
163
164static void base_pgt_free(unsigned long *table)
165{
166 kmem_cache_free(base_pgt_cache, table);
167}
168
169static unsigned long *base_crst_alloc(unsigned long val)
170{
171 unsigned long *table;
172 struct ptdesc *ptdesc;
173
174 ptdesc = pagetable_alloc(GFP_KERNEL, CRST_ALLOC_ORDER);
175 if (!ptdesc)
176 return NULL;
177 table = ptdesc_address(ptdesc);
178 crst_table_init(table, val);
179 return table;
180}
181
182static void base_crst_free(unsigned long *table)
183{
184 if (!table)
185 return;
186 pagetable_free(virt_to_ptdesc(table));
187}
188
189#define BASE_ADDR_END_FUNC(NAME, SIZE) \
190static inline unsigned long base_##NAME##_addr_end(unsigned long addr, \
191 unsigned long end) \
192{ \
193 unsigned long next = (addr + (SIZE)) & ~((SIZE) - 1); \
194 \
195 return (next - 1) < (end - 1) ? next : end; \
196}
197
198BASE_ADDR_END_FUNC(page, PAGE_SIZE)
199BASE_ADDR_END_FUNC(segment, _SEGMENT_SIZE)
200BASE_ADDR_END_FUNC(region3, _REGION3_SIZE)
201BASE_ADDR_END_FUNC(region2, _REGION2_SIZE)
202BASE_ADDR_END_FUNC(region1, _REGION1_SIZE)
203
204static inline unsigned long base_lra(unsigned long address)
205{
206 unsigned long real;
207
208 asm volatile(
209 " lra %0,0(%1)"
210 : "=d" (real) : "a" (address) : "cc");
211 return real;
212}
213
214static int base_page_walk(unsigned long *origin, unsigned long addr,
215 unsigned long end, int alloc)
216{
217 unsigned long *pte, next;
218
219 if (!alloc)
220 return 0;
221 pte = origin;
222 pte += (addr & _PAGE_INDEX) >> PAGE_SHIFT;
223 do {
224 next = base_page_addr_end(addr, end);
225 *pte = base_lra(addr);
226 } while (pte++, addr = next, addr < end);
227 return 0;
228}
229
230static int base_segment_walk(unsigned long *origin, unsigned long addr,
231 unsigned long end, int alloc)
232{
233 unsigned long *ste, next, *table;
234 int rc;
235
236 ste = origin;
237 ste += (addr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT;
238 do {
239 next = base_segment_addr_end(addr, end);
240 if (*ste & _SEGMENT_ENTRY_INVALID) {
241 if (!alloc)
242 continue;
243 table = base_pgt_alloc();
244 if (!table)
245 return -ENOMEM;
246 *ste = __pa(table) | _SEGMENT_ENTRY;
247 }
248 table = __va(*ste & _SEGMENT_ENTRY_ORIGIN);
249 rc = base_page_walk(table, addr, next, alloc);
250 if (rc)
251 return rc;
252 if (!alloc)
253 base_pgt_free(table);
254 } while (ste++, addr = next, addr < end);
255 return 0;
256}
257
258static int base_region3_walk(unsigned long *origin, unsigned long addr,
259 unsigned long end, int alloc)
260{
261 unsigned long *rtte, next, *table;
262 int rc;
263
264 rtte = origin;
265 rtte += (addr & _REGION3_INDEX) >> _REGION3_SHIFT;
266 do {
267 next = base_region3_addr_end(addr, end);
268 if (*rtte & _REGION_ENTRY_INVALID) {
269 if (!alloc)
270 continue;
271 table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY);
272 if (!table)
273 return -ENOMEM;
274 *rtte = __pa(table) | _REGION3_ENTRY;
275 }
276 table = __va(*rtte & _REGION_ENTRY_ORIGIN);
277 rc = base_segment_walk(table, addr, next, alloc);
278 if (rc)
279 return rc;
280 if (!alloc)
281 base_crst_free(table);
282 } while (rtte++, addr = next, addr < end);
283 return 0;
284}
285
286static int base_region2_walk(unsigned long *origin, unsigned long addr,
287 unsigned long end, int alloc)
288{
289 unsigned long *rste, next, *table;
290 int rc;
291
292 rste = origin;
293 rste += (addr & _REGION2_INDEX) >> _REGION2_SHIFT;
294 do {
295 next = base_region2_addr_end(addr, end);
296 if (*rste & _REGION_ENTRY_INVALID) {
297 if (!alloc)
298 continue;
299 table = base_crst_alloc(_REGION3_ENTRY_EMPTY);
300 if (!table)
301 return -ENOMEM;
302 *rste = __pa(table) | _REGION2_ENTRY;
303 }
304 table = __va(*rste & _REGION_ENTRY_ORIGIN);
305 rc = base_region3_walk(table, addr, next, alloc);
306 if (rc)
307 return rc;
308 if (!alloc)
309 base_crst_free(table);
310 } while (rste++, addr = next, addr < end);
311 return 0;
312}
313
314static int base_region1_walk(unsigned long *origin, unsigned long addr,
315 unsigned long end, int alloc)
316{
317 unsigned long *rfte, next, *table;
318 int rc;
319
320 rfte = origin;
321 rfte += (addr & _REGION1_INDEX) >> _REGION1_SHIFT;
322 do {
323 next = base_region1_addr_end(addr, end);
324 if (*rfte & _REGION_ENTRY_INVALID) {
325 if (!alloc)
326 continue;
327 table = base_crst_alloc(_REGION2_ENTRY_EMPTY);
328 if (!table)
329 return -ENOMEM;
330 *rfte = __pa(table) | _REGION1_ENTRY;
331 }
332 table = __va(*rfte & _REGION_ENTRY_ORIGIN);
333 rc = base_region2_walk(table, addr, next, alloc);
334 if (rc)
335 return rc;
336 if (!alloc)
337 base_crst_free(table);
338 } while (rfte++, addr = next, addr < end);
339 return 0;
340}
341
342/**
343 * base_asce_free - free asce and tables returned from base_asce_alloc()
344 * @asce: asce to be freed
345 *
346 * Frees all region, segment, and page tables that were allocated with a
347 * corresponding base_asce_alloc() call.
348 */
349void base_asce_free(unsigned long asce)
350{
351 unsigned long *table = __va(asce & _ASCE_ORIGIN);
352
353 if (!asce)
354 return;
355 switch (asce & _ASCE_TYPE_MASK) {
356 case _ASCE_TYPE_SEGMENT:
357 base_segment_walk(table, 0, _REGION3_SIZE, 0);
358 break;
359 case _ASCE_TYPE_REGION3:
360 base_region3_walk(table, 0, _REGION2_SIZE, 0);
361 break;
362 case _ASCE_TYPE_REGION2:
363 base_region2_walk(table, 0, _REGION1_SIZE, 0);
364 break;
365 case _ASCE_TYPE_REGION1:
366 base_region1_walk(table, 0, TASK_SIZE_MAX, 0);
367 break;
368 }
369 base_crst_free(table);
370}
371
372static int base_pgt_cache_init(void)
373{
374 static DEFINE_MUTEX(base_pgt_cache_mutex);
375 unsigned long sz = _PAGE_TABLE_SIZE;
376
377 if (base_pgt_cache)
378 return 0;
379 mutex_lock(&base_pgt_cache_mutex);
380 if (!base_pgt_cache)
381 base_pgt_cache = kmem_cache_create("base_pgt", sz, sz, 0, NULL);
382 mutex_unlock(&base_pgt_cache_mutex);
383 return base_pgt_cache ? 0 : -ENOMEM;
384}
385
386/**
387 * base_asce_alloc - create kernel mapping without enhanced DAT features
388 * @addr: virtual start address of kernel mapping
389 * @num_pages: number of consecutive pages
390 *
391 * Generate an asce, including all required region, segment and page tables,
392 * that can be used to access the virtual kernel mapping. The difference is
393 * that the returned asce does not make use of any enhanced DAT features like
394 * e.g. large pages. This is required for some I/O functions that pass an
395 * asce, like e.g. some service call requests.
396 *
397 * Note: the returned asce may NEVER be attached to any cpu. It may only be
398 * used for I/O requests. tlb entries that might result because the
399 * asce was attached to a cpu won't be cleared.
400 */
401unsigned long base_asce_alloc(unsigned long addr, unsigned long num_pages)
402{
403 unsigned long asce, *table, end;
404 int rc;
405
406 if (base_pgt_cache_init())
407 return 0;
408 end = addr + num_pages * PAGE_SIZE;
409 if (end <= _REGION3_SIZE) {
410 table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY);
411 if (!table)
412 return 0;
413 rc = base_segment_walk(table, addr, end, 1);
414 asce = __pa(table) | _ASCE_TYPE_SEGMENT | _ASCE_TABLE_LENGTH;
415 } else if (end <= _REGION2_SIZE) {
416 table = base_crst_alloc(_REGION3_ENTRY_EMPTY);
417 if (!table)
418 return 0;
419 rc = base_region3_walk(table, addr, end, 1);
420 asce = __pa(table) | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH;
421 } else if (end <= _REGION1_SIZE) {
422 table = base_crst_alloc(_REGION2_ENTRY_EMPTY);
423 if (!table)
424 return 0;
425 rc = base_region2_walk(table, addr, end, 1);
426 asce = __pa(table) | _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH;
427 } else {
428 table = base_crst_alloc(_REGION1_ENTRY_EMPTY);
429 if (!table)
430 return 0;
431 rc = base_region1_walk(table, addr, end, 1);
432 asce = __pa(table) | _ASCE_TYPE_REGION1 | _ASCE_TABLE_LENGTH;
433 }
434 if (rc) {
435 base_asce_free(asce);
436 asce = 0;
437 }
438 return asce;
439}