projects/linux·blob master · refs

129 lines · 3.9 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2
3#include <linux/gfp.h>
4#include <linux/init.h>
5#include <linux/mm.h>
6#include <linux/time_namespace.h>
7#include <linux/types.h>
8#include <linux/vdso_datastore.h>
9#include <vdso/datapage.h>
10
11static u8 vdso_initdata[VDSO_NR_PAGES * PAGE_SIZE] __aligned(PAGE_SIZE) __initdata = {};
12
13#ifdef CONFIG_GENERIC_GETTIMEOFDAY
14struct vdso_time_data *vdso_k_time_data __ro_after_init =
15 (void *)&vdso_initdata[VDSO_TIME_PAGE_OFFSET * PAGE_SIZE];
16
17static_assert(sizeof(struct vdso_time_data) <= PAGE_SIZE);
18#endif /* CONFIG_GENERIC_GETTIMEOFDAY */
19
20#ifdef CONFIG_VDSO_GETRANDOM
21struct vdso_rng_data *vdso_k_rng_data __ro_after_init =
22 (void *)&vdso_initdata[VDSO_RNG_PAGE_OFFSET * PAGE_SIZE];
23
24static_assert(sizeof(struct vdso_rng_data) <= PAGE_SIZE);
25#endif /* CONFIG_VDSO_GETRANDOM */
26
27#ifdef CONFIG_ARCH_HAS_VDSO_ARCH_DATA
28struct vdso_arch_data *vdso_k_arch_data __ro_after_init =
29 (void *)&vdso_initdata[VDSO_ARCH_PAGES_START * PAGE_SIZE];
30#endif /* CONFIG_ARCH_HAS_VDSO_ARCH_DATA */
31
32static struct page *vdso_data_pages __ro_after_init;
33
34void __init vdso_setup_data_pages(void)
35{
36 unsigned int order = get_order(VDSO_NR_PAGES * PAGE_SIZE);
37
38 /*
39 * Allocate the data pages dynamically. SPARC does not support mapping
40 * static pages to be mapped into userspace.
41 * It is also a requirement for mlockall() support.
42 *
43 * Do not use folios. In time namespaces the pages are mapped in a different order
44 * to userspace, which is not handled by the folio optimizations in finish_fault().
45 */
46 vdso_data_pages = alloc_pages(GFP_KERNEL, order);
47 if (!vdso_data_pages)
48 panic("Unable to allocate VDSO storage pages");
49
50 /* The pages are mapped one-by-one into userspace and each one needs to be refcounted. */
51 split_page(vdso_data_pages, order);
52
53 /* Move the data already written by other subsystems to the new pages */
54 memcpy(page_address(vdso_data_pages), vdso_initdata, VDSO_NR_PAGES * PAGE_SIZE);
55
56 if (IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY))
57 vdso_k_time_data = page_address(vdso_data_pages + VDSO_TIME_PAGE_OFFSET);
58
59 if (IS_ENABLED(CONFIG_VDSO_GETRANDOM))
60 vdso_k_rng_data = page_address(vdso_data_pages + VDSO_RNG_PAGE_OFFSET);
61
62 if (IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA))
63 vdso_k_arch_data = page_address(vdso_data_pages + VDSO_ARCH_PAGES_START);
64}
65
66static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
67 struct vm_area_struct *vma, struct vm_fault *vmf)
68{
69 struct page *page, *timens_page;
70
71 if (unlikely(vmf->flags & FAULT_FLAG_REMOTE))
72 return VM_FAULT_SIGBUS;
73
74 page = vdso_data_pages + vmf->pgoff;
75 timens_page = find_timens_vvar_page(vma);
76
77 switch (vmf->pgoff) {
78 case VDSO_TIME_PAGE_OFFSET:
79 if (!IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY) || !timens_page)
80 break;
81 /*
82 * Fault in VVAR page too, since it will be accessed
83 * to get clock data anyway.
84 */
85 unsigned long addr;
86 vm_fault_t err;
87
88 addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE;
89 err = vmf_insert_page(vma, addr, page);
90 if (unlikely(err & VM_FAULT_ERROR))
91 return err;
92 page = timens_page;
93 break;
94 case VDSO_TIMENS_PAGE_OFFSET:
95 /*
96 * If a task belongs to a time namespace then a namespace
97 * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and
98 * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET
99 * offset.
100 * See also the comment near timens_setup_vdso_data().
101 */
102 if (!IS_ENABLED(CONFIG_TIME_NS) || !timens_page)
103 break;
104 page = vdso_data_pages + VDSO_TIME_PAGE_OFFSET;
105 break;
106 case VDSO_RNG_PAGE_OFFSET:
107 case VDSO_ARCH_PAGES_START ... VDSO_ARCH_PAGES_END:
108 break;
109 default:
110 return VM_FAULT_SIGBUS;
111 }
112
113 get_page(page);
114 vmf->page = page;
115 return 0;
116}
117
118const struct vm_special_mapping vdso_vvar_mapping = {
119 .name = "[vvar]",
120 .fault = vvar_fault,
121};
122
123struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr)
124{
125 return _install_special_mapping(mm, addr, VDSO_NR_PAGES * PAGE_SIZE,
126 VM_READ | VM_MAYREAD | VM_DONTDUMP |
127 VM_MIXEDMAP | VM_SEALED_SYSMAP,
128 &vdso_vvar_mapping);
129}