projects/linux·blob master · refs

236 lines · 6.1 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Access kernel or user memory without faulting.
4 */
5#include <linux/export.h>
6#include <linux/mm.h>
7#include <linux/uaccess.h>
8#include <asm/tlb.h>
9
10bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src,
11 size_t size)
12{
13 return true;
14}
15
16/*
17 * The below only uses kmsan_check_memory() to ensure uninitialized kernel
18 * memory isn't leaked.
19 */
20#define copy_from_kernel_nofault_loop(dst, src, len, type, err_label) \
21 while (len >= sizeof(type)) { \
22 __get_kernel_nofault(dst, src, type, err_label); \
23 kmsan_check_memory(src, sizeof(type)); \
24 dst += sizeof(type); \
25 src += sizeof(type); \
26 len -= sizeof(type); \
27 }
28
29long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
30{
31 unsigned long align = 0;
32
33 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
34 align = (unsigned long)dst | (unsigned long)src;
35
36 if (!copy_from_kernel_nofault_allowed(src, size))
37 return -ERANGE;
38
39 pagefault_disable();
40 if (!(align & 7))
41 copy_from_kernel_nofault_loop(dst, src, size, u64, Efault);
42 if (!(align & 3))
43 copy_from_kernel_nofault_loop(dst, src, size, u32, Efault);
44 if (!(align & 1))
45 copy_from_kernel_nofault_loop(dst, src, size, u16, Efault);
46 copy_from_kernel_nofault_loop(dst, src, size, u8, Efault);
47 pagefault_enable();
48 return 0;
49Efault:
50 pagefault_enable();
51 return -EFAULT;
52}
53EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
54
55#define copy_to_kernel_nofault_loop(dst, src, len, type, err_label) \
56 while (len >= sizeof(type)) { \
57 __put_kernel_nofault(dst, src, type, err_label); \
58 instrument_write(dst, sizeof(type)); \
59 dst += sizeof(type); \
60 src += sizeof(type); \
61 len -= sizeof(type); \
62 }
63
64long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
65{
66 unsigned long align = 0;
67
68 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
69 align = (unsigned long)dst | (unsigned long)src;
70
71 pagefault_disable();
72 if (!(align & 7))
73 copy_to_kernel_nofault_loop(dst, src, size, u64, Efault);
74 if (!(align & 3))
75 copy_to_kernel_nofault_loop(dst, src, size, u32, Efault);
76 if (!(align & 1))
77 copy_to_kernel_nofault_loop(dst, src, size, u16, Efault);
78 copy_to_kernel_nofault_loop(dst, src, size, u8, Efault);
79 pagefault_enable();
80 return 0;
81Efault:
82 pagefault_enable();
83 return -EFAULT;
84}
85
86long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
87{
88 const void *src = unsafe_addr;
89
90 if (unlikely(count <= 0))
91 return 0;
92 if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
93 return -ERANGE;
94
95 pagefault_disable();
96 do {
97 __get_kernel_nofault(dst, src, u8, Efault);
98 dst++;
99 src++;
100 } while (dst[-1] && src - unsafe_addr < count);
101 pagefault_enable();
102
103 dst[-1] = '\0';
104 return src - unsafe_addr;
105Efault:
106 pagefault_enable();
107 dst[0] = '\0';
108 return -EFAULT;
109}
110
111/**
112 * copy_from_user_nofault(): safely attempt to read from a user-space location
113 * @dst: pointer to the buffer that shall take the data
114 * @src: address to read from. This must be a user address.
115 * @size: size of the data chunk
116 *
117 * Safely read from user address @src to the buffer at @dst. If a kernel fault
118 * happens, handle that and return -EFAULT.
119 */
120long copy_from_user_nofault(void *dst, const void __user *src, size_t size)
121{
122 long ret = -EFAULT;
123
124 if (!__access_ok(src, size))
125 return ret;
126
127 if (!nmi_uaccess_okay())
128 return ret;
129
130 pagefault_disable();
131 ret = __copy_from_user_inatomic(dst, src, size);
132 pagefault_enable();
133
134 if (ret)
135 return -EFAULT;
136 return 0;
137}
138EXPORT_SYMBOL_GPL(copy_from_user_nofault);
139
140/**
141 * copy_to_user_nofault(): safely attempt to write to a user-space location
142 * @dst: address to write to
143 * @src: pointer to the data that shall be written
144 * @size: size of the data chunk
145 *
146 * Safely write to address @dst from the buffer at @src. If a kernel fault
147 * happens, handle that and return -EFAULT.
148 */
149long copy_to_user_nofault(void __user *dst, const void *src, size_t size)
150{
151 long ret = -EFAULT;
152
153 if (access_ok(dst, size)) {
154 pagefault_disable();
155 ret = __copy_to_user_inatomic(dst, src, size);
156 pagefault_enable();
157 }
158
159 if (ret)
160 return -EFAULT;
161 return 0;
162}
163EXPORT_SYMBOL_GPL(copy_to_user_nofault);
164
165/**
166 * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user
167 * address.
168 * @dst: Destination address, in kernel space. This buffer must be at
169 * least @count bytes long.
170 * @unsafe_addr: Unsafe user address.
171 * @count: Maximum number of bytes to copy, including the trailing NUL.
172 *
173 * Copies a NUL-terminated string from unsafe user address to kernel buffer.
174 *
175 * On success, returns the length of the string INCLUDING the trailing NUL.
176 *
177 * If access fails, returns -EFAULT (some data may have been copied
178 * and the trailing NUL added).
179 *
180 * If @count is smaller than the length of the string, copies @count-1 bytes,
181 * sets the last byte of @dst buffer to NUL and returns @count.
182 */
183long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr,
184 long count)
185{
186 long ret;
187
188 if (unlikely(count <= 0))
189 return 0;
190
191 pagefault_disable();
192 ret = strncpy_from_user(dst, unsafe_addr, count);
193 pagefault_enable();
194
195 if (ret >= count) {
196 ret = count;
197 dst[ret - 1] = '\0';
198 } else if (ret >= 0) {
199 ret++;
200 }
201
202 return ret;
203}
204
205/**
206 * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL.
207 * @unsafe_addr: The string to measure.
208 * @count: Maximum count (including NUL)
209 *
210 * Get the size of a NUL-terminated string in user space without pagefault.
211 *
212 * Returns the size of the string INCLUDING the terminating NUL.
213 *
214 * If the string is too long, returns a number larger than @count. User
215 * has to check the return value against "> count".
216 * On exception (or invalid count), returns 0.
217 *
218 * Unlike strnlen_user, this can be used from IRQ handler etc. because
219 * it disables pagefaults.
220 */
221long strnlen_user_nofault(const void __user *unsafe_addr, long count)
222{
223 int ret;
224
225 pagefault_disable();
226 ret = strnlen_user(unsafe_addr, count);
227 pagefault_enable();
228
229 return ret;
230}
231
232void __copy_overflow(int size, unsigned long count)
233{
234 WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count);
235}
236EXPORT_SYMBOL(__copy_overflow);