projects/linux·blob master · refs

263 lines · 9.6 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2022-2024 Jason A. Donenfeld <[email protected]>. All Rights Reserved.
4 */
5
6#include <linux/array_size.h>
7#include <linux/minmax.h>
8#include <vdso/datapage.h>
9#include <vdso/getrandom.h>
10#include <vdso/limits.h>
11#include <vdso/page.h>
12#include <vdso/unaligned.h>
13#include <asm/barrier.h>
14#include <asm/vdso/getrandom.h>
15#include <uapi/linux/errno.h>
16#include <uapi/linux/mman.h>
17#include <uapi/linux/random.h>
18
19/* Bring in default accessors */
20#include <vdso/vsyscall.h>
21
22#define MEMCPY_AND_ZERO_SRC(type, dst, src, len) do { \
23 while (len >= sizeof(type)) { \
24 __put_unaligned_t(type, __get_unaligned_t(type, src), dst); \
25 __put_unaligned_t(type, 0, src); \
26 dst += sizeof(type); \
27 src += sizeof(type); \
28 len -= sizeof(type); \
29 } \
30} while (0)
31
32static void memcpy_and_zero_src(void *dst, void *src, size_t len)
33{
34 if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) {
35 if (IS_ENABLED(CONFIG_64BIT))
36 MEMCPY_AND_ZERO_SRC(u64, dst, src, len);
37 MEMCPY_AND_ZERO_SRC(u32, dst, src, len);
38 MEMCPY_AND_ZERO_SRC(u16, dst, src, len);
39 }
40 MEMCPY_AND_ZERO_SRC(u8, dst, src, len);
41}
42
43/**
44 * __cvdso_getrandom_data - Generic vDSO implementation of getrandom() syscall.
45 * @rng_info: Describes state of kernel RNG, memory shared with kernel.
46 * @buffer: Destination buffer to fill with random bytes.
47 * @len: Size of @buffer in bytes.
48 * @flags: Zero or more GRND_* flags.
49 * @opaque_state: Pointer to an opaque state area.
50 * @opaque_len: Length of opaque state area.
51 *
52 * This implements a "fast key erasure" RNG using ChaCha20, in the same way that the kernel's
53 * getrandom() syscall does. It periodically reseeds its key from the kernel's RNG, at the same
54 * schedule that the kernel's RNG is reseeded. If the kernel's RNG is not ready, then this always
55 * calls into the syscall.
56 *
57 * If @buffer, @len, and @flags are 0, and @opaque_len is ~0UL, then @opaque_state is populated
58 * with a struct vgetrandom_opaque_params and the function returns 0; if it does not return 0,
59 * this function should not be used.
60 *
61 * @opaque_state *must* be allocated by calling mmap(2) using the mmap_prot and mmap_flags fields
62 * from the struct vgetrandom_opaque_params, and states must not straddle pages. Unless external
63 * locking is used, one state must be allocated per thread, as it is not safe to call this function
64 * concurrently with the same @opaque_state. However, it is safe to call this using the same
65 * @opaque_state that is shared between main code and signal handling code, within the same thread.
66 *
67 * Returns: The number of random bytes written to @buffer, or a negative value indicating an error.
68 */
69static __always_inline ssize_t
70__cvdso_getrandom_data(const struct vdso_rng_data *rng_info, void *buffer, size_t len,
71 unsigned int flags, void *opaque_state, size_t opaque_len)
72{
73 ssize_t ret = min_t(size_t, INT_MAX & PAGE_MASK /* = MAX_RW_COUNT */, len);
74 struct vgetrandom_state *state = opaque_state;
75 size_t batch_len, nblocks, orig_len = len;
76 bool in_use, have_retried = false;
77 void *orig_buffer = buffer;
78 u64 current_generation;
79 u32 counter[2] = { 0 };
80
81 if (unlikely(opaque_len == ~0UL && !buffer && !len && !flags)) {
82 struct vgetrandom_opaque_params *params = opaque_state;
83 params->size_of_opaque_state = sizeof(*state);
84 params->mmap_prot = PROT_READ | PROT_WRITE;
85 params->mmap_flags = MAP_DROPPABLE | MAP_ANONYMOUS;
86 for (size_t i = 0; i < ARRAY_SIZE(params->reserved); ++i)
87 params->reserved[i] = 0;
88 return 0;
89 }
90
91 /* The state must not straddle a page, since pages can be zeroed at any time. */
92 if (unlikely(((unsigned long)opaque_state & ~PAGE_MASK) + sizeof(*state) > PAGE_SIZE))
93 return -EFAULT;
94
95 /* Handle unexpected flags by falling back to the kernel. */
96 if (unlikely(flags & ~(GRND_NONBLOCK | GRND_RANDOM | GRND_INSECURE)))
97 goto fallback_syscall;
98
99 /* If the caller passes the wrong size, which might happen due to CRIU, fallback. */
100 if (unlikely(opaque_len != sizeof(*state)))
101 goto fallback_syscall;
102
103 /*
104 * If the kernel's RNG is not yet ready, then it's not possible to provide random bytes from
105 * userspace, because A) the various @flags require this to block, or not, depending on
106 * various factors unavailable to userspace, and B) the kernel's behavior before the RNG is
107 * ready is to reseed from the entropy pool at every invocation.
108 */
109 if (unlikely(!READ_ONCE(rng_info->is_ready)))
110 goto fallback_syscall;
111
112 /*
113 * This condition is checked after @rng_info->is_ready, because before the kernel's RNG is
114 * initialized, the @flags parameter may require this to block or return an error, even when
115 * len is zero.
116 */
117 if (unlikely(!len))
118 return 0;
119
120 /*
121 * @state->in_use is basic reentrancy protection against this running in a signal handler
122 * with the same @opaque_state, but obviously not atomic wrt multiple CPUs or more than one
123 * level of reentrancy. If a signal interrupts this after reading @state->in_use, but before
124 * writing @state->in_use, there is still no race, because the signal handler will run to
125 * its completion before returning execution.
126 */
127 in_use = READ_ONCE(state->in_use);
128 if (unlikely(in_use))
129 /* The syscall simply fills the buffer and does not touch @state, so fallback. */
130 goto fallback_syscall;
131 WRITE_ONCE(state->in_use, true);
132
133retry_generation:
134 /*
135 * @rng_info->generation must always be read here, as it serializes @state->key with the
136 * kernel's RNG reseeding schedule.
137 */
138 current_generation = READ_ONCE(rng_info->generation);
139
140 /*
141 * If @state->generation doesn't match the kernel RNG's generation, then it means the
142 * kernel's RNG has reseeded, and so @state->key is reseeded as well.
143 */
144 if (unlikely(state->generation != current_generation)) {
145 /*
146 * Write the generation before filling the key, in case of fork. If there is a fork
147 * just after this line, the parent and child will get different random bytes from
148 * the syscall, which is good. However, were this line to occur after the getrandom
149 * syscall, then both child and parent could have the same bytes and the same
150 * generation counter, so the fork would not be detected. Therefore, write
151 * @state->generation before the call to the getrandom syscall.
152 */
153 WRITE_ONCE(state->generation, current_generation);
154
155 /*
156 * Prevent the syscall from being reordered wrt current_generation. Pairs with the
157 * smp_store_release(&vdso_k_rng_data->generation) in random.c.
158 */
159 smp_rmb();
160
161 /* Reseed @state->key using fresh bytes from the kernel. */
162 if (getrandom_syscall(state->key, sizeof(state->key), 0) != sizeof(state->key)) {
163 /*
164 * If the syscall failed to refresh the key, then @state->key is now
165 * invalid, so invalidate the generation so that it is not used again, and
166 * fallback to using the syscall entirely.
167 */
168 WRITE_ONCE(state->generation, 0);
169
170 /*
171 * Set @state->in_use to false only after the last write to @state in the
172 * line above.
173 */
174 WRITE_ONCE(state->in_use, false);
175
176 goto fallback_syscall;
177 }
178
179 /*
180 * Set @state->pos to beyond the end of the batch, so that the batch is refilled
181 * using the new key.
182 */
183 state->pos = sizeof(state->batch);
184 }
185
186 /* Set len to the total amount of bytes that this function is allowed to read, ret. */
187 len = ret;
188more_batch:
189 /*
190 * First use bytes out of @state->batch, which may have been filled by the last call to this
191 * function.
192 */
193 batch_len = min_t(size_t, sizeof(state->batch) - state->pos, len);
194 if (batch_len) {
195 /* Zeroing at the same time as memcpying helps preserve forward secrecy. */
196 memcpy_and_zero_src(buffer, state->batch + state->pos, batch_len);
197 state->pos += batch_len;
198 buffer += batch_len;
199 len -= batch_len;
200 }
201
202 if (!len) {
203 /* Prevent the loop from being reordered wrt ->generation. */
204 barrier();
205
206 /*
207 * Since @rng_info->generation will never be 0, re-read @state->generation, rather
208 * than using the local current_generation variable, to learn whether a fork
209 * occurred or if @state was zeroed due to memory pressure. Primarily, though, this
210 * indicates whether the kernel's RNG has reseeded, in which case generate a new key
211 * and start over.
212 */
213 if (unlikely(READ_ONCE(state->generation) != READ_ONCE(rng_info->generation))) {
214 /*
215 * Prevent this from looping forever in case of low memory or racing with a
216 * user force-reseeding the kernel's RNG using the ioctl.
217 */
218 if (have_retried) {
219 WRITE_ONCE(state->in_use, false);
220 goto fallback_syscall;
221 }
222
223 have_retried = true;
224 buffer = orig_buffer;
225 goto retry_generation;
226 }
227
228 /*
229 * Set @state->in_use to false only when there will be no more reads or writes of
230 * @state.
231 */
232 WRITE_ONCE(state->in_use, false);
233 return ret;
234 }
235
236 /* Generate blocks of RNG output directly into @buffer while there's enough room left. */
237 nblocks = len / CHACHA_BLOCK_SIZE;
238 if (nblocks) {
239 __arch_chacha20_blocks_nostack(buffer, state->key, counter, nblocks);
240 buffer += nblocks * CHACHA_BLOCK_SIZE;
241 len -= nblocks * CHACHA_BLOCK_SIZE;
242 }
243
244 BUILD_BUG_ON(sizeof(state->batch_key) % CHACHA_BLOCK_SIZE != 0);
245
246 /* Refill the batch and overwrite the key, in order to preserve forward secrecy. */
247 __arch_chacha20_blocks_nostack(state->batch_key, state->key, counter,
248 sizeof(state->batch_key) / CHACHA_BLOCK_SIZE);
249
250 /* Since the batch was just refilled, set the position back to 0 to indicate a full batch. */
251 state->pos = 0;
252 goto more_batch;
253
254fallback_syscall:
255 return getrandom_syscall(orig_buffer, orig_len, flags);
256}
257
258static __always_inline ssize_t
259__cvdso_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state, size_t opaque_len)
260{
261 return __cvdso_getrandom_data(__arch_get_vdso_u_rng_data(), buffer, len, flags,
262 opaque_state, opaque_len);
263}