projects/linux·blob assoc-array-fixes · refs

1712 lines · 52.2 KiB

1// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
2/*
3 * Copyright (C) 2017-2024 Jason A. Donenfeld <[email protected]>. All Rights Reserved.
4 * Copyright Matt Mackall <[email protected]>, 2003, 2004, 2005
5 * Copyright Theodore Ts'o, 1994, 1995, 1996, 1997, 1998, 1999. All rights reserved.
6 *
7 * This driver produces cryptographically secure pseudorandom data. It is divided
8 * into roughly six sections, each with a section header:
9 *
10 * - Initialization and readiness waiting.
11 * - Fast key erasure RNG, the "crng".
12 * - Entropy accumulation and extraction routines.
13 * - Entropy collection routines.
14 * - Userspace reader/writer interfaces.
15 * - Sysctl interface.
16 *
17 * The high level overview is that there is one input pool, into which
18 * various pieces of data are hashed. Prior to initialization, some of that
19 * data is then "credited" as having a certain number of bits of entropy.
20 * When enough bits of entropy are available, the hash is finalized and
21 * handed as a key to a stream cipher that expands it indefinitely for
22 * various consumers. This key is periodically refreshed as the various
23 * entropy collectors, described below, add data to the input pool.
24 */
25
26#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
27
28#include <linux/utsname.h>
29#include <linux/module.h>
30#include <linux/kernel.h>
31#include <linux/major.h>
32#include <linux/string.h>
33#include <linux/fcntl.h>
34#include <linux/slab.h>
35#include <linux/random.h>
36#include <linux/poll.h>
37#include <linux/init.h>
38#include <linux/fs.h>
39#include <linux/blkdev.h>
40#include <linux/interrupt.h>
41#include <linux/mm.h>
42#include <linux/nodemask.h>
43#include <linux/spinlock.h>
44#include <linux/kthread.h>
45#include <linux/percpu.h>
46#include <linux/ptrace.h>
47#include <linux/workqueue.h>
48#include <linux/irq.h>
49#include <linux/ratelimit.h>
50#include <linux/syscalls.h>
51#include <linux/completion.h>
52#include <linux/uuid.h>
53#include <linux/uaccess.h>
54#include <linux/suspend.h>
55#include <linux/siphash.h>
56#include <linux/sched/isolation.h>
57#include <crypto/chacha.h>
58#include <crypto/blake2s.h>
59#include <vdso/datapage.h>
60#include <asm/archrandom.h>
61#include <asm/processor.h>
62#include <asm/irq.h>
63#include <asm/irq_regs.h>
64#include <asm/io.h>
65
66/*********************************************************************
67 *
68 * Initialization and readiness waiting.
69 *
70 * Much of the RNG infrastructure is devoted to various dependencies
71 * being able to wait until the RNG has collected enough entropy and
72 * is ready for safe consumption.
73 *
74 *********************************************************************/
75
76/*
77 * crng_init is protected by base_crng->lock, and only increases
78 * its value (from empty->early->ready).
79 */
80static enum {
81 CRNG_EMPTY = 0, /* Little to no entropy collected */
82 CRNG_EARLY = 1, /* At least POOL_EARLY_BITS collected */
83 CRNG_READY = 2 /* Fully initialized with POOL_READY_BITS collected */
84} crng_init __read_mostly = CRNG_EMPTY;
85static DEFINE_STATIC_KEY_FALSE(crng_is_ready);
86#define crng_ready() (static_branch_likely(&crng_is_ready) || crng_init >= CRNG_READY)
87/* Various types of waiters for crng_init->CRNG_READY transition. */
88static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait);
89static struct fasync_struct *fasync;
90static ATOMIC_NOTIFIER_HEAD(random_ready_notifier);
91
92/* Control how we warn userspace. */
93static struct ratelimit_state urandom_warning =
94 RATELIMIT_STATE_INIT_FLAGS("urandom_warning", HZ, 3, RATELIMIT_MSG_ON_RELEASE);
95static int ratelimit_disable __read_mostly = 0;
96module_param_named(ratelimit_disable, ratelimit_disable, int, 0644);
97MODULE_PARM_DESC(ratelimit_disable, "Disable random ratelimit suppression");
98
99/*
100 * Returns whether or not the input pool has been seeded and thus guaranteed
101 * to supply cryptographically secure random numbers. This applies to: the
102 * /dev/urandom device, the get_random_bytes function, and the get_random_{u8,
103 * u16,u32,u64,long} family of functions.
104 *
105 * Returns: true if the input pool has been seeded.
106 * false if the input pool has not been seeded.
107 */
108bool rng_is_initialized(void)
109{
110 return crng_ready();
111}
112EXPORT_SYMBOL(rng_is_initialized);
113
114static void __cold crng_set_ready(struct work_struct *work)
115{
116 static_branch_enable(&crng_is_ready);
117}
118
119/* Used by wait_for_random_bytes(), and considered an entropy collector, below. */
120static void try_to_generate_entropy(void);
121
122/*
123 * Wait for the input pool to be seeded and thus guaranteed to supply
124 * cryptographically secure random numbers. This applies to: the /dev/urandom
125 * device, the get_random_bytes function, and the get_random_{u8,u16,u32,u64,
126 * long} family of functions. Using any of these functions without first
127 * calling this function forfeits the guarantee of security.
128 *
129 * Returns: 0 if the input pool has been seeded.
130 * -ERESTARTSYS if the function was interrupted by a signal.
131 */
132int wait_for_random_bytes(void)
133{
134 while (!crng_ready()) {
135 int ret;
136
137 try_to_generate_entropy();
138 ret = wait_event_interruptible_timeout(crng_init_wait, crng_ready(), HZ);
139 if (ret)
140 return ret > 0 ? 0 : ret;
141 }
142 return 0;
143}
144EXPORT_SYMBOL(wait_for_random_bytes);
145
146/*
147 * Add a callback function that will be invoked when the crng is initialised,
148 * or immediately if it already has been. Only use this is you are absolutely
149 * sure it is required. Most users should instead be able to test
150 * `rng_is_initialized()` on demand, or make use of `get_random_bytes_wait()`.
151 */
152int __cold execute_with_initialized_rng(struct notifier_block *nb)
153{
154 unsigned long flags;
155 int ret = 0;
156
157 spin_lock_irqsave(&random_ready_notifier.lock, flags);
158 if (crng_ready())
159 nb->notifier_call(nb, 0, NULL);
160 else
161 ret = raw_notifier_chain_register((struct raw_notifier_head *)&random_ready_notifier.head, nb);
162 spin_unlock_irqrestore(&random_ready_notifier.lock, flags);
163 return ret;
164}
165
166/*********************************************************************
167 *
168 * Fast key erasure RNG, the "crng".
169 *
170 * These functions expand entropy from the entropy extractor into
171 * long streams for external consumption using the "fast key erasure"
172 * RNG described at <https://blog.cr.yp.to/20170723-random.html>.
173 *
174 * There are a few exported interfaces for use by other drivers:
175 *
176 * void get_random_bytes(void *buf, size_t len)
177 * u8 get_random_u8()
178 * u16 get_random_u16()
179 * u32 get_random_u32()
180 * u32 get_random_u32_below(u32 ceil)
181 * u32 get_random_u32_above(u32 floor)
182 * u32 get_random_u32_inclusive(u32 floor, u32 ceil)
183 * u64 get_random_u64()
184 * unsigned long get_random_long()
185 *
186 * These interfaces will return the requested number of random bytes
187 * into the given buffer or as a return value. This is equivalent to
188 * a read from /dev/urandom. The u8, u16, u32, u64, long family of
189 * functions may be higher performance for one-off random integers,
190 * because they do a bit of buffering and do not invoke reseeding
191 * until the buffer is emptied.
192 *
193 *********************************************************************/
194
195enum {
196 CRNG_RESEED_START_INTERVAL = HZ,
197 CRNG_RESEED_INTERVAL = 60 * HZ
198};
199
200static struct {
201 u8 key[CHACHA_KEY_SIZE] __aligned(__alignof__(long));
202 unsigned long generation;
203 spinlock_t lock;
204} base_crng = {
205 .lock = __SPIN_LOCK_UNLOCKED(base_crng.lock)
206};
207
208struct crng {
209 u8 key[CHACHA_KEY_SIZE];
210 unsigned long generation;
211 local_lock_t lock;
212};
213
214static DEFINE_PER_CPU(struct crng, crngs) = {
215 .generation = ULONG_MAX,
216 .lock = INIT_LOCAL_LOCK(crngs.lock),
217};
218
219/*
220 * Return the interval until the next reseeding, which is normally
221 * CRNG_RESEED_INTERVAL, but during early boot, it is at an interval
222 * proportional to the uptime.
223 */
224static unsigned int crng_reseed_interval(void)
225{
226 static bool early_boot = true;
227
228 if (unlikely(READ_ONCE(early_boot))) {
229 time64_t uptime = ktime_get_seconds();
230 if (uptime >= CRNG_RESEED_INTERVAL / HZ * 2)
231 WRITE_ONCE(early_boot, false);
232 else
233 return max_t(unsigned int, CRNG_RESEED_START_INTERVAL,
234 (unsigned int)uptime / 2 * HZ);
235 }
236 return CRNG_RESEED_INTERVAL;
237}
238
239/* Used by crng_reseed() and crng_make_state() to extract a new seed from the input pool. */
240static void extract_entropy(void *buf, size_t len);
241
242/* This extracts a new crng key from the input pool. */
243static void crng_reseed(struct work_struct *work)
244{
245 static DECLARE_DELAYED_WORK(next_reseed, crng_reseed);
246 unsigned long flags;
247 unsigned long next_gen;
248 u8 key[CHACHA_KEY_SIZE];
249
250 /* Immediately schedule the next reseeding, so that it fires sooner rather than later. */
251 if (likely(system_dfl_wq))
252 queue_delayed_work(system_dfl_wq, &next_reseed, crng_reseed_interval());
253
254 extract_entropy(key, sizeof(key));
255
256 /*
257 * We copy the new key into the base_crng, overwriting the old one,
258 * and update the generation counter. We avoid hitting ULONG_MAX,
259 * because the per-cpu crngs are initialized to ULONG_MAX, so this
260 * forces new CPUs that come online to always initialize.
261 */
262 spin_lock_irqsave(&base_crng.lock, flags);
263 memcpy(base_crng.key, key, sizeof(base_crng.key));
264 next_gen = base_crng.generation + 1;
265 if (next_gen == ULONG_MAX)
266 ++next_gen;
267 WRITE_ONCE(base_crng.generation, next_gen);
268
269 /* base_crng.generation's invalid value is ULONG_MAX, while
270 * vdso_k_rng_data->generation's invalid value is 0, so add one to the
271 * former to arrive at the latter. Use smp_store_release so that this
272 * is ordered with the write above to base_crng.generation. Pairs with
273 * the smp_rmb() before the syscall in the vDSO code.
274 *
275 * Cast to unsigned long for 32-bit architectures, since atomic 64-bit
276 * operations are not supported on those architectures. This is safe
277 * because base_crng.generation is a 32-bit value. On big-endian
278 * architectures it will be stored in the upper 32 bits, but that's okay
279 * because the vDSO side only checks whether the value changed, without
280 * actually using or interpreting the value.
281 */
282 if (IS_ENABLED(CONFIG_VDSO_GETRANDOM))
283 smp_store_release((unsigned long *)&vdso_k_rng_data->generation, next_gen + 1);
284
285 if (!static_branch_likely(&crng_is_ready))
286 crng_init = CRNG_READY;
287 spin_unlock_irqrestore(&base_crng.lock, flags);
288 memzero_explicit(key, sizeof(key));
289}
290
291/*
292 * This generates a ChaCha block using the provided key, and then
293 * immediately overwrites that key with half the block. It returns
294 * the resultant ChaCha state to the user, along with the second
295 * half of the block containing 32 bytes of random data that may
296 * be used; random_data_len may not be greater than 32.
297 *
298 * The returned ChaCha state contains within it a copy of the old
299 * key value, at index 4, so the state should always be zeroed out
300 * immediately after using in order to maintain forward secrecy.
301 * If the state cannot be erased in a timely manner, then it is
302 * safer to set the random_data parameter to &chacha_state->x[4]
303 * so that this function overwrites it before returning.
304 */
305static void crng_fast_key_erasure(u8 key[CHACHA_KEY_SIZE],
306 struct chacha_state *chacha_state,
307 u8 *random_data, size_t random_data_len)
308{
309 u8 first_block[CHACHA_BLOCK_SIZE];
310
311 BUG_ON(random_data_len > 32);
312
313 chacha_init_consts(chacha_state);
314 memcpy(&chacha_state->x[4], key, CHACHA_KEY_SIZE);
315 memset(&chacha_state->x[12], 0, sizeof(u32) * 4);
316 chacha20_block(chacha_state, first_block);
317
318 memcpy(key, first_block, CHACHA_KEY_SIZE);
319 memcpy(random_data, first_block + CHACHA_KEY_SIZE, random_data_len);
320 memzero_explicit(first_block, sizeof(first_block));
321}
322
323/*
324 * This function returns a ChaCha state that you may use for generating
325 * random data. It also returns up to 32 bytes on its own of random data
326 * that may be used; random_data_len may not be greater than 32.
327 */
328static void crng_make_state(struct chacha_state *chacha_state,
329 u8 *random_data, size_t random_data_len)
330{
331 unsigned long flags;
332 struct crng *crng;
333
334 BUG_ON(random_data_len > 32);
335
336 /*
337 * For the fast path, we check whether we're ready, unlocked first, and
338 * then re-check once locked later. In the case where we're really not
339 * ready, we do fast key erasure with the base_crng directly, extracting
340 * when crng_init is CRNG_EMPTY.
341 */
342 if (!crng_ready()) {
343 bool ready;
344
345 spin_lock_irqsave(&base_crng.lock, flags);
346 ready = crng_ready();
347 if (!ready) {
348 if (crng_init == CRNG_EMPTY)
349 extract_entropy(base_crng.key, sizeof(base_crng.key));
350 crng_fast_key_erasure(base_crng.key, chacha_state,
351 random_data, random_data_len);
352 }
353 spin_unlock_irqrestore(&base_crng.lock, flags);
354 if (!ready)
355 return;
356 }
357
358 local_lock_irqsave(&crngs.lock, flags);
359 crng = raw_cpu_ptr(&crngs);
360
361 /*
362 * If our per-cpu crng is older than the base_crng, then it means
363 * somebody reseeded the base_crng. In that case, we do fast key
364 * erasure on the base_crng, and use its output as the new key
365 * for our per-cpu crng. This brings us up to date with base_crng.
366 */
367 if (unlikely(crng->generation != READ_ONCE(base_crng.generation))) {
368 spin_lock(&base_crng.lock);
369 crng_fast_key_erasure(base_crng.key, chacha_state,
370 crng->key, sizeof(crng->key));
371 crng->generation = base_crng.generation;
372 spin_unlock(&base_crng.lock);
373 }
374
375 /*
376 * Finally, when we've made it this far, our per-cpu crng has an up
377 * to date key, and we can do fast key erasure with it to produce
378 * some random data and a ChaCha state for the caller. All other
379 * branches of this function are "unlikely", so most of the time we
380 * should wind up here immediately.
381 */
382 crng_fast_key_erasure(crng->key, chacha_state, random_data, random_data_len);
383 local_unlock_irqrestore(&crngs.lock, flags);
384}
385
386static void _get_random_bytes(void *buf, size_t len)
387{
388 struct chacha_state chacha_state;
389 u8 tmp[CHACHA_BLOCK_SIZE];
390 size_t first_block_len;
391
392 if (!len)
393 return;
394
395 first_block_len = min_t(size_t, 32, len);
396 crng_make_state(&chacha_state, buf, first_block_len);
397 len -= first_block_len;
398 buf += first_block_len;
399
400 while (len) {
401 if (len < CHACHA_BLOCK_SIZE) {
402 chacha20_block(&chacha_state, tmp);
403 memcpy(buf, tmp, len);
404 memzero_explicit(tmp, sizeof(tmp));
405 break;
406 }
407
408 chacha20_block(&chacha_state, buf);
409 if (unlikely(chacha_state.x[12] == 0))
410 ++chacha_state.x[13];
411 len -= CHACHA_BLOCK_SIZE;
412 buf += CHACHA_BLOCK_SIZE;
413 }
414
415 chacha_zeroize_state(&chacha_state);
416}
417
418/*
419 * This returns random bytes in arbitrary quantities. The quality of the
420 * random bytes is as good as /dev/urandom. In order to ensure that the
421 * randomness provided by this function is okay, the function
422 * wait_for_random_bytes() should be called and return 0 at least once
423 * at any point prior.
424 */
425void get_random_bytes(void *buf, size_t len)
426{
427 _get_random_bytes(buf, len);
428}
429EXPORT_SYMBOL(get_random_bytes);
430
431static ssize_t get_random_bytes_user(struct iov_iter *iter)
432{
433 struct chacha_state chacha_state;
434 u8 block[CHACHA_BLOCK_SIZE];
435 size_t ret = 0, copied;
436
437 if (unlikely(!iov_iter_count(iter)))
438 return 0;
439
440 /*
441 * Immediately overwrite the ChaCha key at index 4 with random
442 * bytes, in case userspace causes copy_to_iter() below to sleep
443 * forever, so that we still retain forward secrecy in that case.
444 */
445 crng_make_state(&chacha_state, (u8 *)&chacha_state.x[4],
446 CHACHA_KEY_SIZE);
447 /*
448 * However, if we're doing a read of len <= 32, we don't need to
449 * use chacha_state after, so we can simply return those bytes to
450 * the user directly.
451 */
452 if (iov_iter_count(iter) <= CHACHA_KEY_SIZE) {
453 ret = copy_to_iter(&chacha_state.x[4], CHACHA_KEY_SIZE, iter);
454 goto out_zero_chacha;
455 }
456
457 for (;;) {
458 chacha20_block(&chacha_state, block);
459 if (unlikely(chacha_state.x[12] == 0))
460 ++chacha_state.x[13];
461
462 copied = copy_to_iter(block, sizeof(block), iter);
463 ret += copied;
464 if (!iov_iter_count(iter) || copied != sizeof(block))
465 break;
466
467 BUILD_BUG_ON(PAGE_SIZE % sizeof(block) != 0);
468 if (ret % PAGE_SIZE == 0) {
469 if (signal_pending(current))
470 break;
471 cond_resched();
472 }
473 }
474
475 memzero_explicit(block, sizeof(block));
476out_zero_chacha:
477 chacha_zeroize_state(&chacha_state);
478 return ret ? ret : -EFAULT;
479}
480
481/*
482 * Batched entropy returns random integers. The quality of the random
483 * number is as good as /dev/urandom. In order to ensure that the randomness
484 * provided by this function is okay, the function wait_for_random_bytes()
485 * should be called and return 0 at least once at any point prior.
486 */
487
488#define DEFINE_BATCHED_ENTROPY(type) \
489struct batch_ ##type { \
490 /* \
491 * We make this 1.5x a ChaCha block, so that we get the \
492 * remaining 32 bytes from fast key erasure, plus one full \
493 * block from the detached ChaCha state. We can increase \
494 * the size of this later if needed so long as we keep the \
495 * formula of (integer_blocks + 0.5) * CHACHA_BLOCK_SIZE. \
496 */ \
497 type entropy[CHACHA_BLOCK_SIZE * 3 / (2 * sizeof(type))]; \
498 local_lock_t lock; \
499 unsigned long generation; \
500 unsigned int position; \
501}; \
502 \
503static DEFINE_PER_CPU(struct batch_ ##type, batched_entropy_ ##type) = { \
504 .lock = INIT_LOCAL_LOCK(batched_entropy_ ##type.lock), \
505 .position = UINT_MAX \
506}; \
507 \
508type get_random_ ##type(void) \
509{ \
510 type ret; \
511 unsigned long flags; \
512 struct batch_ ##type *batch; \
513 unsigned long next_gen; \
514 \
515 if (!crng_ready()) { \
516 _get_random_bytes(&ret, sizeof(ret)); \
517 return ret; \
518 } \
519 \
520 local_lock_irqsave(&batched_entropy_ ##type.lock, flags); \
521 batch = raw_cpu_ptr(&batched_entropy_##type); \
522 \
523 next_gen = READ_ONCE(base_crng.generation); \
524 if (batch->position >= ARRAY_SIZE(batch->entropy) || \
525 next_gen != batch->generation) { \
526 _get_random_bytes(batch->entropy, sizeof(batch->entropy)); \
527 batch->position = 0; \
528 batch->generation = next_gen; \
529 } \
530 \
531 ret = batch->entropy[batch->position]; \
532 batch->entropy[batch->position] = 0; \
533 ++batch->position; \
534 local_unlock_irqrestore(&batched_entropy_ ##type.lock, flags); \
535 return ret; \
536} \
537EXPORT_SYMBOL(get_random_ ##type);
538
539DEFINE_BATCHED_ENTROPY(u8)
540DEFINE_BATCHED_ENTROPY(u16)
541DEFINE_BATCHED_ENTROPY(u32)
542DEFINE_BATCHED_ENTROPY(u64)
543
544u32 __get_random_u32_below(u32 ceil)
545{
546 /*
547 * This is the slow path for variable ceil. It is still fast, most of
548 * the time, by doing traditional reciprocal multiplication and
549 * opportunistically comparing the lower half to ceil itself, before
550 * falling back to computing a larger bound, and then rejecting samples
551 * whose lower half would indicate a range indivisible by ceil. The use
552 * of `-ceil % ceil` is analogous to `2^32 % ceil`, but is computable
553 * in 32-bits.
554 */
555 u32 rand = get_random_u32();
556 u64 mult;
557
558 /*
559 * This function is technically undefined for ceil == 0, and in fact
560 * for the non-underscored constant version in the header, we build bug
561 * on that. But for the non-constant case, it's convenient to have that
562 * evaluate to being a straight call to get_random_u32(), so that
563 * get_random_u32_inclusive() can work over its whole range without
564 * undefined behavior.
565 */
566 if (unlikely(!ceil))
567 return rand;
568
569 mult = (u64)ceil * rand;
570 if (unlikely((u32)mult < ceil)) {
571 u32 bound = -ceil % ceil;
572 while (unlikely((u32)mult < bound))
573 mult = (u64)ceil * get_random_u32();
574 }
575 return mult >> 32;
576}
577EXPORT_SYMBOL(__get_random_u32_below);
578
579#ifdef CONFIG_SMP
580/*
581 * This function is called when the CPU is coming up, with entry
582 * CPUHP_RANDOM_PREPARE, which comes before CPUHP_WORKQUEUE_PREP.
583 */
584int __cold random_prepare_cpu(unsigned int cpu)
585{
586 /*
587 * When the cpu comes back online, immediately invalidate both
588 * the per-cpu crng and all batches, so that we serve fresh
589 * randomness.
590 */
591 per_cpu_ptr(&crngs, cpu)->generation = ULONG_MAX;
592 per_cpu_ptr(&batched_entropy_u8, cpu)->position = UINT_MAX;
593 per_cpu_ptr(&batched_entropy_u16, cpu)->position = UINT_MAX;
594 per_cpu_ptr(&batched_entropy_u32, cpu)->position = UINT_MAX;
595 per_cpu_ptr(&batched_entropy_u64, cpu)->position = UINT_MAX;
596 return 0;
597}
598#endif
599
600
601/**********************************************************************
602 *
603 * Entropy accumulation and extraction routines.
604 *
605 * Callers may add entropy via:
606 *
607 * static void mix_pool_bytes(const void *buf, size_t len)
608 *
609 * After which, if added entropy should be credited:
610 *
611 * static void credit_init_bits(size_t bits)
612 *
613 * Finally, extract entropy via:
614 *
615 * static void extract_entropy(void *buf, size_t len)
616 *
617 **********************************************************************/
618
619enum {
620 POOL_BITS = BLAKE2S_HASH_SIZE * 8,
621 POOL_READY_BITS = POOL_BITS, /* When crng_init->CRNG_READY */
622 POOL_EARLY_BITS = POOL_READY_BITS / 2 /* When crng_init->CRNG_EARLY */
623};
624
625static struct {
626 struct blake2s_ctx hash;
627 spinlock_t lock;
628 unsigned int init_bits;
629} input_pool = {
630 .hash.h = { BLAKE2S_IV0 ^ (0x01010000 | BLAKE2S_HASH_SIZE),
631 BLAKE2S_IV1, BLAKE2S_IV2, BLAKE2S_IV3, BLAKE2S_IV4,
632 BLAKE2S_IV5, BLAKE2S_IV6, BLAKE2S_IV7 },
633 .hash.outlen = BLAKE2S_HASH_SIZE,
634 .lock = __SPIN_LOCK_UNLOCKED(input_pool.lock),
635};
636
637static void _mix_pool_bytes(const void *buf, size_t len)
638{
639 blake2s_update(&input_pool.hash, buf, len);
640}
641
642/*
643 * This function adds bytes into the input pool. It does not
644 * update the initialization bit counter; the caller should call
645 * credit_init_bits if this is appropriate.
646 */
647static void mix_pool_bytes(const void *buf, size_t len)
648{
649 unsigned long flags;
650
651 spin_lock_irqsave(&input_pool.lock, flags);
652 _mix_pool_bytes(buf, len);
653 spin_unlock_irqrestore(&input_pool.lock, flags);
654}
655
656/*
657 * This is an HKDF-like construction for using the hashed collected entropy
658 * as a PRF key, that's then expanded block-by-block.
659 */
660static void extract_entropy(void *buf, size_t len)
661{
662 unsigned long flags;
663 u8 seed[BLAKE2S_HASH_SIZE], next_key[BLAKE2S_HASH_SIZE];
664 struct {
665 unsigned long rdseed[32 / sizeof(long)];
666 size_t counter;
667 } block;
668 size_t i, longs;
669
670 for (i = 0; i < ARRAY_SIZE(block.rdseed);) {
671 longs = arch_get_random_seed_longs(&block.rdseed[i], ARRAY_SIZE(block.rdseed) - i);
672 if (longs) {
673 i += longs;
674 continue;
675 }
676 longs = arch_get_random_longs(&block.rdseed[i], ARRAY_SIZE(block.rdseed) - i);
677 if (longs) {
678 i += longs;
679 continue;
680 }
681 block.rdseed[i++] = random_get_entropy();
682 }
683
684 spin_lock_irqsave(&input_pool.lock, flags);
685
686 /* seed = HASHPRF(last_key, entropy_input) */
687 blake2s_final(&input_pool.hash, seed);
688
689 /* next_key = HASHPRF(seed, RDSEED || 0) */
690 block.counter = 0;
691 blake2s(seed, sizeof(seed), (const u8 *)&block, sizeof(block), next_key, sizeof(next_key));
692 blake2s_init_key(&input_pool.hash, BLAKE2S_HASH_SIZE, next_key, sizeof(next_key));
693
694 spin_unlock_irqrestore(&input_pool.lock, flags);
695 memzero_explicit(next_key, sizeof(next_key));
696
697 while (len) {
698 i = min_t(size_t, len, BLAKE2S_HASH_SIZE);
699 /* output = HASHPRF(seed, RDSEED || ++counter) */
700 ++block.counter;
701 blake2s(seed, sizeof(seed), (const u8 *)&block, sizeof(block), buf, i);
702 len -= i;
703 buf += i;
704 }
705
706 memzero_explicit(seed, sizeof(seed));
707 memzero_explicit(&block, sizeof(block));
708}
709
710#define credit_init_bits(bits) if (!crng_ready()) _credit_init_bits(bits)
711
712static void __cold _credit_init_bits(size_t bits)
713{
714 static DECLARE_WORK(set_ready, crng_set_ready);
715 unsigned int new, orig, add;
716 unsigned long flags;
717 int m;
718
719 if (!bits)
720 return;
721
722 add = min_t(size_t, bits, POOL_BITS);
723
724 orig = READ_ONCE(input_pool.init_bits);
725 do {
726 new = min_t(unsigned int, POOL_BITS, orig + add);
727 } while (!try_cmpxchg(&input_pool.init_bits, &orig, new));
728
729 if (orig < POOL_READY_BITS && new >= POOL_READY_BITS) {
730 crng_reseed(NULL); /* Sets crng_init to CRNG_READY under base_crng.lock. */
731 if (system_dfl_wq)
732 queue_work(system_dfl_wq, &set_ready);
733 atomic_notifier_call_chain(&random_ready_notifier, 0, NULL);
734 if (IS_ENABLED(CONFIG_VDSO_GETRANDOM))
735 WRITE_ONCE(vdso_k_rng_data->is_ready, true);
736 wake_up_interruptible(&crng_init_wait);
737 kill_fasync(&fasync, SIGIO, POLL_IN);
738 pr_notice("crng init done\n");
739 m = ratelimit_state_get_miss(&urandom_warning);
740 if (m)
741 pr_notice("%d urandom warning(s) missed due to ratelimiting\n", m);
742 } else if (orig < POOL_EARLY_BITS && new >= POOL_EARLY_BITS) {
743 spin_lock_irqsave(&base_crng.lock, flags);
744 /* Check if crng_init is CRNG_EMPTY, to avoid race with crng_reseed(). */
745 if (crng_init == CRNG_EMPTY) {
746 extract_entropy(base_crng.key, sizeof(base_crng.key));
747 crng_init = CRNG_EARLY;
748 }
749 spin_unlock_irqrestore(&base_crng.lock, flags);
750 }
751}
752
753
754/**********************************************************************
755 *
756 * Entropy collection routines.
757 *
758 * The following exported functions are used for pushing entropy into
759 * the above entropy accumulation routines:
760 *
761 * void add_device_randomness(const void *buf, size_t len);
762 * void add_hwgenerator_randomness(const void *buf, size_t len, size_t entropy, bool sleep_after);
763 * void add_bootloader_randomness(const void *buf, size_t len);
764 * void add_vmfork_randomness(const void *unique_vm_id, size_t len);
765 * void add_interrupt_randomness(int irq);
766 * void add_input_randomness(unsigned int type, unsigned int code, unsigned int value);
767 * void add_disk_randomness(struct gendisk *disk);
768 *
769 * add_device_randomness() adds data to the input pool that
770 * is likely to differ between two devices (or possibly even per boot).
771 * This would be things like MAC addresses or serial numbers, or the
772 * read-out of the RTC. This does *not* credit any actual entropy to
773 * the pool, but it initializes the pool to different values for devices
774 * that might otherwise be identical and have very little entropy
775 * available to them (particularly common in the embedded world).
776 *
777 * add_hwgenerator_randomness() is for true hardware RNGs, and will credit
778 * entropy as specified by the caller. If the entropy pool is full it will
779 * block until more entropy is needed.
780 *
781 * add_bootloader_randomness() is called by bootloader drivers, such as EFI
782 * and device tree, and credits its input depending on whether or not the
783 * command line option 'random.trust_bootloader' is set.
784 *
785 * add_vmfork_randomness() adds a unique (but not necessarily secret) ID
786 * representing the current instance of a VM to the pool, without crediting,
787 * and then force-reseeds the crng so that it takes effect immediately.
788 *
789 * add_interrupt_randomness() uses the interrupt timing as random
790 * inputs to the entropy pool. Using the cycle counters and the irq source
791 * as inputs, it feeds the input pool roughly once a second or after 64
792 * interrupts, crediting 1 bit of entropy for whichever comes first.
793 *
794 * add_input_randomness() uses the input layer interrupt timing, as well
795 * as the event type information from the hardware.
796 *
797 * add_disk_randomness() uses what amounts to the seek time of block
798 * layer request events, on a per-disk_devt basis, as input to the
799 * entropy pool. Note that high-speed solid state drives with very low
800 * seek times do not make for good sources of entropy, as their seek
801 * times are usually fairly consistent.
802 *
803 * The last two routines try to estimate how many bits of entropy
804 * to credit. They do this by keeping track of the first and second
805 * order deltas of the event timings.
806 *
807 **********************************************************************/
808
809static bool trust_cpu __initdata = true;
810static bool trust_bootloader __initdata = true;
811static int __init parse_trust_cpu(char *arg)
812{
813 return kstrtobool(arg, &trust_cpu);
814}
815static int __init parse_trust_bootloader(char *arg)
816{
817 return kstrtobool(arg, &trust_bootloader);
818}
819early_param("random.trust_cpu", parse_trust_cpu);
820early_param("random.trust_bootloader", parse_trust_bootloader);
821
822static int random_pm_notification(struct notifier_block *nb, unsigned long action, void *data)
823{
824 unsigned long flags, entropy = random_get_entropy();
825
826 /*
827 * Encode a representation of how long the system has been suspended,
828 * in a way that is distinct from prior system suspends.
829 */
830 ktime_t stamps[] = { ktime_get(), ktime_get_boottime(), ktime_get_real() };
831
832 spin_lock_irqsave(&input_pool.lock, flags);
833 _mix_pool_bytes(&action, sizeof(action));
834 _mix_pool_bytes(stamps, sizeof(stamps));
835 _mix_pool_bytes(&entropy, sizeof(entropy));
836 spin_unlock_irqrestore(&input_pool.lock, flags);
837
838 if (crng_ready() && (action == PM_RESTORE_PREPARE ||
839 (action == PM_POST_SUSPEND && !IS_ENABLED(CONFIG_PM_AUTOSLEEP) &&
840 !IS_ENABLED(CONFIG_PM_USERSPACE_AUTOSLEEP)))) {
841 crng_reseed(NULL);
842 pr_notice("crng reseeded on system resumption\n");
843 }
844 return 0;
845}
846
847static struct notifier_block pm_notifier = { .notifier_call = random_pm_notification };
848
849/*
850 * This is called extremely early, before time keeping functionality is
851 * available, but arch randomness is. Interrupts are not yet enabled.
852 */
853void __init random_init_early(const char *command_line)
854{
855 unsigned long entropy[BLAKE2S_BLOCK_SIZE / sizeof(long)];
856 size_t i, longs, arch_bits;
857
858#if defined(LATENT_ENTROPY_PLUGIN)
859 static const u8 compiletime_seed[BLAKE2S_BLOCK_SIZE] __initconst __latent_entropy;
860 _mix_pool_bytes(compiletime_seed, sizeof(compiletime_seed));
861#endif
862
863 for (i = 0, arch_bits = sizeof(entropy) * 8; i < ARRAY_SIZE(entropy);) {
864 longs = arch_get_random_seed_longs(entropy, ARRAY_SIZE(entropy) - i);
865 if (longs) {
866 _mix_pool_bytes(entropy, sizeof(*entropy) * longs);
867 i += longs;
868 continue;
869 }
870 longs = arch_get_random_longs(entropy, ARRAY_SIZE(entropy) - i);
871 if (longs) {
872 _mix_pool_bytes(entropy, sizeof(*entropy) * longs);
873 i += longs;
874 continue;
875 }
876 arch_bits -= sizeof(*entropy) * 8;
877 ++i;
878 }
879
880 _mix_pool_bytes(init_utsname(), sizeof(*(init_utsname())));
881 _mix_pool_bytes(command_line, strlen(command_line));
882
883 /* Reseed if already seeded by earlier phases. */
884 if (crng_ready())
885 crng_reseed(NULL);
886 else if (trust_cpu)
887 _credit_init_bits(arch_bits);
888}
889
890/*
891 * This is called a little bit after the prior function, and now there is
892 * access to timestamps counters. Interrupts are not yet enabled.
893 */
894void __init random_init(void)
895{
896 unsigned long entropy = random_get_entropy();
897 ktime_t now = ktime_get_real();
898
899 _mix_pool_bytes(&now, sizeof(now));
900 _mix_pool_bytes(&entropy, sizeof(entropy));
901 add_latent_entropy();
902
903 /*
904 * If we were initialized by the cpu or bootloader before workqueues
905 * are initialized, then we should enable the static branch here.
906 */
907 if (!static_branch_likely(&crng_is_ready) && crng_init >= CRNG_READY)
908 crng_set_ready(NULL);
909
910 /* Reseed if already seeded by earlier phases. */
911 if (crng_ready())
912 crng_reseed(NULL);
913
914 WARN_ON(register_pm_notifier(&pm_notifier));
915
916 WARN(!entropy, "Missing cycle counter and fallback timer; RNG "
917 "entropy collection will consequently suffer.");
918}
919
920/*
921 * Add device- or boot-specific data to the input pool to help
922 * initialize it.
923 *
924 * None of this adds any entropy; it is meant to avoid the problem of
925 * the entropy pool having similar initial state across largely
926 * identical devices.
927 */
928void add_device_randomness(const void *buf, size_t len)
929{
930 unsigned long entropy = random_get_entropy();
931 unsigned long flags;
932
933 spin_lock_irqsave(&input_pool.lock, flags);
934 _mix_pool_bytes(&entropy, sizeof(entropy));
935 _mix_pool_bytes(buf, len);
936 spin_unlock_irqrestore(&input_pool.lock, flags);
937}
938EXPORT_SYMBOL(add_device_randomness);
939
940/*
941 * Interface for in-kernel drivers of true hardware RNGs. Those devices
942 * may produce endless random bits, so this function will sleep for
943 * some amount of time after, if the sleep_after parameter is true.
944 */
945void add_hwgenerator_randomness(const void *buf, size_t len, size_t entropy, bool sleep_after)
946{
947 mix_pool_bytes(buf, len);
948 credit_init_bits(entropy);
949
950 /*
951 * Throttle writing to once every reseed interval, unless we're not yet
952 * initialized or no entropy is credited.
953 */
954 if (sleep_after && !kthread_should_stop() && (crng_ready() || !entropy))
955 schedule_timeout_interruptible(crng_reseed_interval());
956}
957EXPORT_SYMBOL_GPL(add_hwgenerator_randomness);
958
959/*
960 * Handle random seed passed by bootloader, and credit it depending
961 * on the command line option 'random.trust_bootloader'.
962 */
963void __init add_bootloader_randomness(const void *buf, size_t len)
964{
965 mix_pool_bytes(buf, len);
966 if (trust_bootloader)
967 credit_init_bits(len * 8);
968}
969
970#if IS_ENABLED(CONFIG_VMGENID)
971static BLOCKING_NOTIFIER_HEAD(vmfork_chain);
972
973/*
974 * Handle a new unique VM ID, which is unique, not secret, so we
975 * don't credit it, but we do immediately force a reseed after so
976 * that it's used by the crng posthaste.
977 */
978void __cold add_vmfork_randomness(const void *unique_vm_id, size_t len)
979{
980 add_device_randomness(unique_vm_id, len);
981 if (crng_ready()) {
982 crng_reseed(NULL);
983 pr_notice("crng reseeded due to virtual machine fork\n");
984 }
985 blocking_notifier_call_chain(&vmfork_chain, 0, NULL);
986}
987#if IS_MODULE(CONFIG_VMGENID)
988EXPORT_SYMBOL_GPL(add_vmfork_randomness);
989#endif
990
991int __cold register_random_vmfork_notifier(struct notifier_block *nb)
992{
993 return blocking_notifier_chain_register(&vmfork_chain, nb);
994}
995EXPORT_SYMBOL_GPL(register_random_vmfork_notifier);
996
997int __cold unregister_random_vmfork_notifier(struct notifier_block *nb)
998{
999 return blocking_notifier_chain_unregister(&vmfork_chain, nb);
1000}
1001EXPORT_SYMBOL_GPL(unregister_random_vmfork_notifier);
1002#endif
1003
1004struct fast_pool {
1005 unsigned long pool[4];
1006 unsigned long last;
1007 unsigned int count;
1008 struct timer_list mix;
1009};
1010
1011static void mix_interrupt_randomness(struct timer_list *work);
1012
1013static DEFINE_PER_CPU(struct fast_pool, irq_randomness) = {
1014#ifdef CONFIG_64BIT
1015#define FASTMIX_PERM SIPHASH_PERMUTATION
1016 .pool = { SIPHASH_CONST_0, SIPHASH_CONST_1, SIPHASH_CONST_2, SIPHASH_CONST_3 },
1017#else
1018#define FASTMIX_PERM HSIPHASH_PERMUTATION
1019 .pool = { HSIPHASH_CONST_0, HSIPHASH_CONST_1, HSIPHASH_CONST_2, HSIPHASH_CONST_3 },
1020#endif
1021 .mix = __TIMER_INITIALIZER(mix_interrupt_randomness, 0)
1022};
1023
1024/*
1025 * This is [Half]SipHash-1-x, starting from an empty key. Because
1026 * the key is fixed, it assumes that its inputs are non-malicious,
1027 * and therefore this has no security on its own. s represents the
1028 * four-word SipHash state, while v represents a two-word input.
1029 */
1030static void fast_mix(unsigned long s[4], unsigned long v1, unsigned long v2)
1031{
1032 s[3] ^= v1;
1033 FASTMIX_PERM(s[0], s[1], s[2], s[3]);
1034 s[0] ^= v1;
1035 s[3] ^= v2;
1036 FASTMIX_PERM(s[0], s[1], s[2], s[3]);
1037 s[0] ^= v2;
1038}
1039
1040#ifdef CONFIG_SMP
1041/*
1042 * This function is called when the CPU has just come online, with
1043 * entry CPUHP_AP_RANDOM_ONLINE, just after CPUHP_AP_WORKQUEUE_ONLINE.
1044 */
1045int __cold random_online_cpu(unsigned int cpu)
1046{
1047 /*
1048 * During CPU shutdown and before CPU onlining, add_interrupt_
1049 * randomness() may schedule mix_interrupt_randomness(), and
1050 * set the MIX_INFLIGHT flag. However, because the worker can
1051 * be scheduled on a different CPU during this period, that
1052 * flag will never be cleared. For that reason, we zero out
1053 * the flag here, which runs just after workqueues are onlined
1054 * for the CPU again. This also has the effect of setting the
1055 * irq randomness count to zero so that new accumulated irqs
1056 * are fresh.
1057 */
1058 per_cpu_ptr(&irq_randomness, cpu)->count = 0;
1059 return 0;
1060}
1061#endif
1062
1063static void mix_interrupt_randomness(struct timer_list *work)
1064{
1065 struct fast_pool *fast_pool = container_of(work, struct fast_pool, mix);
1066 /*
1067 * The size of the copied stack pool is explicitly 2 longs so that we
1068 * only ever ingest half of the siphash output each time, retaining
1069 * the other half as the next "key" that carries over. The entropy is
1070 * supposed to be sufficiently dispersed between bits so on average
1071 * we don't wind up "losing" some.
1072 */
1073 unsigned long pool[2];
1074 unsigned int count;
1075
1076 /* Check to see if we're running on the wrong CPU due to hotplug. */
1077 local_irq_disable();
1078 if (fast_pool != this_cpu_ptr(&irq_randomness)) {
1079 local_irq_enable();
1080 return;
1081 }
1082
1083 /*
1084 * Copy the pool to the stack so that the mixer always has a
1085 * consistent view, before we reenable irqs again.
1086 */
1087 memcpy(pool, fast_pool->pool, sizeof(pool));
1088 count = fast_pool->count;
1089 fast_pool->count = 0;
1090 fast_pool->last = jiffies;
1091 local_irq_enable();
1092
1093 mix_pool_bytes(pool, sizeof(pool));
1094 credit_init_bits(clamp_t(unsigned int, (count & U16_MAX) / 64, 1, sizeof(pool) * 8));
1095
1096 memzero_explicit(pool, sizeof(pool));
1097}
1098
1099void add_interrupt_randomness(int irq)
1100{
1101 enum { MIX_INFLIGHT = 1U << 31 };
1102 unsigned long entropy = random_get_entropy();
1103 struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness);
1104 struct pt_regs *regs = get_irq_regs();
1105 unsigned int new_count;
1106
1107 fast_mix(fast_pool->pool, entropy,
1108 (regs ? instruction_pointer(regs) : _RET_IP_) ^ swab(irq));
1109 new_count = ++fast_pool->count;
1110
1111 if (new_count & MIX_INFLIGHT)
1112 return;
1113
1114 if (new_count < 1024 && !time_is_before_jiffies(fast_pool->last + HZ))
1115 return;
1116
1117 fast_pool->count |= MIX_INFLIGHT;
1118 if (!timer_pending(&fast_pool->mix)) {
1119 fast_pool->mix.expires = jiffies;
1120 add_timer_on(&fast_pool->mix, raw_smp_processor_id());
1121 }
1122}
1123EXPORT_SYMBOL_GPL(add_interrupt_randomness);
1124
1125/* There is one of these per entropy source */
1126struct timer_rand_state {
1127 unsigned long last_time;
1128 long last_delta, last_delta2;
1129};
1130
1131/*
1132 * This function adds entropy to the entropy "pool" by using timing
1133 * delays. It uses the timer_rand_state structure to make an estimate
1134 * of how many bits of entropy this call has added to the pool. The
1135 * value "num" is also added to the pool; it should somehow describe
1136 * the type of event that just happened.
1137 */
1138static void add_timer_randomness(struct timer_rand_state *state, unsigned int num)
1139{
1140 unsigned long entropy = random_get_entropy(), now = jiffies, flags;
1141 long delta, delta2, delta3;
1142 unsigned int bits;
1143
1144 /*
1145 * If we're in a hard IRQ, add_interrupt_randomness() will be called
1146 * sometime after, so mix into the fast pool.
1147 */
1148 if (in_hardirq()) {
1149 fast_mix(this_cpu_ptr(&irq_randomness)->pool, entropy, num);
1150 } else {
1151 spin_lock_irqsave(&input_pool.lock, flags);
1152 _mix_pool_bytes(&entropy, sizeof(entropy));
1153 _mix_pool_bytes(&num, sizeof(num));
1154 spin_unlock_irqrestore(&input_pool.lock, flags);
1155 }
1156
1157 if (crng_ready())
1158 return;
1159
1160 /*
1161 * Calculate number of bits of randomness we probably added.
1162 * We take into account the first, second and third-order deltas
1163 * in order to make our estimate.
1164 */
1165 delta = now - READ_ONCE(state->last_time);
1166 WRITE_ONCE(state->last_time, now);
1167
1168 delta2 = delta - READ_ONCE(state->last_delta);
1169 WRITE_ONCE(state->last_delta, delta);
1170
1171 delta3 = delta2 - READ_ONCE(state->last_delta2);
1172 WRITE_ONCE(state->last_delta2, delta2);
1173
1174 if (delta < 0)
1175 delta = -delta;
1176 if (delta2 < 0)
1177 delta2 = -delta2;
1178 if (delta3 < 0)
1179 delta3 = -delta3;
1180 if (delta > delta2)
1181 delta = delta2;
1182 if (delta > delta3)
1183 delta = delta3;
1184
1185 /*
1186 * delta is now minimum absolute delta. Round down by 1 bit
1187 * on general principles, and limit entropy estimate to 11 bits.
1188 */
1189 bits = min(fls(delta >> 1), 11);
1190
1191 /*
1192 * As mentioned above, if we're in a hard IRQ, add_interrupt_randomness()
1193 * will run after this, which uses a different crediting scheme of 1 bit
1194 * per every 64 interrupts. In order to let that function do accounting
1195 * close to the one in this function, we credit a full 64/64 bit per bit,
1196 * and then subtract one to account for the extra one added.
1197 */
1198 if (in_hardirq())
1199 this_cpu_ptr(&irq_randomness)->count += max(1u, bits * 64) - 1;
1200 else
1201 _credit_init_bits(bits);
1202}
1203
1204void add_input_randomness(unsigned int type, unsigned int code, unsigned int value)
1205{
1206 static unsigned char last_value;
1207 static struct timer_rand_state input_timer_state = { INITIAL_JIFFIES };
1208
1209 /* Ignore autorepeat and the like. */
1210 if (value == last_value)
1211 return;
1212
1213 last_value = value;
1214 add_timer_randomness(&input_timer_state,
1215 (type << 4) ^ code ^ (code >> 4) ^ value);
1216}
1217EXPORT_SYMBOL_GPL(add_input_randomness);
1218
1219#ifdef CONFIG_BLOCK
1220void add_disk_randomness(struct gendisk *disk)
1221{
1222 if (!disk || !disk->random)
1223 return;
1224 /* First major is 1, so we get >= 0x200 here. */
1225 add_timer_randomness(disk->random, 0x100 + disk_devt(disk));
1226}
1227EXPORT_SYMBOL_GPL(add_disk_randomness);
1228
1229void __cold rand_initialize_disk(struct gendisk *disk)
1230{
1231 struct timer_rand_state *state;
1232
1233 /*
1234 * If kzalloc returns null, we just won't use that entropy
1235 * source.
1236 */
1237 state = kzalloc_obj(struct timer_rand_state);
1238 if (state) {
1239 state->last_time = INITIAL_JIFFIES;
1240 disk->random = state;
1241 }
1242}
1243#endif
1244
1245struct entropy_timer_state {
1246 unsigned long entropy;
1247 struct timer_list timer;
1248 atomic_t samples;
1249 unsigned int samples_per_bit;
1250};
1251
1252/*
1253 * Each time the timer fires, we expect that we got an unpredictable jump in
1254 * the cycle counter. Even if the timer is running on another CPU, the timer
1255 * activity will be touching the stack of the CPU that is generating entropy.
1256 *
1257 * Note that we don't re-arm the timer in the timer itself - we are happy to be
1258 * scheduled away, since that just makes the load more complex, but we do not
1259 * want the timer to keep ticking unless the entropy loop is running.
1260 *
1261 * So the re-arming always happens in the entropy loop itself.
1262 */
1263static void __cold entropy_timer(struct timer_list *timer)
1264{
1265 struct entropy_timer_state *state = container_of(timer, struct entropy_timer_state, timer);
1266 unsigned long entropy = random_get_entropy();
1267
1268 mix_pool_bytes(&entropy, sizeof(entropy));
1269 if (atomic_inc_return(&state->samples) % state->samples_per_bit == 0)
1270 credit_init_bits(1);
1271}
1272
1273/*
1274 * If we have an actual cycle counter, see if we can generate enough entropy
1275 * with timing noise.
1276 */
1277static void __cold try_to_generate_entropy(void)
1278{
1279 enum { NUM_TRIAL_SAMPLES = 8192, MAX_SAMPLES_PER_BIT = HZ / 15 };
1280 u8 stack_bytes[sizeof(struct entropy_timer_state) + SMP_CACHE_BYTES - 1];
1281 struct entropy_timer_state *stack = PTR_ALIGN((void *)stack_bytes, SMP_CACHE_BYTES);
1282 unsigned int i, num_different = 0;
1283 unsigned long last = random_get_entropy();
1284 cpumask_var_t timer_cpus;
1285 int cpu = -1;
1286
1287 for (i = 0; i < NUM_TRIAL_SAMPLES - 1; ++i) {
1288 stack->entropy = random_get_entropy();
1289 if (stack->entropy != last)
1290 ++num_different;
1291 last = stack->entropy;
1292 }
1293 stack->samples_per_bit = DIV_ROUND_UP(NUM_TRIAL_SAMPLES, num_different + 1);
1294 if (stack->samples_per_bit > MAX_SAMPLES_PER_BIT)
1295 return;
1296
1297 atomic_set(&stack->samples, 0);
1298 timer_setup_on_stack(&stack->timer, entropy_timer, 0);
1299 if (!alloc_cpumask_var(&timer_cpus, GFP_KERNEL))
1300 goto out;
1301
1302 while (!crng_ready() && !signal_pending(current)) {
1303 /*
1304 * Check !timer_pending() and then ensure that any previous callback has finished
1305 * executing by checking timer_delete_sync_try(), before queueing the next one.
1306 */
1307 if (!timer_pending(&stack->timer) && timer_delete_sync_try(&stack->timer) >= 0) {
1308 unsigned int num_cpus;
1309
1310 /*
1311 * Preemption must be disabled here, both to read the current CPU number
1312 * and to avoid scheduling a timer on a dead CPU.
1313 */
1314 preempt_disable();
1315
1316 /* Only schedule callbacks on timer CPUs that are online. */
1317 cpumask_and(timer_cpus, housekeeping_cpumask(HK_TYPE_TIMER), cpu_online_mask);
1318 num_cpus = cpumask_weight(timer_cpus);
1319 /* In very bizarre case of misconfiguration, fallback to all online. */
1320 if (unlikely(num_cpus == 0)) {
1321 *timer_cpus = *cpu_online_mask;
1322 num_cpus = cpumask_weight(timer_cpus);
1323 }
1324
1325 /* Basic CPU round-robin, which avoids the current CPU. */
1326 do {
1327 cpu = cpumask_next(cpu, timer_cpus);
1328 if (cpu >= nr_cpu_ids)
1329 cpu = cpumask_first(timer_cpus);
1330 } while (cpu == smp_processor_id() && num_cpus > 1);
1331
1332 /* Expiring the timer at `jiffies` means it's the next tick. */
1333 stack->timer.expires = jiffies;
1334
1335 add_timer_on(&stack->timer, cpu);
1336
1337 preempt_enable();
1338 }
1339 mix_pool_bytes(&stack->entropy, sizeof(stack->entropy));
1340 schedule();
1341 stack->entropy = random_get_entropy();
1342 }
1343 mix_pool_bytes(&stack->entropy, sizeof(stack->entropy));
1344
1345 free_cpumask_var(timer_cpus);
1346out:
1347 timer_delete_sync(&stack->timer);
1348 timer_destroy_on_stack(&stack->timer);
1349}
1350
1351
1352/**********************************************************************
1353 *
1354 * Userspace reader/writer interfaces.
1355 *
1356 * getrandom(2) is the primary modern interface into the RNG and should
1357 * be used in preference to anything else.
1358 *
1359 * Reading from /dev/random has the same functionality as calling
1360 * getrandom(2) with flags=0. In earlier versions, however, it had
1361 * vastly different semantics and should therefore be avoided, to
1362 * prevent backwards compatibility issues.
1363 *
1364 * Reading from /dev/urandom has the same functionality as calling
1365 * getrandom(2) with flags=GRND_INSECURE. Because it does not block
1366 * waiting for the RNG to be ready, it should not be used.
1367 *
1368 * Writing to either /dev/random or /dev/urandom adds entropy to
1369 * the input pool but does not credit it.
1370 *
1371 * Polling on /dev/random indicates when the RNG is initialized, on
1372 * the read side, and when it wants new entropy, on the write side.
1373 *
1374 * Both /dev/random and /dev/urandom have the same set of ioctls for
1375 * adding entropy, getting the entropy count, zeroing the count, and
1376 * reseeding the crng.
1377 *
1378 **********************************************************************/
1379
1380SYSCALL_DEFINE3(getrandom, char __user *, ubuf, size_t, len, unsigned int, flags)
1381{
1382 struct iov_iter iter;
1383 int ret;
1384
1385 if (flags & ~(GRND_NONBLOCK | GRND_RANDOM | GRND_INSECURE))
1386 return -EINVAL;
1387
1388 /*
1389 * Requesting insecure and blocking randomness at the same time makes
1390 * no sense.
1391 */
1392 if ((flags & (GRND_INSECURE | GRND_RANDOM)) == (GRND_INSECURE | GRND_RANDOM))
1393 return -EINVAL;
1394
1395 if (!crng_ready() && !(flags & GRND_INSECURE)) {
1396 if (flags & GRND_NONBLOCK)
1397 return -EAGAIN;
1398 ret = wait_for_random_bytes();
1399 if (unlikely(ret))
1400 return ret;
1401 }
1402
1403 ret = import_ubuf(ITER_DEST, ubuf, len, &iter);
1404 if (unlikely(ret))
1405 return ret;
1406 return get_random_bytes_user(&iter);
1407}
1408
1409static __poll_t random_poll(struct file *file, poll_table *wait)
1410{
1411 poll_wait(file, &crng_init_wait, wait);
1412 return crng_ready() ? EPOLLIN | EPOLLRDNORM : EPOLLOUT | EPOLLWRNORM;
1413}
1414
1415static ssize_t write_pool_user(struct iov_iter *iter)
1416{
1417 u8 block[BLAKE2S_BLOCK_SIZE];
1418 ssize_t ret = 0;
1419 size_t copied;
1420
1421 if (unlikely(!iov_iter_count(iter)))
1422 return 0;
1423
1424 for (;;) {
1425 copied = copy_from_iter(block, sizeof(block), iter);
1426 ret += copied;
1427 mix_pool_bytes(block, copied);
1428 if (!iov_iter_count(iter) || copied != sizeof(block))
1429 break;
1430
1431 BUILD_BUG_ON(PAGE_SIZE % sizeof(block) != 0);
1432 if (ret % PAGE_SIZE == 0) {
1433 if (signal_pending(current))
1434 break;
1435 cond_resched();
1436 }
1437 }
1438
1439 memzero_explicit(block, sizeof(block));
1440 return ret ? ret : -EFAULT;
1441}
1442
1443static ssize_t random_write_iter(struct kiocb *kiocb, struct iov_iter *iter)
1444{
1445 return write_pool_user(iter);
1446}
1447
1448static ssize_t urandom_read_iter(struct kiocb *kiocb, struct iov_iter *iter)
1449{
1450 static int maxwarn = 10;
1451
1452 /*
1453 * Opportunistically attempt to initialize the RNG on platforms that
1454 * have fast cycle counters, but don't (for now) require it to succeed.
1455 */
1456 if (!crng_ready())
1457 try_to_generate_entropy();
1458
1459 if (!crng_ready()) {
1460 if (!ratelimit_disable && maxwarn <= 0)
1461 ratelimit_state_inc_miss(&urandom_warning);
1462 else if (ratelimit_disable || __ratelimit(&urandom_warning)) {
1463 --maxwarn;
1464 pr_notice("%s: uninitialized urandom read (%zu bytes read)\n",
1465 current->comm, iov_iter_count(iter));
1466 }
1467 }
1468
1469 return get_random_bytes_user(iter);
1470}
1471
1472static ssize_t random_read_iter(struct kiocb *kiocb, struct iov_iter *iter)
1473{
1474 int ret;
1475
1476 if (!crng_ready() &&
1477 ((kiocb->ki_flags & (IOCB_NOWAIT | IOCB_NOIO)) ||
1478 (kiocb->ki_filp->f_flags & O_NONBLOCK)))
1479 return -EAGAIN;
1480
1481 ret = wait_for_random_bytes();
1482 if (ret != 0)
1483 return ret;
1484 return get_random_bytes_user(iter);
1485}
1486
1487static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
1488{
1489 int __user *p = (int __user *)arg;
1490 int ent_count;
1491
1492 switch (cmd) {
1493 case RNDGETENTCNT:
1494 /* Inherently racy, no point locking. */
1495 if (put_user(input_pool.init_bits, p))
1496 return -EFAULT;
1497 return 0;
1498 case RNDADDTOENTCNT:
1499 if (!capable(CAP_SYS_ADMIN))
1500 return -EPERM;
1501 if (get_user(ent_count, p))
1502 return -EFAULT;
1503 if (ent_count < 0)
1504 return -EINVAL;
1505 credit_init_bits(ent_count);
1506 return 0;
1507 case RNDADDENTROPY: {
1508 struct iov_iter iter;
1509 ssize_t ret;
1510 int len;
1511
1512 if (!capable(CAP_SYS_ADMIN))
1513 return -EPERM;
1514 if (get_user(ent_count, p++))
1515 return -EFAULT;
1516 if (ent_count < 0)
1517 return -EINVAL;
1518 if (get_user(len, p++))
1519 return -EFAULT;
1520 ret = import_ubuf(ITER_SOURCE, p, len, &iter);
1521 if (unlikely(ret))
1522 return ret;
1523 ret = write_pool_user(&iter);
1524 if (unlikely(ret < 0))
1525 return ret;
1526 /* Since we're crediting, enforce that it was all written into the pool. */
1527 if (unlikely(ret != len))
1528 return -EFAULT;
1529 credit_init_bits(ent_count);
1530 return 0;
1531 }
1532 case RNDZAPENTCNT:
1533 case RNDCLEARPOOL:
1534 /* No longer has any effect. */
1535 if (!capable(CAP_SYS_ADMIN))
1536 return -EPERM;
1537 return 0;
1538 case RNDRESEEDCRNG:
1539 if (!capable(CAP_SYS_ADMIN))
1540 return -EPERM;
1541 if (!crng_ready())
1542 return -ENODATA;
1543 crng_reseed(NULL);
1544 return 0;
1545 default:
1546 return -EINVAL;
1547 }
1548}
1549
1550static int random_fasync(int fd, struct file *filp, int on)
1551{
1552 return fasync_helper(fd, filp, on, &fasync);
1553}
1554
1555const struct file_operations random_fops = {
1556 .read_iter = random_read_iter,
1557 .write_iter = random_write_iter,
1558 .poll = random_poll,
1559 .unlocked_ioctl = random_ioctl,
1560 .compat_ioctl = compat_ptr_ioctl,
1561 .fasync = random_fasync,
1562 .llseek = noop_llseek,
1563 .splice_read = copy_splice_read,
1564 .splice_write = iter_file_splice_write,
1565};
1566
1567const struct file_operations urandom_fops = {
1568 .read_iter = urandom_read_iter,
1569 .write_iter = random_write_iter,
1570 .unlocked_ioctl = random_ioctl,
1571 .compat_ioctl = compat_ptr_ioctl,
1572 .fasync = random_fasync,
1573 .llseek = noop_llseek,
1574 .splice_read = copy_splice_read,
1575 .splice_write = iter_file_splice_write,
1576};
1577
1578
1579/********************************************************************
1580 *
1581 * Sysctl interface.
1582 *
1583 * These are partly unused legacy knobs with dummy values to not break
1584 * userspace and partly still useful things. They are usually accessible
1585 * in /proc/sys/kernel/random/ and are as follows:
1586 *
1587 * - boot_id - a UUID representing the current boot.
1588 *
1589 * - uuid - a random UUID, different each time the file is read.
1590 *
1591 * - poolsize - the number of bits of entropy that the input pool can
1592 * hold, tied to the POOL_BITS constant.
1593 *
1594 * - entropy_avail - the number of bits of entropy currently in the
1595 * input pool. Always <= poolsize.
1596 *
1597 * - write_wakeup_threshold - the amount of entropy in the input pool
1598 * below which write polls to /dev/random will unblock, requesting
1599 * more entropy, tied to the POOL_READY_BITS constant. It is writable
1600 * to avoid breaking old userspaces, but writing to it does not
1601 * change any behavior of the RNG.
1602 *
1603 * - urandom_min_reseed_secs - fixed to the value CRNG_RESEED_INTERVAL.
1604 * It is writable to avoid breaking old userspaces, but writing
1605 * to it does not change any behavior of the RNG.
1606 *
1607 ********************************************************************/
1608
1609#ifdef CONFIG_SYSCTL
1610
1611#include <linux/sysctl.h>
1612
1613static int sysctl_random_min_urandom_seed = CRNG_RESEED_INTERVAL / HZ;
1614static int sysctl_random_write_wakeup_bits = POOL_READY_BITS;
1615static int sysctl_poolsize = POOL_BITS;
1616static u8 sysctl_bootid[UUID_SIZE];
1617
1618/*
1619 * This function is used to return both the bootid UUID, and random
1620 * UUID. The difference is in whether table->data is NULL; if it is,
1621 * then a new UUID is generated and returned to the user.
1622 */
1623static int proc_do_uuid(const struct ctl_table *table, int write, void *buf,
1624 size_t *lenp, loff_t *ppos)
1625{
1626 u8 tmp_uuid[UUID_SIZE], *uuid;
1627 char uuid_string[UUID_STRING_LEN + 1];
1628 struct ctl_table fake_table = {
1629 .data = uuid_string,
1630 .maxlen = UUID_STRING_LEN
1631 };
1632
1633 if (write)
1634 return -EPERM;
1635
1636 uuid = table->data;
1637 if (!uuid) {
1638 uuid = tmp_uuid;
1639 generate_random_uuid(uuid);
1640 } else {
1641 static DEFINE_SPINLOCK(bootid_spinlock);
1642
1643 spin_lock(&bootid_spinlock);
1644 if (!uuid[8])
1645 generate_random_uuid(uuid);
1646 spin_unlock(&bootid_spinlock);
1647 }
1648
1649 snprintf(uuid_string, sizeof(uuid_string), "%pU", uuid);
1650 return proc_dostring(&fake_table, 0, buf, lenp, ppos);
1651}
1652
1653/* The same as proc_dointvec, but writes don't change anything. */
1654static int proc_do_rointvec(const struct ctl_table *table, int write, void *buf,
1655 size_t *lenp, loff_t *ppos)
1656{
1657 return write ? 0 : proc_dointvec(table, 0, buf, lenp, ppos);
1658}
1659
1660static const struct ctl_table random_table[] = {
1661 {
1662 .procname = "poolsize",
1663 .data = &sysctl_poolsize,
1664 .maxlen = sizeof(int),
1665 .mode = 0444,
1666 .proc_handler = proc_dointvec,
1667 },
1668 {
1669 .procname = "entropy_avail",
1670 .data = &input_pool.init_bits,
1671 .maxlen = sizeof(int),
1672 .mode = 0444,
1673 .proc_handler = proc_dointvec,
1674 },
1675 {
1676 .procname = "write_wakeup_threshold",
1677 .data = &sysctl_random_write_wakeup_bits,
1678 .maxlen = sizeof(int),
1679 .mode = 0644,
1680 .proc_handler = proc_do_rointvec,
1681 },
1682 {
1683 .procname = "urandom_min_reseed_secs",
1684 .data = &sysctl_random_min_urandom_seed,
1685 .maxlen = sizeof(int),
1686 .mode = 0644,
1687 .proc_handler = proc_do_rointvec,
1688 },
1689 {
1690 .procname = "boot_id",
1691 .data = &sysctl_bootid,
1692 .mode = 0444,
1693 .proc_handler = proc_do_uuid,
1694 },
1695 {
1696 .procname = "uuid",
1697 .mode = 0444,
1698 .proc_handler = proc_do_uuid,
1699 },
1700};
1701
1702/*
1703 * random_init() is called before sysctl_init(),
1704 * so we cannot call register_sysctl_init() in random_init()
1705 */
1706static int __init random_sysctls_init(void)
1707{
1708 register_sysctl_init("kernel/random", random_table);
1709 return 0;
1710}
1711device_initcall(random_sysctls_init);
1712#endif