1 /* $NetBSD: kern_entropy.c,v 1.73 2025/03/11 14:30:28 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 2019 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Taylor R. Campbell.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * Entropy subsystem
34 *
35 * * Each CPU maintains a per-CPU entropy pool so that gathering
36 * entropy requires no interprocessor synchronization, except
37 * early at boot when we may be scrambling to gather entropy as
38 * soon as possible.
39 *
40 * - entropy_enter gathers entropy and never drops it on the
41 * floor, at the cost of sometimes having to do cryptography.
42 *
43 * - entropy_enter_intr gathers entropy or drops it on the
44 * floor, with low latency. Work to stir the pool or kick the
45 * housekeeping thread is scheduled in soft interrupts.
46 *
47 * * entropy_enter immediately enters into the global pool if it
48 * can transition to full entropy in one swell foop. Otherwise,
49 * it defers to a housekeeping thread that consolidates entropy,
50 * but only when the CPUs collectively have full entropy, in
51 * order to mitigate iterative-guessing attacks.
52 *
53 * * The entropy housekeeping thread continues to consolidate
54 * entropy even after we think we have full entropy, in case we
55 * are wrong, but is limited to one discretionary consolidation
56 * per minute, and only when new entropy is actually coming in,
57 * to limit performance impact.
58 *
59 * * The entropy epoch is the number that changes when we
60 * transition from partial entropy to full entropy, so that
61 * users can easily determine when to reseed. This also
62 * facilitates an operator explicitly causing everything to
63 * reseed by sysctl -w kern.entropy.consolidate=1.
64 *
65 * * Entropy depletion is available for testing (or if you're into
66 * that sort of thing), with sysctl -w kern.entropy.depletion=1;
67 * the logic to support it is small, to minimize chance of bugs.
68 *
69 * * While cold, a single global entropy pool is available for
70 * entering and extracting, serialized through splhigh/splx.
71 * The per-CPU entropy pool data structures are initialized in
72 * entropy_init and entropy_init_late (separated mainly for
73 * hysterical raisins at this point), but are not used until the
74 * system is warm, at which point access to the global entropy
75 * pool is limited to thread and softint context and serialized
76 * by E->lock.
77 */
78
79 #include <sys/cdefs.h>
80 __KERNEL_RCSID(0, "$NetBSD: kern_entropy.c,v 1.73 2025/03/11 14:30:28 riastradh Exp $");
81
82 #include <sys/param.h>
83 #include <sys/types.h>
84 #include <sys/atomic.h>
85 #include <sys/compat_stub.h>
86 #include <sys/condvar.h>
87 #include <sys/cpu.h>
88 #include <sys/entropy.h>
89 #include <sys/errno.h>
90 #include <sys/evcnt.h>
91 #include <sys/event.h>
92 #include <sys/file.h>
93 #include <sys/intr.h>
94 #include <sys/kauth.h>
95 #include <sys/kernel.h>
96 #include <sys/kmem.h>
97 #include <sys/kthread.h>
98 #include <sys/lwp.h>
99 #include <sys/module_hook.h>
100 #include <sys/mutex.h>
101 #include <sys/percpu.h>
102 #include <sys/poll.h>
103 #include <sys/proc.h>
104 #include <sys/queue.h>
105 #include <sys/reboot.h>
106 #include <sys/rnd.h> /* legacy kernel API */
107 #include <sys/rndio.h> /* userland ioctl interface */
108 #include <sys/rndsource.h> /* kernel rndsource driver API */
109 #include <sys/select.h>
110 #include <sys/selinfo.h>
111 #include <sys/sha1.h> /* for boot seed checksum */
112 #include <sys/stdint.h>
113 #include <sys/sysctl.h>
114 #include <sys/syslog.h>
115 #include <sys/systm.h>
116 #include <sys/time.h>
117 #include <sys/xcall.h>
118
119 #include <lib/libkern/entpool.h>
120
121 #include <machine/limits.h>
122
123 #ifdef __HAVE_CPU_COUNTER
124 #include <machine/cpu_counter.h>
125 #endif
126
127 #define MINENTROPYBYTES ENTROPY_CAPACITY
128 #define MINENTROPYBITS (MINENTROPYBYTES*NBBY)
129 #define MINSAMPLES (2*MINENTROPYBITS)
130
131 /*
132 * struct entropy_cpu
133 *
134 * Per-CPU entropy state. The pool is allocated separately
135 * because percpu(9) sometimes moves per-CPU objects around
136 * without zeroing them, which would lead to unwanted copies of
137 * sensitive secrets. The evcnt is allocated separately because
138 * evcnt(9) assumes it stays put in memory.
139 */
140 struct entropy_cpu {
141 struct entropy_cpu_evcnt {
142 struct evcnt softint;
143 struct evcnt intrdrop;
144 struct evcnt intrtrunc;
145 } *ec_evcnt;
146 struct entpool *ec_pool;
147 unsigned ec_bitspending;
148 unsigned ec_samplespending;
149 bool ec_locked;
150 };
151
152 /*
153 * struct entropy_cpu_lock
154 *
155 * State for locking the per-CPU entropy state.
156 */
157 struct entropy_cpu_lock {
158 int ecl_s;
159 long ecl_pctr;
160 };
161
162 /*
163 * struct rndsource_cpu
164 *
165 * Per-CPU rndsource state.
166 */
167 struct rndsource_cpu {
168 unsigned rc_entropybits;
169 unsigned rc_timesamples;
170 unsigned rc_datasamples;
171 rnd_delta_t rc_timedelta;
172 };
173
174 /*
175 * entropy_global (a.k.a. E for short in this file)
176 *
177 * Global entropy state. Writes protected by the global lock.
178 * Some fields, marked (A), can be read outside the lock, and are
179 * maintained with atomic_load/store_relaxed.
180 */
181 struct {
182 kmutex_t lock; /* covers all global state */
183 struct entpool pool; /* global pool for extraction */
184 unsigned bitsneeded; /* (A) needed globally */
185 unsigned bitspending; /* pending in per-CPU pools */
186 unsigned samplesneeded; /* (A) needed globally */
187 unsigned samplespending; /* pending in per-CPU pools */
188 unsigned timestamp; /* (A) time of last consolidation */
189 unsigned epoch; /* (A) changes when needed -> 0 */
190 kcondvar_t cv; /* notifies state changes */
191 struct selinfo selq; /* notifies needed -> 0 */
192 struct lwp *sourcelock; /* lock on list of sources */
193 kcondvar_t sourcelock_cv; /* notifies sourcelock release */
194 LIST_HEAD(,krndsource) sources; /* list of entropy sources */
195 bool consolidate; /* kick thread to consolidate */
196 bool seed_rndsource; /* true if seed source is attached */
197 bool seeded; /* true if seed file already loaded */
198 } entropy_global __cacheline_aligned = {
199 /* Fields that must be initialized when the kernel is loaded. */
200 .bitsneeded = MINENTROPYBITS,
201 .samplesneeded = MINSAMPLES,
202 .epoch = (unsigned)-1, /* -1 means entropy never consolidated */
203 .sources = LIST_HEAD_INITIALIZER(entropy_global.sources),
204 };
205
206 #define E (&entropy_global) /* declutter */
207
208 /* Read-mostly globals */
209 static struct percpu *entropy_percpu __read_mostly; /* struct entropy_cpu */
210 static void *entropy_sih __read_mostly; /* softint handler */
211 static struct lwp *entropy_lwp __read_mostly; /* housekeeping thread */
212
213 static struct krndsource seed_rndsource __read_mostly;
214
215 /*
216 * Event counters
217 *
218 * Must be careful with adding these because they can serve as
219 * side channels.
220 */
221 static struct evcnt entropy_discretionary_evcnt =
222 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "discretionary");
223 EVCNT_ATTACH_STATIC(entropy_discretionary_evcnt);
224 static struct evcnt entropy_immediate_evcnt =
225 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "immediate");
226 EVCNT_ATTACH_STATIC(entropy_immediate_evcnt);
227 static struct evcnt entropy_partial_evcnt =
228 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "partial");
229 EVCNT_ATTACH_STATIC(entropy_partial_evcnt);
230 static struct evcnt entropy_consolidate_evcnt =
231 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "consolidate");
232 EVCNT_ATTACH_STATIC(entropy_consolidate_evcnt);
233 static struct evcnt entropy_extract_fail_evcnt =
234 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "extract fail");
235 EVCNT_ATTACH_STATIC(entropy_extract_fail_evcnt);
236 static struct evcnt entropy_request_evcnt =
237 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "request");
238 EVCNT_ATTACH_STATIC(entropy_request_evcnt);
239 static struct evcnt entropy_deplete_evcnt =
240 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "deplete");
241 EVCNT_ATTACH_STATIC(entropy_deplete_evcnt);
242 static struct evcnt entropy_notify_evcnt =
243 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "notify");
244 EVCNT_ATTACH_STATIC(entropy_notify_evcnt);
245
246 /* Sysctl knobs */
247 static bool entropy_collection = 1;
248 static bool entropy_depletion = 0; /* Silly! */
249
250 static const struct sysctlnode *entropy_sysctlroot;
251 static struct sysctllog *entropy_sysctllog;
252
253 /* Forward declarations */
254 static void entropy_init_cpu(void *, void *, struct cpu_info *);
255 static void entropy_fini_cpu(void *, void *, struct cpu_info *);
256 static void entropy_account_cpu(struct entropy_cpu *);
257 static void entropy_enter(const void *, size_t, unsigned, bool);
258 static bool entropy_enter_intr(const void *, size_t, unsigned, bool);
259 static void entropy_softintr(void *);
260 static void entropy_thread(void *);
261 static bool entropy_pending(void);
262 static void entropy_pending_cpu(void *, void *, struct cpu_info *);
263 static void entropy_do_consolidate(void);
264 static void entropy_consolidate_xc(void *, void *);
265 static void entropy_notify(void);
266 static int sysctl_entropy_consolidate(SYSCTLFN_ARGS);
267 static int sysctl_entropy_gather(SYSCTLFN_ARGS);
268 static void filt_entropy_read_detach(struct knote *);
269 static int filt_entropy_read_event(struct knote *, long);
270 static int entropy_request(size_t, int);
271 static void rnd_add_data_internal(struct krndsource *, const void *,
272 uint32_t, uint32_t, bool);
273 static void rnd_add_data_1(struct krndsource *, const void *, uint32_t,
274 uint32_t, bool, uint32_t, bool);
275 static unsigned rndsource_entropybits(struct krndsource *);
276 static void rndsource_entropybits_cpu(void *, void *, struct cpu_info *);
277 static void rndsource_to_user(struct krndsource *, rndsource_t *);
278 static void rndsource_to_user_est(struct krndsource *, rndsource_est_t *);
279 static void rndsource_to_user_est_cpu(void *, void *, struct cpu_info *);
280
281 /*
282 * entropy_timer()
283 *
284 * Cycle counter, time counter, or anything that changes a wee bit
285 * unpredictably.
286 */
287 static inline uint32_t
entropy_timer(void)288 entropy_timer(void)
289 {
290 struct bintime bt;
291 uint32_t v;
292
293 /* If we have a CPU cycle counter, use the low 32 bits. */
294 #ifdef __HAVE_CPU_COUNTER
295 if (__predict_true(cpu_hascounter()))
296 return cpu_counter32();
297 #endif /* __HAVE_CPU_COUNTER */
298
299 /* If we're cold, tough. Can't binuptime while cold. */
300 if (__predict_false(cold))
301 return 0;
302
303 /* Fold the 128 bits of binuptime into 32 bits. */
304 binuptime(&bt);
305 v = bt.frac;
306 v ^= bt.frac >> 32;
307 v ^= bt.sec;
308 v ^= bt.sec >> 32;
309 return v;
310 }
311
312 static void
attach_seed_rndsource(void)313 attach_seed_rndsource(void)
314 {
315
316 KASSERT(!cpu_intr_p());
317 KASSERT(!cpu_softintr_p());
318 KASSERT(cold);
319
320 /*
321 * First called no later than entropy_init, while we are still
322 * single-threaded, so no need for RUN_ONCE.
323 */
324 if (E->seed_rndsource)
325 return;
326
327 rnd_attach_source(&seed_rndsource, "seed", RND_TYPE_UNKNOWN,
328 RND_FLAG_COLLECT_VALUE);
329 E->seed_rndsource = true;
330 }
331
332 /*
333 * entropy_init()
334 *
335 * Initialize the entropy subsystem. Panic on failure.
336 *
337 * Requires percpu(9) and sysctl(9) to be initialized. Must run
338 * while cold.
339 */
340 static void
entropy_init(void)341 entropy_init(void)
342 {
343 uint32_t extra[2];
344 struct krndsource *rs;
345 unsigned i = 0;
346
347 KASSERT(cold);
348
349 /* Grab some cycle counts early at boot. */
350 extra[i++] = entropy_timer();
351
352 /* Run the entropy pool cryptography self-test. */
353 if (entpool_selftest() == -1)
354 panic("entropy pool crypto self-test failed");
355
356 /* Create the sysctl directory. */
357 sysctl_createv(&entropy_sysctllog, 0, NULL, &entropy_sysctlroot,
358 CTLFLAG_PERMANENT, CTLTYPE_NODE, "entropy",
359 SYSCTL_DESCR("Entropy (random number sources) options"),
360 NULL, 0, NULL, 0,
361 CTL_KERN, KERN_ENTROPY, CTL_EOL);
362
363 /* Create the sysctl knobs. */
364 /* XXX These shouldn't be writable at securelevel>0. */
365 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
366 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "collection",
367 SYSCTL_DESCR("Automatically collect entropy from hardware"),
368 NULL, 0, &entropy_collection, 0, CTL_CREATE, CTL_EOL);
369 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
370 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "depletion",
371 SYSCTL_DESCR("`Deplete' entropy pool when observed"),
372 NULL, 0, &entropy_depletion, 0, CTL_CREATE, CTL_EOL);
373 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
374 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, "consolidate",
375 SYSCTL_DESCR("Trigger entropy consolidation now"),
376 sysctl_entropy_consolidate, 0, NULL, 0, CTL_CREATE, CTL_EOL);
377 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
378 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, "gather",
379 SYSCTL_DESCR("Trigger entropy gathering from sources now"),
380 sysctl_entropy_gather, 0, NULL, 0, CTL_CREATE, CTL_EOL);
381 /* XXX These should maybe not be readable at securelevel>0. */
382 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
383 CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
384 "needed",
385 SYSCTL_DESCR("Systemwide entropy deficit (bits of entropy)"),
386 NULL, 0, &E->bitsneeded, 0, CTL_CREATE, CTL_EOL);
387 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
388 CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
389 "pending",
390 SYSCTL_DESCR("Number of bits of entropy pending on CPUs"),
391 NULL, 0, &E->bitspending, 0, CTL_CREATE, CTL_EOL);
392 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
393 CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
394 "samplesneeded",
395 SYSCTL_DESCR("Systemwide entropy deficit (samples)"),
396 NULL, 0, &E->samplesneeded, 0, CTL_CREATE, CTL_EOL);
397 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
398 CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
399 "samplespending",
400 SYSCTL_DESCR("Number of samples pending on CPUs"),
401 NULL, 0, &E->samplespending, 0, CTL_CREATE, CTL_EOL);
402 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
403 CTLFLAG_PERMANENT|CTLFLAG_READONLY, CTLTYPE_INT,
404 "epoch", SYSCTL_DESCR("Entropy epoch"),
405 NULL, 0, &E->epoch, 0, KERN_ENTROPY_EPOCH, CTL_EOL);
406
407 /* Initialize the global state for multithreaded operation. */
408 mutex_init(&E->lock, MUTEX_DEFAULT, IPL_SOFTSERIAL);
409 cv_init(&E->cv, "entropy");
410 selinit(&E->selq);
411 cv_init(&E->sourcelock_cv, "entsrclock");
412
413 /* Make sure the seed source is attached. */
414 attach_seed_rndsource();
415
416 /* Note if the bootloader didn't provide a seed. */
417 if (!E->seeded)
418 aprint_debug("entropy: no seed from bootloader\n");
419
420 /* Allocate the per-CPU records for all early entropy sources. */
421 LIST_FOREACH(rs, &E->sources, list)
422 rs->state = percpu_alloc(sizeof(struct rndsource_cpu));
423
424 /* Allocate and initialize the per-CPU state. */
425 entropy_percpu = percpu_create(sizeof(struct entropy_cpu),
426 entropy_init_cpu, entropy_fini_cpu, NULL);
427
428 /* Enter the boot cycle count to get started. */
429 extra[i++] = entropy_timer();
430 KASSERT(i == __arraycount(extra));
431 entropy_enter(extra, sizeof extra, /*nbits*/0, /*count*/false);
432 explicit_memset(extra, 0, sizeof extra);
433 }
434
435 /*
436 * entropy_init_late()
437 *
438 * Late initialization. Panic on failure.
439 *
440 * Requires CPUs to have been detected and LWPs to have started.
441 * Must run while cold.
442 */
443 static void
entropy_init_late(void)444 entropy_init_late(void)
445 {
446 int error;
447
448 KASSERT(cold);
449
450 /*
451 * Establish the softint at the highest softint priority level.
452 * Must happen after CPU detection.
453 */
454 entropy_sih = softint_establish(SOFTINT_SERIAL|SOFTINT_MPSAFE,
455 &entropy_softintr, NULL);
456 if (entropy_sih == NULL)
457 panic("unable to establish entropy softint");
458
459 /*
460 * Create the entropy housekeeping thread. Must happen after
461 * lwpinit.
462 */
463 error = kthread_create(PRI_NONE, KTHREAD_MPSAFE|KTHREAD_TS, NULL,
464 entropy_thread, NULL, &entropy_lwp, "entbutler");
465 if (error)
466 panic("unable to create entropy housekeeping thread: %d",
467 error);
468 }
469
470 /*
471 * entropy_init_cpu(ptr, cookie, ci)
472 *
473 * percpu(9) constructor for per-CPU entropy pool.
474 */
475 static void
entropy_init_cpu(void * ptr,void * cookie,struct cpu_info * ci)476 entropy_init_cpu(void *ptr, void *cookie, struct cpu_info *ci)
477 {
478 struct entropy_cpu *ec = ptr;
479 const char *cpuname;
480
481 ec->ec_evcnt = kmem_alloc(sizeof(*ec->ec_evcnt), KM_SLEEP);
482 ec->ec_pool = kmem_zalloc(sizeof(*ec->ec_pool), KM_SLEEP);
483 ec->ec_bitspending = 0;
484 ec->ec_samplespending = 0;
485 ec->ec_locked = false;
486
487 /* XXX ci_cpuname may not be initialized early enough. */
488 cpuname = ci->ci_cpuname[0] == '\0' ? "cpu0" : ci->ci_cpuname;
489 evcnt_attach_dynamic(&ec->ec_evcnt->softint, EVCNT_TYPE_MISC, NULL,
490 cpuname, "entropy softint");
491 evcnt_attach_dynamic(&ec->ec_evcnt->intrdrop, EVCNT_TYPE_MISC, NULL,
492 cpuname, "entropy intrdrop");
493 evcnt_attach_dynamic(&ec->ec_evcnt->intrtrunc, EVCNT_TYPE_MISC, NULL,
494 cpuname, "entropy intrtrunc");
495 }
496
497 /*
498 * entropy_fini_cpu(ptr, cookie, ci)
499 *
500 * percpu(9) destructor for per-CPU entropy pool.
501 */
502 static void
entropy_fini_cpu(void * ptr,void * cookie,struct cpu_info * ci)503 entropy_fini_cpu(void *ptr, void *cookie, struct cpu_info *ci)
504 {
505 struct entropy_cpu *ec = ptr;
506
507 /*
508 * Zero any lingering data. Disclosure of the per-CPU pool
509 * shouldn't retroactively affect the security of any keys
510 * generated, because entpool(9) erases whatever we have just
511 * drawn out of any pool, but better safe than sorry.
512 */
513 explicit_memset(ec->ec_pool, 0, sizeof(*ec->ec_pool));
514
515 evcnt_detach(&ec->ec_evcnt->intrtrunc);
516 evcnt_detach(&ec->ec_evcnt->intrdrop);
517 evcnt_detach(&ec->ec_evcnt->softint);
518
519 kmem_free(ec->ec_pool, sizeof(*ec->ec_pool));
520 kmem_free(ec->ec_evcnt, sizeof(*ec->ec_evcnt));
521 }
522
523 /*
524 * ec = entropy_cpu_get(&lock)
525 * entropy_cpu_put(&lock, ec)
526 *
527 * Lock and unlock the per-CPU entropy state. This only prevents
528 * access on the same CPU -- by hard interrupts, by soft
529 * interrupts, or by other threads.
530 *
531 * Blocks soft interrupts and preemption altogether; doesn't block
532 * hard interrupts, but causes samples in hard interrupts to be
533 * dropped.
534 */
535 static struct entropy_cpu *
entropy_cpu_get(struct entropy_cpu_lock * lock)536 entropy_cpu_get(struct entropy_cpu_lock *lock)
537 {
538 struct entropy_cpu *ec;
539
540 ec = percpu_getref(entropy_percpu);
541 lock->ecl_s = splsoftserial();
542 KASSERT(!ec->ec_locked);
543 ec->ec_locked = true;
544 lock->ecl_pctr = lwp_pctr();
545 __insn_barrier();
546
547 return ec;
548 }
549
550 static void
entropy_cpu_put(struct entropy_cpu_lock * lock,struct entropy_cpu * ec)551 entropy_cpu_put(struct entropy_cpu_lock *lock, struct entropy_cpu *ec)
552 {
553
554 KASSERT(ec == percpu_getptr_remote(entropy_percpu, curcpu()));
555 KASSERT(ec->ec_locked);
556
557 __insn_barrier();
558 KASSERT(lock->ecl_pctr == lwp_pctr());
559 ec->ec_locked = false;
560 splx(lock->ecl_s);
561 percpu_putref(entropy_percpu);
562 }
563
564 /*
565 * entropy_seed(seed)
566 *
567 * Seed the entropy pool with seed. Meant to be called as early
568 * as possible by the bootloader; may be called before or after
569 * entropy_init. Must be called before system reaches userland.
570 * Must be called in thread or soft interrupt context, not in hard
571 * interrupt context. Must be called at most once.
572 *
573 * Overwrites the seed in place. Caller may then free the memory.
574 */
575 static void
entropy_seed(rndsave_t * seed)576 entropy_seed(rndsave_t *seed)
577 {
578 SHA1_CTX ctx;
579 uint8_t digest[SHA1_DIGEST_LENGTH];
580 bool seeded;
581
582 KASSERT(!cpu_intr_p());
583 KASSERT(!cpu_softintr_p());
584 KASSERT(cold);
585
586 /*
587 * Verify the checksum. If the checksum fails, take the data
588 * but ignore the entropy estimate -- the file may have been
589 * incompletely written with garbage, which is harmless to add
590 * but may not be as unpredictable as alleged.
591 */
592 SHA1Init(&ctx);
593 SHA1Update(&ctx, (const void *)&seed->entropy, sizeof(seed->entropy));
594 SHA1Update(&ctx, seed->data, sizeof(seed->data));
595 SHA1Final(digest, &ctx);
596 CTASSERT(sizeof(seed->digest) == sizeof(digest));
597 if (!consttime_memequal(digest, seed->digest, sizeof(digest))) {
598 printf("entropy: invalid seed checksum\n");
599 seed->entropy = 0;
600 }
601 explicit_memset(&ctx, 0, sizeof ctx);
602 explicit_memset(digest, 0, sizeof digest);
603
604 /*
605 * If the entropy is insensibly large, try byte-swapping.
606 * Otherwise assume the file is corrupted and act as though it
607 * has zero entropy.
608 */
609 if (howmany(seed->entropy, NBBY) > sizeof(seed->data)) {
610 seed->entropy = bswap32(seed->entropy);
611 if (howmany(seed->entropy, NBBY) > sizeof(seed->data))
612 seed->entropy = 0;
613 }
614
615 /* Make sure the seed source is attached. */
616 attach_seed_rndsource();
617
618 /* Test and set E->seeded. */
619 seeded = E->seeded;
620 E->seeded = (seed->entropy > 0);
621
622 /*
623 * If we've been seeded, may be re-entering the same seed
624 * (e.g., bootloader vs module init, or something). No harm in
625 * entering it twice, but it contributes no additional entropy.
626 */
627 if (seeded) {
628 printf("entropy: double-seeded by bootloader\n");
629 seed->entropy = 0;
630 } else {
631 printf("entropy: entering seed from bootloader"
632 " with %u bits of entropy\n", (unsigned)seed->entropy);
633 }
634
635 /* Enter it into the pool and promptly zero it. */
636 rnd_add_data(&seed_rndsource, seed->data, sizeof(seed->data),
637 seed->entropy);
638 explicit_memset(seed, 0, sizeof(*seed));
639 }
640
641 /*
642 * entropy_bootrequest()
643 *
644 * Request entropy from all sources at boot, once config is
645 * complete and interrupts are running but we are still cold.
646 */
647 void
entropy_bootrequest(void)648 entropy_bootrequest(void)
649 {
650 int error;
651
652 KASSERT(!cpu_intr_p());
653 KASSERT(!cpu_softintr_p());
654 KASSERT(cold);
655
656 /*
657 * Request enough to satisfy the maximum entropy shortage.
658 * This is harmless overkill if the bootloader provided a seed.
659 */
660 error = entropy_request(MINENTROPYBYTES, ENTROPY_WAIT);
661 KASSERTMSG(error == 0, "error=%d", error);
662 }
663
664 /*
665 * entropy_epoch()
666 *
667 * Returns the current entropy epoch. If this changes, you should
668 * reseed. If -1, means system entropy has not yet reached full
669 * entropy or been explicitly consolidated; never reverts back to
670 * -1. Never zero, so you can always use zero as an uninitialized
671 * sentinel value meaning `reseed ASAP'.
672 *
673 * Usage model:
674 *
675 * struct foo {
676 * struct crypto_prng prng;
677 * unsigned epoch;
678 * } *foo;
679 *
680 * unsigned epoch = entropy_epoch();
681 * if (__predict_false(epoch != foo->epoch)) {
682 * uint8_t seed[32];
683 * if (entropy_extract(seed, sizeof seed, 0) != 0)
684 * warn("no entropy");
685 * crypto_prng_reseed(&foo->prng, seed, sizeof seed);
686 * foo->epoch = epoch;
687 * }
688 */
689 unsigned
entropy_epoch(void)690 entropy_epoch(void)
691 {
692
693 /*
694 * Unsigned int, so no need for seqlock for an atomic read, but
695 * make sure we read it afresh each time.
696 */
697 return atomic_load_relaxed(&E->epoch);
698 }
699
700 /*
701 * entropy_ready()
702 *
703 * True if the entropy pool has full entropy.
704 */
705 bool
entropy_ready(void)706 entropy_ready(void)
707 {
708
709 return atomic_load_relaxed(&E->bitsneeded) == 0;
710 }
711
712 /*
713 * entropy_account_cpu(ec)
714 *
715 * Consider whether to consolidate entropy into the global pool
716 * after we just added some into the current CPU's pending pool.
717 *
718 * - If this CPU can provide enough entropy now, do so.
719 *
720 * - If this and whatever else is available on other CPUs can
721 * provide enough entropy, kick the consolidation thread.
722 *
723 * - Otherwise, do as little as possible, except maybe consolidate
724 * entropy at most once a minute.
725 *
726 * Caller must be bound to a CPU and therefore have exclusive
727 * access to ec. Will acquire and release the global lock.
728 */
729 static void
entropy_account_cpu(struct entropy_cpu * ec)730 entropy_account_cpu(struct entropy_cpu *ec)
731 {
732 struct entropy_cpu_lock lock;
733 struct entropy_cpu *ec0;
734 unsigned bitsdiff, samplesdiff;
735
736 KASSERT(!cpu_intr_p());
737 KASSERT(!cold);
738 KASSERT(curlwp->l_pflag & LP_BOUND);
739
740 /*
741 * If there's no entropy needed, and entropy has been
742 * consolidated in the last minute, do nothing.
743 */
744 if (__predict_true(atomic_load_relaxed(&E->bitsneeded) == 0) &&
745 __predict_true(!atomic_load_relaxed(&entropy_depletion)) &&
746 __predict_true((time_uptime - E->timestamp) <= 60))
747 return;
748
749 /*
750 * Consider consolidation, under the global lock and with the
751 * per-CPU state locked.
752 */
753 mutex_enter(&E->lock);
754 ec0 = entropy_cpu_get(&lock);
755 KASSERT(ec0 == ec);
756
757 if (ec->ec_bitspending == 0 && ec->ec_samplespending == 0) {
758 /* Raced with consolidation xcall. Nothing to do. */
759 } else if (E->bitsneeded != 0 && E->bitsneeded <= ec->ec_bitspending) {
760 /*
761 * If we have not yet attained full entropy but we can
762 * now, do so. This way we disseminate entropy
763 * promptly when it becomes available early at boot;
764 * otherwise we leave it to the entropy consolidation
765 * thread, which is rate-limited to mitigate side
766 * channels and abuse.
767 */
768 uint8_t buf[ENTPOOL_CAPACITY];
769
770 /* Transfer from the local pool to the global pool. */
771 entpool_extract(ec->ec_pool, buf, sizeof buf);
772 entpool_enter(&E->pool, buf, sizeof buf);
773 atomic_store_relaxed(&ec->ec_bitspending, 0);
774 atomic_store_relaxed(&ec->ec_samplespending, 0);
775 atomic_store_relaxed(&E->bitsneeded, 0);
776 atomic_store_relaxed(&E->samplesneeded, 0);
777
778 /* Notify waiters that we now have full entropy. */
779 entropy_notify();
780 entropy_immediate_evcnt.ev_count++;
781 } else {
782 /* Determine how much we can add to the global pool. */
783 KASSERTMSG(E->bitspending <= MINENTROPYBITS,
784 "E->bitspending=%u", E->bitspending);
785 bitsdiff = MIN(ec->ec_bitspending,
786 MINENTROPYBITS - E->bitspending);
787 KASSERTMSG(E->samplespending <= MINSAMPLES,
788 "E->samplespending=%u", E->samplespending);
789 samplesdiff = MIN(ec->ec_samplespending,
790 MINSAMPLES - E->samplespending);
791
792 /*
793 * This should make a difference unless we are already
794 * saturated.
795 */
796 KASSERTMSG((bitsdiff || samplesdiff ||
797 E->bitspending == MINENTROPYBITS ||
798 E->samplespending == MINSAMPLES),
799 "bitsdiff=%u E->bitspending=%u ec->ec_bitspending=%u"
800 "samplesdiff=%u E->samplespending=%u"
801 " ec->ec_samplespending=%u"
802 " minentropybits=%u minsamples=%u",
803 bitsdiff, E->bitspending, ec->ec_bitspending,
804 samplesdiff, E->samplespending, ec->ec_samplespending,
805 (unsigned)MINENTROPYBITS, (unsigned)MINSAMPLES);
806
807 /* Add to the global, subtract from the local. */
808 E->bitspending += bitsdiff;
809 KASSERTMSG(E->bitspending <= MINENTROPYBITS,
810 "E->bitspending=%u", E->bitspending);
811 atomic_store_relaxed(&ec->ec_bitspending,
812 ec->ec_bitspending - bitsdiff);
813
814 E->samplespending += samplesdiff;
815 KASSERTMSG(E->samplespending <= MINSAMPLES,
816 "E->samplespending=%u", E->samplespending);
817 atomic_store_relaxed(&ec->ec_samplespending,
818 ec->ec_samplespending - samplesdiff);
819
820 /* One or the other must have gone up from zero. */
821 KASSERT(E->bitspending || E->samplespending);
822
823 if (E->bitsneeded <= E->bitspending ||
824 E->samplesneeded <= E->samplespending) {
825 /*
826 * Enough bits or at least samples between all
827 * the per-CPU pools. Leave a note for the
828 * housekeeping thread to consolidate entropy
829 * next time it wakes up -- and wake it up if
830 * this is the first time, to speed things up.
831 *
832 * If we don't need any entropy, this doesn't
833 * mean much, but it is the only time we ever
834 * gather additional entropy in case the
835 * accounting has been overly optimistic. This
836 * happens at most once a minute, so there's
837 * negligible performance cost.
838 */
839 E->consolidate = true;
840 if (E->epoch == (unsigned)-1)
841 cv_broadcast(&E->cv);
842 if (E->bitsneeded == 0)
843 entropy_discretionary_evcnt.ev_count++;
844 } else {
845 /* Can't get full entropy. Keep gathering. */
846 entropy_partial_evcnt.ev_count++;
847 }
848 }
849
850 entropy_cpu_put(&lock, ec);
851 mutex_exit(&E->lock);
852 }
853
854 /*
855 * entropy_enter_early(buf, len, nbits)
856 *
857 * Do entropy bookkeeping globally, before we have established
858 * per-CPU pools. Enter directly into the global pool in the hope
859 * that we enter enough before the first entropy_extract to thwart
860 * iterative-guessing attacks; entropy_extract will warn if not.
861 */
862 static void
entropy_enter_early(const void * buf,size_t len,unsigned nbits)863 entropy_enter_early(const void *buf, size_t len, unsigned nbits)
864 {
865 bool notify = false;
866 int s;
867
868 KASSERT(cold);
869
870 /*
871 * We're early at boot before multithreading and multi-CPU
872 * operation, and we don't have softints yet to defer
873 * processing from interrupt context, so we have to enter the
874 * samples directly into the global pool. But interrupts may
875 * be enabled, and we enter this path from interrupt context,
876 * so block interrupts until we're done.
877 */
878 s = splhigh();
879
880 /* Enter it into the pool. */
881 entpool_enter(&E->pool, buf, len);
882
883 /*
884 * Decide whether to notify reseed -- we will do so if either:
885 * (a) we transition from partial entropy to full entropy, or
886 * (b) we get a batch of full entropy all at once.
887 * We don't count timing samples because we assume, while cold,
888 * there's not likely to be much jitter yet.
889 */
890 notify |= (E->bitsneeded && E->bitsneeded <= nbits);
891 notify |= (nbits >= MINENTROPYBITS);
892
893 /*
894 * Subtract from the needed count and notify if appropriate.
895 * We don't count samples here because entropy_timer might
896 * still be returning zero at this point if there's no CPU
897 * cycle counter.
898 */
899 E->bitsneeded -= MIN(E->bitsneeded, nbits);
900 if (notify) {
901 entropy_notify();
902 entropy_immediate_evcnt.ev_count++;
903 }
904
905 splx(s);
906 }
907
908 /*
909 * entropy_enter(buf, len, nbits, count)
910 *
911 * Enter len bytes of data from buf into the system's entropy
912 * pool, stirring as necessary when the internal buffer fills up.
913 * nbits is a lower bound on the number of bits of entropy in the
914 * process that led to this sample.
915 */
916 static void
entropy_enter(const void * buf,size_t len,unsigned nbits,bool count)917 entropy_enter(const void *buf, size_t len, unsigned nbits, bool count)
918 {
919 struct entropy_cpu_lock lock;
920 struct entropy_cpu *ec;
921 unsigned bitspending, samplespending;
922 int bound;
923
924 KASSERTMSG(!cpu_intr_p(),
925 "use entropy_enter_intr from interrupt context");
926 KASSERTMSG(howmany(nbits, NBBY) <= len,
927 "impossible entropy rate: %u bits in %zu-byte string", nbits, len);
928
929 /*
930 * If we're still cold, just use entropy_enter_early to put
931 * samples directly into the global pool.
932 */
933 if (__predict_false(cold)) {
934 entropy_enter_early(buf, len, nbits);
935 return;
936 }
937
938 /*
939 * Bind ourselves to the current CPU so we don't switch CPUs
940 * between entering data into the current CPU's pool (and
941 * updating the pending count) and transferring it to the
942 * global pool in entropy_account_cpu.
943 */
944 bound = curlwp_bind();
945
946 /*
947 * With the per-CPU state locked, enter into the per-CPU pool
948 * and count up what we can add.
949 *
950 * We don't count samples while cold because entropy_timer
951 * might still be returning zero if there's no CPU cycle
952 * counter.
953 */
954 ec = entropy_cpu_get(&lock);
955 entpool_enter(ec->ec_pool, buf, len);
956 bitspending = ec->ec_bitspending;
957 bitspending += MIN(MINENTROPYBITS - bitspending, nbits);
958 atomic_store_relaxed(&ec->ec_bitspending, bitspending);
959 samplespending = ec->ec_samplespending;
960 if (__predict_true(count)) {
961 samplespending += MIN(MINSAMPLES - samplespending, 1);
962 atomic_store_relaxed(&ec->ec_samplespending, samplespending);
963 }
964 entropy_cpu_put(&lock, ec);
965
966 /* Consolidate globally if appropriate based on what we added. */
967 if (bitspending > 0 || samplespending >= MINSAMPLES)
968 entropy_account_cpu(ec);
969
970 curlwp_bindx(bound);
971 }
972
973 /*
974 * entropy_enter_intr(buf, len, nbits, count)
975 *
976 * Enter up to len bytes of data from buf into the system's
977 * entropy pool without stirring. nbits is a lower bound on the
978 * number of bits of entropy in the process that led to this
979 * sample. If the sample could be entered completely, assume
980 * nbits of entropy pending; otherwise assume none, since we don't
981 * know whether some parts of the sample are constant, for
982 * instance. Schedule a softint to stir the entropy pool if
983 * needed. Return true if used fully, false if truncated at all.
984 *
985 * Using this in thread or softint context with no spin locks held
986 * will work, but you might as well use entropy_enter in that
987 * case.
988 */
989 static bool
entropy_enter_intr(const void * buf,size_t len,unsigned nbits,bool count)990 entropy_enter_intr(const void *buf, size_t len, unsigned nbits, bool count)
991 {
992 struct entropy_cpu *ec;
993 bool fullyused = false;
994 uint32_t bitspending, samplespending;
995 int s;
996
997 KASSERTMSG(howmany(nbits, NBBY) <= len,
998 "impossible entropy rate: %u bits in %zu-byte string", nbits, len);
999
1000 /*
1001 * If we're still cold, just use entropy_enter_early to put
1002 * samples directly into the global pool.
1003 */
1004 if (__predict_false(cold)) {
1005 entropy_enter_early(buf, len, nbits);
1006 return true;
1007 }
1008
1009 /*
1010 * In case we were called in thread or interrupt context with
1011 * interrupts unblocked, block soft interrupts up to
1012 * IPL_SOFTSERIAL. This way logic that is safe in interrupt
1013 * context or under a spin lock is also safe in less
1014 * restrictive contexts.
1015 */
1016 s = splsoftserial();
1017
1018 /*
1019 * Acquire the per-CPU state. If someone is in the middle of
1020 * using it, drop the sample. Otherwise, take the lock so that
1021 * higher-priority interrupts will drop their samples.
1022 */
1023 ec = percpu_getref(entropy_percpu);
1024 if (ec->ec_locked) {
1025 ec->ec_evcnt->intrdrop.ev_count++;
1026 goto out0;
1027 }
1028 ec->ec_locked = true;
1029 __insn_barrier();
1030
1031 /*
1032 * Enter as much as we can into the per-CPU pool. If it was
1033 * truncated, schedule a softint to stir the pool and stop.
1034 */
1035 if (!entpool_enter_nostir(ec->ec_pool, buf, len)) {
1036 if (__predict_true(!cold))
1037 softint_schedule(entropy_sih);
1038 ec->ec_evcnt->intrtrunc.ev_count++;
1039 goto out1;
1040 }
1041 fullyused = true;
1042
1043 /*
1044 * Count up what we can contribute.
1045 *
1046 * We don't count samples while cold because entropy_timer
1047 * might still be returning zero if there's no CPU cycle
1048 * counter.
1049 */
1050 bitspending = ec->ec_bitspending;
1051 bitspending += MIN(MINENTROPYBITS - bitspending, nbits);
1052 atomic_store_relaxed(&ec->ec_bitspending, bitspending);
1053 if (__predict_true(count)) {
1054 samplespending = ec->ec_samplespending;
1055 samplespending += MIN(MINSAMPLES - samplespending, 1);
1056 atomic_store_relaxed(&ec->ec_samplespending, samplespending);
1057 }
1058
1059 /* Schedule a softint if we added anything and it matters. */
1060 if (__predict_false(atomic_load_relaxed(&E->bitsneeded) ||
1061 atomic_load_relaxed(&entropy_depletion)) &&
1062 (nbits != 0 || count) &&
1063 __predict_true(!cold))
1064 softint_schedule(entropy_sih);
1065
1066 out1: /* Release the per-CPU state. */
1067 KASSERT(ec->ec_locked);
1068 __insn_barrier();
1069 ec->ec_locked = false;
1070 out0: percpu_putref(entropy_percpu);
1071 splx(s);
1072
1073 return fullyused;
1074 }
1075
1076 /*
1077 * entropy_softintr(cookie)
1078 *
1079 * Soft interrupt handler for entering entropy. Takes care of
1080 * stirring the local CPU's entropy pool if it filled up during
1081 * hard interrupts, and promptly crediting entropy from the local
1082 * CPU's entropy pool to the global entropy pool if needed.
1083 */
1084 static void
entropy_softintr(void * cookie)1085 entropy_softintr(void *cookie)
1086 {
1087 struct entropy_cpu_lock lock;
1088 struct entropy_cpu *ec;
1089 unsigned bitspending, samplespending;
1090
1091 /*
1092 * With the per-CPU state locked, stir the pool if necessary
1093 * and determine if there's any pending entropy on this CPU to
1094 * account globally.
1095 */
1096 ec = entropy_cpu_get(&lock);
1097 ec->ec_evcnt->softint.ev_count++;
1098 entpool_stir(ec->ec_pool);
1099 bitspending = ec->ec_bitspending;
1100 samplespending = ec->ec_samplespending;
1101 entropy_cpu_put(&lock, ec);
1102
1103 /* Consolidate globally if appropriate based on what we added. */
1104 if (bitspending > 0 || samplespending >= MINSAMPLES)
1105 entropy_account_cpu(ec);
1106 }
1107
1108 /*
1109 * entropy_thread(cookie)
1110 *
1111 * Handle any asynchronous entropy housekeeping.
1112 */
1113 static void
entropy_thread(void * cookie)1114 entropy_thread(void *cookie)
1115 {
1116 bool consolidate;
1117
1118 #ifndef _RUMPKERNEL /* XXX rump starts threads before cold */
1119 KASSERT(!cold);
1120 #endif
1121
1122 for (;;) {
1123 /*
1124 * Wait until there's full entropy somewhere among the
1125 * CPUs, as confirmed at most once per minute, or
1126 * someone wants to consolidate.
1127 */
1128 if (entropy_pending()) {
1129 consolidate = true;
1130 } else {
1131 mutex_enter(&E->lock);
1132 if (!E->consolidate)
1133 cv_timedwait(&E->cv, &E->lock, 60*hz);
1134 consolidate = E->consolidate;
1135 E->consolidate = false;
1136 mutex_exit(&E->lock);
1137 }
1138
1139 if (consolidate) {
1140 /* Do it. */
1141 entropy_do_consolidate();
1142
1143 /* Mitigate abuse. */
1144 kpause("entropy", false, hz, NULL);
1145 }
1146 }
1147 }
1148
1149 struct entropy_pending_count {
1150 uint32_t bitspending;
1151 uint32_t samplespending;
1152 };
1153
1154 /*
1155 * entropy_pending()
1156 *
1157 * True if enough bits or samples are pending on other CPUs to
1158 * warrant consolidation.
1159 */
1160 static bool
entropy_pending(void)1161 entropy_pending(void)
1162 {
1163 struct entropy_pending_count count = { 0, 0 }, *C = &count;
1164
1165 percpu_foreach(entropy_percpu, &entropy_pending_cpu, C);
1166 return C->bitspending >= MINENTROPYBITS ||
1167 C->samplespending >= MINSAMPLES;
1168 }
1169
1170 static void
entropy_pending_cpu(void * ptr,void * cookie,struct cpu_info * ci)1171 entropy_pending_cpu(void *ptr, void *cookie, struct cpu_info *ci)
1172 {
1173 struct entropy_cpu *ec = ptr;
1174 struct entropy_pending_count *C = cookie;
1175 uint32_t cpu_bitspending;
1176 uint32_t cpu_samplespending;
1177
1178 cpu_bitspending = atomic_load_relaxed(&ec->ec_bitspending);
1179 cpu_samplespending = atomic_load_relaxed(&ec->ec_samplespending);
1180 C->bitspending += MIN(MINENTROPYBITS - C->bitspending,
1181 cpu_bitspending);
1182 C->samplespending += MIN(MINSAMPLES - C->samplespending,
1183 cpu_samplespending);
1184 }
1185
1186 /*
1187 * entropy_do_consolidate()
1188 *
1189 * Issue a cross-call to gather entropy on all CPUs and advance
1190 * the entropy epoch.
1191 */
1192 static void
entropy_do_consolidate(void)1193 entropy_do_consolidate(void)
1194 {
1195 static const struct timeval interval = {.tv_sec = 60, .tv_usec = 0};
1196 static struct timeval lasttime; /* serialized by E->lock */
1197 struct entpool pool;
1198 uint8_t buf[ENTPOOL_CAPACITY];
1199 unsigned bitsdiff, samplesdiff;
1200 uint64_t ticket;
1201
1202 KASSERT(!cold);
1203 ASSERT_SLEEPABLE();
1204
1205 /* Gather entropy on all CPUs into a temporary pool. */
1206 memset(&pool, 0, sizeof pool);
1207 ticket = xc_broadcast(0, &entropy_consolidate_xc, &pool, NULL);
1208 xc_wait(ticket);
1209
1210 /* Acquire the lock to notify waiters. */
1211 mutex_enter(&E->lock);
1212
1213 /* Count another consolidation. */
1214 entropy_consolidate_evcnt.ev_count++;
1215
1216 /* Note when we last consolidated, i.e. now. */
1217 E->timestamp = time_uptime;
1218
1219 /* Mix what we gathered into the global pool. */
1220 entpool_extract(&pool, buf, sizeof buf);
1221 entpool_enter(&E->pool, buf, sizeof buf);
1222 explicit_memset(&pool, 0, sizeof pool);
1223
1224 /* Count the entropy that was gathered. */
1225 bitsdiff = MIN(E->bitsneeded, E->bitspending);
1226 atomic_store_relaxed(&E->bitsneeded, E->bitsneeded - bitsdiff);
1227 E->bitspending -= bitsdiff;
1228 if (__predict_false(E->bitsneeded > 0) && bitsdiff != 0) {
1229 if ((boothowto & AB_DEBUG) != 0 &&
1230 ratecheck(&lasttime, &interval)) {
1231 printf("WARNING:"
1232 " consolidating less than full entropy\n");
1233 }
1234 }
1235
1236 samplesdiff = MIN(E->samplesneeded, E->samplespending);
1237 atomic_store_relaxed(&E->samplesneeded,
1238 E->samplesneeded - samplesdiff);
1239 E->samplespending -= samplesdiff;
1240
1241 /* Advance the epoch and notify waiters. */
1242 entropy_notify();
1243
1244 /* Release the lock. */
1245 mutex_exit(&E->lock);
1246 }
1247
1248 /*
1249 * entropy_consolidate_xc(vpool, arg2)
1250 *
1251 * Extract output from the local CPU's input pool and enter it
1252 * into a temporary pool passed as vpool.
1253 */
1254 static void
entropy_consolidate_xc(void * vpool,void * arg2 __unused)1255 entropy_consolidate_xc(void *vpool, void *arg2 __unused)
1256 {
1257 struct entpool *pool = vpool;
1258 struct entropy_cpu_lock lock;
1259 struct entropy_cpu *ec;
1260 uint8_t buf[ENTPOOL_CAPACITY];
1261 uint32_t extra[7];
1262 unsigned i = 0;
1263
1264 /* Grab CPU number and cycle counter to mix extra into the pool. */
1265 extra[i++] = cpu_number();
1266 extra[i++] = entropy_timer();
1267
1268 /*
1269 * With the per-CPU state locked, extract from the per-CPU pool
1270 * and count it as no longer pending.
1271 */
1272 ec = entropy_cpu_get(&lock);
1273 extra[i++] = entropy_timer();
1274 entpool_extract(ec->ec_pool, buf, sizeof buf);
1275 atomic_store_relaxed(&ec->ec_bitspending, 0);
1276 atomic_store_relaxed(&ec->ec_samplespending, 0);
1277 extra[i++] = entropy_timer();
1278 entropy_cpu_put(&lock, ec);
1279 extra[i++] = entropy_timer();
1280
1281 /*
1282 * Copy over statistics, and enter the per-CPU extract and the
1283 * extra timing into the temporary pool, under the global lock.
1284 */
1285 mutex_enter(&E->lock);
1286 extra[i++] = entropy_timer();
1287 entpool_enter(pool, buf, sizeof buf);
1288 explicit_memset(buf, 0, sizeof buf);
1289 extra[i++] = entropy_timer();
1290 KASSERT(i == __arraycount(extra));
1291 entpool_enter(pool, extra, sizeof extra);
1292 explicit_memset(extra, 0, sizeof extra);
1293 mutex_exit(&E->lock);
1294 }
1295
1296 /*
1297 * entropy_notify()
1298 *
1299 * Caller just contributed entropy to the global pool. Advance
1300 * the entropy epoch and notify waiters.
1301 *
1302 * Caller must hold the global entropy lock.
1303 */
1304 static void
entropy_notify(void)1305 entropy_notify(void)
1306 {
1307 static const struct timeval interval = {.tv_sec = 60, .tv_usec = 0};
1308 static struct timeval lasttime; /* serialized by E->lock */
1309 static bool ready = false, besteffort = false;
1310 unsigned epoch;
1311
1312 KASSERT(__predict_false(cold) || mutex_owned(&E->lock));
1313
1314 /*
1315 * If this is the first time, print a message to the console
1316 * that we're ready so operators can compare it to the timing
1317 * of other events.
1318 *
1319 * If we didn't get full entropy from reliable sources, report
1320 * instead that we are running on fumes with best effort. (If
1321 * we ever do get full entropy after that, print the ready
1322 * message once.)
1323 */
1324 if (__predict_false(!ready)) {
1325 if (E->bitsneeded == 0) {
1326 printf("entropy: ready\n");
1327 ready = true;
1328 } else if (E->samplesneeded == 0 && !besteffort) {
1329 printf("entropy: best effort\n");
1330 besteffort = true;
1331 }
1332 }
1333
1334 /* Set the epoch; roll over from UINTMAX-1 to 1. */
1335 if (__predict_true(!atomic_load_relaxed(&entropy_depletion)) ||
1336 ratecheck(&lasttime, &interval)) {
1337 epoch = E->epoch + 1;
1338 if (epoch == 0 || epoch == (unsigned)-1)
1339 epoch = 1;
1340 atomic_store_relaxed(&E->epoch, epoch);
1341 }
1342 KASSERT(E->epoch != (unsigned)-1);
1343
1344 /* Notify waiters. */
1345 if (__predict_true(!cold)) {
1346 cv_broadcast(&E->cv);
1347 selnotify(&E->selq, POLLIN|POLLRDNORM, NOTE_SUBMIT);
1348 }
1349
1350 /* Count another notification. */
1351 entropy_notify_evcnt.ev_count++;
1352 }
1353
1354 /*
1355 * entropy_consolidate()
1356 *
1357 * Trigger entropy consolidation and wait for it to complete, or
1358 * return EINTR if interrupted by a signal.
1359 *
1360 * This should be used sparingly, not periodically -- requiring
1361 * conscious intervention by the operator or a clear policy
1362 * decision. Otherwise, the kernel will automatically consolidate
1363 * when enough entropy has been gathered into per-CPU pools to
1364 * transition to full entropy.
1365 */
1366 int
entropy_consolidate(void)1367 entropy_consolidate(void)
1368 {
1369 uint64_t ticket;
1370 int error;
1371
1372 KASSERT(!cold);
1373 ASSERT_SLEEPABLE();
1374
1375 mutex_enter(&E->lock);
1376 ticket = entropy_consolidate_evcnt.ev_count;
1377 E->consolidate = true;
1378 cv_broadcast(&E->cv);
1379 while (ticket == entropy_consolidate_evcnt.ev_count) {
1380 error = cv_wait_sig(&E->cv, &E->lock);
1381 if (error)
1382 break;
1383 }
1384 mutex_exit(&E->lock);
1385
1386 return error;
1387 }
1388
1389 /*
1390 * sysctl -w kern.entropy.consolidate=1
1391 *
1392 * Trigger entropy consolidation and wait for it to complete.
1393 * Writable only by superuser. This, writing to /dev/random, and
1394 * ioctl(RNDADDDATA) are the only ways for the system to
1395 * consolidate entropy if the operator knows something the kernel
1396 * doesn't about how unpredictable the pending entropy pools are.
1397 */
1398 static int
sysctl_entropy_consolidate(SYSCTLFN_ARGS)1399 sysctl_entropy_consolidate(SYSCTLFN_ARGS)
1400 {
1401 struct sysctlnode node = *rnode;
1402 int arg = 0;
1403 int error;
1404
1405 node.sysctl_data = &arg;
1406 error = sysctl_lookup(SYSCTLFN_CALL(&node));
1407 if (error || newp == NULL)
1408 return error;
1409 if (arg)
1410 error = entropy_consolidate();
1411
1412 return error;
1413 }
1414
1415 /*
1416 * entropy_gather()
1417 *
1418 * Trigger gathering entropy from all on-demand sources, and, if
1419 * requested, wait for synchronous sources (but not asynchronous
1420 * sources) to complete, or fail with EINTR if interrupted by a
1421 * signal.
1422 */
1423 int
entropy_gather(void)1424 entropy_gather(void)
1425 {
1426 int error;
1427
1428 mutex_enter(&E->lock);
1429 error = entropy_request(ENTROPY_CAPACITY, ENTROPY_WAIT|ENTROPY_SIG);
1430 mutex_exit(&E->lock);
1431
1432 return error;
1433 }
1434
1435 /*
1436 * sysctl -w kern.entropy.gather=1
1437 *
1438 * Trigger gathering entropy from all on-demand sources, and wait
1439 * for synchronous sources (but not asynchronous sources) to
1440 * complete. Writable only by superuser.
1441 */
1442 static int
sysctl_entropy_gather(SYSCTLFN_ARGS)1443 sysctl_entropy_gather(SYSCTLFN_ARGS)
1444 {
1445 struct sysctlnode node = *rnode;
1446 int arg = 0;
1447 int error;
1448
1449 node.sysctl_data = &arg;
1450 error = sysctl_lookup(SYSCTLFN_CALL(&node));
1451 if (error || newp == NULL)
1452 return error;
1453 if (arg)
1454 error = entropy_gather();
1455
1456 return error;
1457 }
1458
1459 /*
1460 * entropy_extract(buf, len, flags)
1461 *
1462 * Extract len bytes from the global entropy pool into buf.
1463 *
1464 * Caller MUST NOT expose these bytes directly -- must use them
1465 * ONLY to seed a cryptographic pseudorandom number generator
1466 * (`CPRNG'), a.k.a. deterministic random bit generator (`DRBG'),
1467 * and then erase them. entropy_extract does not, on its own,
1468 * provide backtracking resistance -- it must be combined with a
1469 * PRNG/DRBG that does.
1470 *
1471 * This may be used very early at boot, before even entropy_init
1472 * has been called.
1473 *
1474 * You generally shouldn't use this directly -- use cprng(9)
1475 * instead.
1476 *
1477 * Flags may have:
1478 *
1479 * ENTROPY_WAIT Wait for entropy if not available yet.
1480 * ENTROPY_SIG Allow interruption by a signal during wait.
1481 * ENTROPY_HARDFAIL Either fill the buffer with full entropy,
1482 * or fail without filling it at all.
1483 *
1484 * Return zero on success, or error on failure:
1485 *
1486 * EWOULDBLOCK No entropy and ENTROPY_WAIT not set.
1487 * EINTR/ERESTART No entropy, ENTROPY_SIG set, and interrupted.
1488 *
1489 * If ENTROPY_WAIT is set, allowed only in thread context. If
1490 * ENTROPY_WAIT is not set, allowed also in softint context -- may
1491 * sleep on an adaptive lock up to IPL_SOFTSERIAL. Forbidden in
1492 * hard interrupt context.
1493 */
1494 int
entropy_extract(void * buf,size_t len,int flags)1495 entropy_extract(void *buf, size_t len, int flags)
1496 {
1497 static const struct timeval interval = {.tv_sec = 60, .tv_usec = 0};
1498 static struct timeval lasttime; /* serialized by E->lock */
1499 bool printed = false;
1500 int s = -1/*XXXGCC*/, error;
1501
1502 if (ISSET(flags, ENTROPY_WAIT)) {
1503 ASSERT_SLEEPABLE();
1504 KASSERT(!cold);
1505 }
1506
1507 /* Refuse to operate in interrupt context. */
1508 KASSERT(!cpu_intr_p());
1509
1510 /*
1511 * If we're cold, we are only contending with interrupts on the
1512 * current CPU, so block them. Otherwise, we are _not_
1513 * contending with interrupts on the current CPU, but we are
1514 * contending with other threads, to exclude them with a mutex.
1515 */
1516 if (__predict_false(cold))
1517 s = splhigh();
1518 else
1519 mutex_enter(&E->lock);
1520
1521 /* Wait until there is enough entropy in the system. */
1522 error = 0;
1523 if (E->bitsneeded > 0 && E->samplesneeded == 0) {
1524 /*
1525 * We don't have full entropy from reliable sources,
1526 * but we gathered a plausible number of samples from
1527 * other sources such as timers. Try asking for more
1528 * from any sources we can, but don't worry if it
1529 * fails -- best effort.
1530 */
1531 (void)entropy_request(ENTROPY_CAPACITY, flags);
1532 } else while (E->bitsneeded > 0 && E->samplesneeded > 0) {
1533 /* Ask for more, synchronously if possible. */
1534 error = entropy_request(len, flags);
1535 if (error)
1536 break;
1537
1538 /* If we got enough, we're done. */
1539 if (E->bitsneeded == 0 || E->samplesneeded == 0) {
1540 KASSERT(error == 0);
1541 break;
1542 }
1543
1544 /* If not waiting, stop here. */
1545 if (!ISSET(flags, ENTROPY_WAIT)) {
1546 error = EWOULDBLOCK;
1547 break;
1548 }
1549
1550 /* Wait for some entropy to come in and try again. */
1551 KASSERT(!cold);
1552 if (!printed) {
1553 printf("entropy: pid %d (%s) waiting for entropy(7)\n",
1554 curproc->p_pid, curproc->p_comm);
1555 printed = true;
1556 }
1557
1558 if (ISSET(flags, ENTROPY_SIG)) {
1559 error = cv_timedwait_sig(&E->cv, &E->lock, hz);
1560 if (error && error != EWOULDBLOCK)
1561 break;
1562 } else {
1563 cv_timedwait(&E->cv, &E->lock, hz);
1564 }
1565 }
1566
1567 /*
1568 * Count failure -- but fill the buffer nevertheless, unless
1569 * the caller specified ENTROPY_HARDFAIL.
1570 */
1571 if (error) {
1572 if (ISSET(flags, ENTROPY_HARDFAIL))
1573 goto out;
1574 entropy_extract_fail_evcnt.ev_count++;
1575 }
1576
1577 /*
1578 * Report a warning if we haven't yet reached full entropy.
1579 * This is the only case where we consider entropy to be
1580 * `depleted' without kern.entropy.depletion enabled -- when we
1581 * only have partial entropy, an adversary may be able to
1582 * narrow the state of the pool down to a small number of
1583 * possibilities; the output then enables them to confirm a
1584 * guess, reducing its entropy from the adversary's perspective
1585 * to zero.
1586 *
1587 * This should only happen if the operator has chosen to
1588 * consolidate, either through sysctl kern.entropy.consolidate
1589 * or by writing less than full entropy to /dev/random as root
1590 * (which /dev/random promises will immediately affect
1591 * subsequent output, for better or worse).
1592 */
1593 if (E->bitsneeded > 0 && E->samplesneeded > 0) {
1594 if (__predict_false(E->epoch == (unsigned)-1) &&
1595 ratecheck(&lasttime, &interval)) {
1596 printf("WARNING:"
1597 " system needs entropy for security;"
1598 " see entropy(7)\n");
1599 }
1600 atomic_store_relaxed(&E->bitsneeded, MINENTROPYBITS);
1601 atomic_store_relaxed(&E->samplesneeded, MINSAMPLES);
1602 }
1603
1604 /* Extract data from the pool, and `deplete' if we're doing that. */
1605 entpool_extract(&E->pool, buf, len);
1606 if (__predict_false(atomic_load_relaxed(&entropy_depletion)) &&
1607 error == 0) {
1608 unsigned cost = MIN(len, ENTROPY_CAPACITY)*NBBY;
1609 unsigned bitsneeded = E->bitsneeded;
1610 unsigned samplesneeded = E->samplesneeded;
1611
1612 bitsneeded += MIN(MINENTROPYBITS - bitsneeded, cost);
1613 samplesneeded += MIN(MINSAMPLES - samplesneeded, cost);
1614
1615 atomic_store_relaxed(&E->bitsneeded, bitsneeded);
1616 atomic_store_relaxed(&E->samplesneeded, samplesneeded);
1617 entropy_deplete_evcnt.ev_count++;
1618 }
1619
1620 out: /* Release the global lock and return the error. */
1621 if (__predict_false(cold))
1622 splx(s);
1623 else
1624 mutex_exit(&E->lock);
1625 return error;
1626 }
1627
1628 /*
1629 * entropy_poll(events)
1630 *
1631 * Return the subset of events ready, and if it is not all of
1632 * events, record curlwp as waiting for entropy.
1633 */
1634 int
entropy_poll(int events)1635 entropy_poll(int events)
1636 {
1637 int revents = 0;
1638
1639 KASSERT(!cold);
1640
1641 /* Always ready for writing. */
1642 revents |= events & (POLLOUT|POLLWRNORM);
1643
1644 /* Narrow it down to reads. */
1645 events &= POLLIN|POLLRDNORM;
1646 if (events == 0)
1647 return revents;
1648
1649 /*
1650 * If we have reached full entropy and we're not depleting
1651 * entropy, we are forever ready.
1652 */
1653 if (__predict_true(atomic_load_relaxed(&E->bitsneeded) == 0 ||
1654 atomic_load_relaxed(&E->samplesneeded) == 0) &&
1655 __predict_true(!atomic_load_relaxed(&entropy_depletion)))
1656 return revents | events;
1657
1658 /*
1659 * Otherwise, check whether we need entropy under the lock. If
1660 * we don't, we're ready; if we do, add ourselves to the queue.
1661 */
1662 mutex_enter(&E->lock);
1663 if (E->bitsneeded == 0 || E->samplesneeded == 0)
1664 revents |= events;
1665 else
1666 selrecord(curlwp, &E->selq);
1667 mutex_exit(&E->lock);
1668
1669 return revents;
1670 }
1671
1672 /*
1673 * filt_entropy_read_detach(kn)
1674 *
1675 * struct filterops::f_detach callback for entropy read events:
1676 * remove kn from the list of waiters.
1677 */
1678 static void
filt_entropy_read_detach(struct knote * kn)1679 filt_entropy_read_detach(struct knote *kn)
1680 {
1681
1682 KASSERT(!cold);
1683
1684 mutex_enter(&E->lock);
1685 selremove_knote(&E->selq, kn);
1686 mutex_exit(&E->lock);
1687 }
1688
1689 /*
1690 * filt_entropy_read_event(kn, hint)
1691 *
1692 * struct filterops::f_event callback for entropy read events:
1693 * poll for entropy. Caller must hold the global entropy lock if
1694 * hint is NOTE_SUBMIT, and must not if hint is not NOTE_SUBMIT.
1695 */
1696 static int
filt_entropy_read_event(struct knote * kn,long hint)1697 filt_entropy_read_event(struct knote *kn, long hint)
1698 {
1699 int ret;
1700
1701 KASSERT(!cold);
1702
1703 /* Acquire the lock, if caller is outside entropy subsystem. */
1704 if (hint == NOTE_SUBMIT)
1705 KASSERT(mutex_owned(&E->lock));
1706 else
1707 mutex_enter(&E->lock);
1708
1709 /*
1710 * If we still need entropy, can't read anything; if not, can
1711 * read arbitrarily much.
1712 */
1713 if (E->bitsneeded != 0 && E->samplesneeded != 0) {
1714 ret = 0;
1715 } else {
1716 if (atomic_load_relaxed(&entropy_depletion))
1717 kn->kn_data = ENTROPY_CAPACITY; /* bytes */
1718 else
1719 kn->kn_data = MIN(INT64_MAX, SSIZE_MAX);
1720 ret = 1;
1721 }
1722
1723 /* Release the lock, if caller is outside entropy subsystem. */
1724 if (hint == NOTE_SUBMIT)
1725 KASSERT(mutex_owned(&E->lock));
1726 else
1727 mutex_exit(&E->lock);
1728
1729 return ret;
1730 }
1731
1732 /* XXX Makes sense only for /dev/u?random. */
1733 static const struct filterops entropy_read_filtops = {
1734 .f_flags = FILTEROP_ISFD | FILTEROP_MPSAFE,
1735 .f_attach = NULL,
1736 .f_detach = filt_entropy_read_detach,
1737 .f_event = filt_entropy_read_event,
1738 };
1739
1740 /*
1741 * entropy_kqfilter(kn)
1742 *
1743 * Register kn to receive entropy event notifications. May be
1744 * EVFILT_READ or EVFILT_WRITE; anything else yields EINVAL.
1745 */
1746 int
entropy_kqfilter(struct knote * kn)1747 entropy_kqfilter(struct knote *kn)
1748 {
1749
1750 KASSERT(!cold);
1751
1752 switch (kn->kn_filter) {
1753 case EVFILT_READ:
1754 /* Enter into the global select queue. */
1755 mutex_enter(&E->lock);
1756 kn->kn_fop = &entropy_read_filtops;
1757 selrecord_knote(&E->selq, kn);
1758 mutex_exit(&E->lock);
1759 return 0;
1760 case EVFILT_WRITE:
1761 /* Can always dump entropy into the system. */
1762 kn->kn_fop = &seltrue_filtops;
1763 return 0;
1764 default:
1765 return EINVAL;
1766 }
1767 }
1768
1769 /*
1770 * rndsource_setcb(rs, get, getarg)
1771 *
1772 * Set the request callback for the entropy source rs, if it can
1773 * provide entropy on demand. Must precede rnd_attach_source.
1774 */
1775 void
rndsource_setcb(struct krndsource * rs,void (* get)(size_t,void *),void * getarg)1776 rndsource_setcb(struct krndsource *rs, void (*get)(size_t, void *),
1777 void *getarg)
1778 {
1779
1780 rs->get = get;
1781 rs->getarg = getarg;
1782 }
1783
1784 /*
1785 * rnd_attach_source(rs, name, type, flags)
1786 *
1787 * Attach the entropy source rs. Must be done after
1788 * rndsource_setcb, if any, and before any calls to rnd_add_data.
1789 */
1790 void
rnd_attach_source(struct krndsource * rs,const char * name,uint32_t type,uint32_t flags)1791 rnd_attach_source(struct krndsource *rs, const char *name, uint32_t type,
1792 uint32_t flags)
1793 {
1794 uint32_t extra[4];
1795 unsigned i = 0;
1796
1797 KASSERTMSG(name[0] != '\0', "rndsource must have nonempty name");
1798
1799 /* Grab cycle counter to mix extra into the pool. */
1800 extra[i++] = entropy_timer();
1801
1802 /*
1803 * Apply some standard flags:
1804 *
1805 * - We do not bother with network devices by default, for
1806 * hysterical raisins (perhaps: because it is often the case
1807 * that an adversary can influence network packet timings).
1808 */
1809 switch (type) {
1810 case RND_TYPE_NET:
1811 flags |= RND_FLAG_NO_COLLECT;
1812 break;
1813 }
1814
1815 /* Sanity-check the callback if RND_FLAG_HASCB is set. */
1816 KASSERT(!ISSET(flags, RND_FLAG_HASCB) || rs->get != NULL);
1817
1818 /* Initialize the random source. */
1819 memset(rs->name, 0, sizeof(rs->name)); /* paranoia */
1820 strlcpy(rs->name, name, sizeof(rs->name));
1821 memset(&rs->time_delta, 0, sizeof(rs->time_delta));
1822 memset(&rs->value_delta, 0, sizeof(rs->value_delta));
1823 rs->total = 0;
1824 rs->type = type;
1825 rs->flags = flags;
1826 if (entropy_percpu != NULL)
1827 rs->state = percpu_alloc(sizeof(struct rndsource_cpu));
1828 extra[i++] = entropy_timer();
1829
1830 /* Wire it into the global list of random sources. */
1831 if (__predict_true(!cold))
1832 mutex_enter(&E->lock);
1833 LIST_INSERT_HEAD(&E->sources, rs, list);
1834 if (__predict_true(!cold))
1835 mutex_exit(&E->lock);
1836 extra[i++] = entropy_timer();
1837
1838 /* Request that it provide entropy ASAP, if we can. */
1839 if (ISSET(flags, RND_FLAG_HASCB))
1840 (*rs->get)(ENTROPY_CAPACITY, rs->getarg);
1841 extra[i++] = entropy_timer();
1842
1843 /* Mix the extra into the pool. */
1844 KASSERT(i == __arraycount(extra));
1845 entropy_enter(extra, sizeof extra, 0, /*count*/__predict_true(!cold));
1846 explicit_memset(extra, 0, sizeof extra);
1847 }
1848
1849 /*
1850 * rnd_detach_source(rs)
1851 *
1852 * Detach the entropy source rs. May sleep waiting for users to
1853 * drain. Further use is not allowed.
1854 */
1855 void
rnd_detach_source(struct krndsource * rs)1856 rnd_detach_source(struct krndsource *rs)
1857 {
1858
1859 /*
1860 * If we're cold (shouldn't happen, but hey), just remove it
1861 * from the list -- there's nothing allocated.
1862 */
1863 if (__predict_false(cold) && entropy_percpu == NULL) {
1864 LIST_REMOVE(rs, list);
1865 return;
1866 }
1867
1868 /* We may have to wait for entropy_request. */
1869 ASSERT_SLEEPABLE();
1870
1871 /* Wait until the source list is not in use, and remove it. */
1872 mutex_enter(&E->lock);
1873 while (E->sourcelock)
1874 cv_wait(&E->sourcelock_cv, &E->lock);
1875 LIST_REMOVE(rs, list);
1876 mutex_exit(&E->lock);
1877
1878 /* Free the per-CPU data. */
1879 percpu_free(rs->state, sizeof(struct rndsource_cpu));
1880 }
1881
1882 /*
1883 * rnd_lock_sources(flags)
1884 *
1885 * Lock the list of entropy sources. Caller must hold the global
1886 * entropy lock. If successful, no rndsource will go away until
1887 * rnd_unlock_sources even while the caller releases the global
1888 * entropy lock.
1889 *
1890 * May be called very early at boot, before entropy_init.
1891 *
1892 * If flags & ENTROPY_WAIT, wait for concurrent access to finish.
1893 * If flags & ENTROPY_SIG, allow interruption by signal.
1894 */
1895 static int __attribute__((warn_unused_result))
rnd_lock_sources(int flags)1896 rnd_lock_sources(int flags)
1897 {
1898 int error;
1899
1900 KASSERT(__predict_false(cold) || mutex_owned(&E->lock));
1901 KASSERT(!cpu_intr_p());
1902
1903 while (E->sourcelock) {
1904 KASSERT(!cold);
1905 if (!ISSET(flags, ENTROPY_WAIT))
1906 return EWOULDBLOCK;
1907 if (ISSET(flags, ENTROPY_SIG)) {
1908 error = cv_wait_sig(&E->sourcelock_cv, &E->lock);
1909 if (error)
1910 return error;
1911 } else {
1912 cv_wait(&E->sourcelock_cv, &E->lock);
1913 }
1914 }
1915
1916 E->sourcelock = curlwp;
1917 return 0;
1918 }
1919
1920 /*
1921 * rnd_unlock_sources()
1922 *
1923 * Unlock the list of sources after rnd_lock_sources. Caller must
1924 * hold the global entropy lock.
1925 *
1926 * May be called very early at boot, before entropy_init.
1927 */
1928 static void
rnd_unlock_sources(void)1929 rnd_unlock_sources(void)
1930 {
1931
1932 KASSERT(__predict_false(cold) || mutex_owned(&E->lock));
1933 KASSERT(!cpu_intr_p());
1934
1935 KASSERTMSG(E->sourcelock == curlwp, "lwp %p releasing lock held by %p",
1936 curlwp, E->sourcelock);
1937 E->sourcelock = NULL;
1938 if (__predict_true(!cold))
1939 cv_signal(&E->sourcelock_cv);
1940 }
1941
1942 /*
1943 * rnd_sources_locked()
1944 *
1945 * True if we hold the list of rndsources locked, for diagnostic
1946 * assertions.
1947 *
1948 * May be called very early at boot, before entropy_init.
1949 */
1950 static bool __diagused
rnd_sources_locked(void)1951 rnd_sources_locked(void)
1952 {
1953
1954 return E->sourcelock == curlwp;
1955 }
1956
1957 /*
1958 * entropy_request(nbytes, flags)
1959 *
1960 * Request nbytes bytes of entropy from all sources in the system.
1961 * OK if we overdo it. Caller must hold the global entropy lock;
1962 * will release and re-acquire it.
1963 *
1964 * May be called very early at boot, before entropy_init.
1965 *
1966 * If flags & ENTROPY_WAIT, wait for concurrent access to finish.
1967 * If flags & ENTROPY_SIG, allow interruption by signal.
1968 */
1969 static int
entropy_request(size_t nbytes,int flags)1970 entropy_request(size_t nbytes, int flags)
1971 {
1972 struct krndsource *rs;
1973 int error;
1974
1975 KASSERT(__predict_false(cold) || mutex_owned(&E->lock));
1976 KASSERT(!cpu_intr_p());
1977 if ((flags & ENTROPY_WAIT) != 0 && __predict_false(!cold))
1978 ASSERT_SLEEPABLE();
1979
1980 /*
1981 * Lock the list of entropy sources to block rnd_detach_source
1982 * until we're done, and to serialize calls to the entropy
1983 * callbacks as guaranteed to drivers.
1984 */
1985 error = rnd_lock_sources(flags);
1986 if (error)
1987 return error;
1988 entropy_request_evcnt.ev_count++;
1989
1990 /* Clamp to the maximum reasonable request. */
1991 nbytes = MIN(nbytes, ENTROPY_CAPACITY);
1992
1993 /* Walk the list of sources. */
1994 LIST_FOREACH(rs, &E->sources, list) {
1995 /* Skip sources without callbacks. */
1996 if (!ISSET(rs->flags, RND_FLAG_HASCB))
1997 continue;
1998
1999 /*
2000 * Skip sources that are disabled altogether -- we
2001 * would just ignore their samples anyway.
2002 */
2003 if (ISSET(rs->flags, RND_FLAG_NO_COLLECT))
2004 continue;
2005
2006 /* Drop the lock while we call the callback. */
2007 if (__predict_true(!cold))
2008 mutex_exit(&E->lock);
2009 (*rs->get)(nbytes, rs->getarg);
2010 if (__predict_true(!cold))
2011 mutex_enter(&E->lock);
2012 }
2013
2014 /* Request done; unlock the list of entropy sources. */
2015 rnd_unlock_sources();
2016 return 0;
2017 }
2018
2019 static inline uint32_t
rnd_delta_estimate(rnd_delta_t * d,uint32_t v,int32_t delta)2020 rnd_delta_estimate(rnd_delta_t *d, uint32_t v, int32_t delta)
2021 {
2022 int32_t delta2, delta3;
2023
2024 /*
2025 * Calculate the second and third order differentials
2026 */
2027 delta2 = d->dx - delta;
2028 if (delta2 < 0)
2029 delta2 = -delta2; /* XXX arithmetic overflow */
2030
2031 delta3 = d->d2x - delta2;
2032 if (delta3 < 0)
2033 delta3 = -delta3; /* XXX arithmetic overflow */
2034
2035 d->x = v;
2036 d->dx = delta;
2037 d->d2x = delta2;
2038
2039 /*
2040 * If any delta is 0, we got no entropy. If all are non-zero, we
2041 * might have something.
2042 */
2043 if (delta == 0 || delta2 == 0 || delta3 == 0)
2044 return 0;
2045
2046 return 1;
2047 }
2048
2049 static inline uint32_t
rnd_dt_estimate(struct krndsource * rs,uint32_t t)2050 rnd_dt_estimate(struct krndsource *rs, uint32_t t)
2051 {
2052 int32_t delta;
2053 uint32_t ret;
2054 rnd_delta_t *d;
2055 struct rndsource_cpu *rc;
2056
2057 rc = percpu_getref(rs->state);
2058 d = &rc->rc_timedelta;
2059
2060 if (t < d->x) {
2061 delta = UINT32_MAX - d->x + t;
2062 } else {
2063 delta = d->x - t;
2064 }
2065
2066 if (delta < 0) {
2067 delta = -delta; /* XXX arithmetic overflow */
2068 }
2069
2070 ret = rnd_delta_estimate(d, t, delta);
2071
2072 KASSERT(d->x == t);
2073 KASSERT(d->dx == delta);
2074 percpu_putref(rs->state);
2075 return ret;
2076 }
2077
2078 /*
2079 * rnd_add_uint32(rs, value)
2080 *
2081 * Enter 32 bits of data from an entropy source into the pool.
2082 *
2083 * May be called from any context or with spin locks held, but may
2084 * drop data.
2085 *
2086 * This is meant for cheaply taking samples from devices that
2087 * aren't designed to be hardware random number generators.
2088 */
2089 void
rnd_add_uint32(struct krndsource * rs,uint32_t value)2090 rnd_add_uint32(struct krndsource *rs, uint32_t value)
2091 {
2092 bool intr_p = true;
2093
2094 rnd_add_data_internal(rs, &value, sizeof value, 0, intr_p);
2095 }
2096
2097 void
_rnd_add_uint32(struct krndsource * rs,uint32_t value)2098 _rnd_add_uint32(struct krndsource *rs, uint32_t value)
2099 {
2100 bool intr_p = true;
2101
2102 rnd_add_data_internal(rs, &value, sizeof value, 0, intr_p);
2103 }
2104
2105 void
_rnd_add_uint64(struct krndsource * rs,uint64_t value)2106 _rnd_add_uint64(struct krndsource *rs, uint64_t value)
2107 {
2108 bool intr_p = true;
2109
2110 rnd_add_data_internal(rs, &value, sizeof value, 0, intr_p);
2111 }
2112
2113 /*
2114 * rnd_add_data(rs, buf, len, entropybits)
2115 *
2116 * Enter data from an entropy source into the pool, with a
2117 * driver's estimate of how much entropy the physical source of
2118 * the data has. If RND_FLAG_NO_ESTIMATE, we ignore the driver's
2119 * estimate and treat it as zero.
2120 *
2121 * rs MAY but SHOULD NOT be NULL. If rs is NULL, MUST NOT be
2122 * called from interrupt context or with spin locks held.
2123 *
2124 * If rs is non-NULL, MAY but SHOULD NOT be called from interrupt
2125 * context, in which case act like rnd_add_data_intr -- if the
2126 * sample buffer is full, schedule a softint and drop any
2127 * additional data on the floor. (This may change later once we
2128 * fix drivers that still call this from interrupt context to use
2129 * rnd_add_data_intr instead.) MUST NOT be called with spin locks
2130 * held if not in hard interrupt context -- i.e., MUST NOT be
2131 * called in thread context or softint context with spin locks
2132 * held.
2133 */
2134 void
rnd_add_data(struct krndsource * rs,const void * buf,uint32_t len,uint32_t entropybits)2135 rnd_add_data(struct krndsource *rs, const void *buf, uint32_t len,
2136 uint32_t entropybits)
2137 {
2138 bool intr_p = cpu_intr_p(); /* XXX make this unconditionally false */
2139
2140 /*
2141 * Weird legacy exception that we should rip out and replace by
2142 * creating new rndsources to attribute entropy to the callers:
2143 * If there's no rndsource, just enter the data and time now.
2144 */
2145 if (rs == NULL) {
2146 uint32_t extra;
2147
2148 KASSERT(!intr_p);
2149 KASSERTMSG(howmany(entropybits, NBBY) <= len,
2150 "%s: impossible entropy rate:"
2151 " %"PRIu32" bits in %"PRIu32"-byte string",
2152 rs ? rs->name : "(anonymous)", entropybits, len);
2153 entropy_enter(buf, len, entropybits, /*count*/false);
2154 extra = entropy_timer();
2155 entropy_enter(&extra, sizeof extra, 0, /*count*/false);
2156 explicit_memset(&extra, 0, sizeof extra);
2157 return;
2158 }
2159
2160 rnd_add_data_internal(rs, buf, len, entropybits, intr_p);
2161 }
2162
2163 /*
2164 * rnd_add_data_intr(rs, buf, len, entropybits)
2165 *
2166 * Try to enter data from an entropy source into the pool, with a
2167 * driver's estimate of how much entropy the physical source of
2168 * the data has. If RND_FLAG_NO_ESTIMATE, we ignore the driver's
2169 * estimate and treat it as zero. If the sample buffer is full,
2170 * schedule a softint and drop any additional data on the floor.
2171 */
2172 void
rnd_add_data_intr(struct krndsource * rs,const void * buf,uint32_t len,uint32_t entropybits)2173 rnd_add_data_intr(struct krndsource *rs, const void *buf, uint32_t len,
2174 uint32_t entropybits)
2175 {
2176 bool intr_p = true;
2177
2178 rnd_add_data_internal(rs, buf, len, entropybits, intr_p);
2179 }
2180
2181 /*
2182 * rnd_add_data_internal(rs, buf, len, entropybits, intr_p)
2183 *
2184 * Internal subroutine to decide whether or not to enter data or
2185 * timing for a particular rndsource, and if so, to enter it.
2186 *
2187 * intr_p is true for callers from interrupt context or spin locks
2188 * held, and false for callers from thread or soft interrupt
2189 * context and no spin locks held.
2190 */
2191 static void
rnd_add_data_internal(struct krndsource * rs,const void * buf,uint32_t len,uint32_t entropybits,bool intr_p)2192 rnd_add_data_internal(struct krndsource *rs, const void *buf, uint32_t len,
2193 uint32_t entropybits, bool intr_p)
2194 {
2195 uint32_t flags;
2196
2197 KASSERTMSG(howmany(entropybits, NBBY) <= len,
2198 "%s: impossible entropy rate:"
2199 " %"PRIu32" bits in %"PRIu32"-byte string",
2200 rs ? rs->name : "(anonymous)", entropybits, len);
2201
2202 /*
2203 * Hold up the reset xcall before it zeroes the entropy counts
2204 * on this CPU or globally. Otherwise, we might leave some
2205 * nonzero entropy attributed to an untrusted source in the
2206 * event of a race with a change to flags.
2207 */
2208 kpreempt_disable();
2209
2210 /* Load a snapshot of the flags. Ioctl may change them under us. */
2211 flags = atomic_load_relaxed(&rs->flags);
2212
2213 /*
2214 * Skip if:
2215 * - we're not collecting entropy, or
2216 * - the operator doesn't want to collect entropy from this, or
2217 * - neither data nor timings are being collected from this.
2218 */
2219 if (!atomic_load_relaxed(&entropy_collection) ||
2220 ISSET(flags, RND_FLAG_NO_COLLECT) ||
2221 !ISSET(flags, RND_FLAG_COLLECT_VALUE|RND_FLAG_COLLECT_TIME))
2222 goto out;
2223
2224 /* If asked, ignore the estimate. */
2225 if (ISSET(flags, RND_FLAG_NO_ESTIMATE))
2226 entropybits = 0;
2227
2228 /* If we are collecting data, enter them. */
2229 if (ISSET(flags, RND_FLAG_COLLECT_VALUE)) {
2230 rnd_add_data_1(rs, buf, len, entropybits, /*count*/false,
2231 RND_FLAG_COLLECT_VALUE, intr_p);
2232 }
2233
2234 /* If we are collecting timings, enter one. */
2235 if (ISSET(flags, RND_FLAG_COLLECT_TIME)) {
2236 uint32_t extra;
2237 bool count;
2238
2239 /* Sample a timer. */
2240 extra = entropy_timer();
2241
2242 /* If asked, do entropy estimation on the time. */
2243 if ((flags & (RND_FLAG_ESTIMATE_TIME|RND_FLAG_NO_ESTIMATE)) ==
2244 RND_FLAG_ESTIMATE_TIME && __predict_true(!cold))
2245 count = rnd_dt_estimate(rs, extra);
2246 else
2247 count = false;
2248
2249 rnd_add_data_1(rs, &extra, sizeof extra, 0, count,
2250 RND_FLAG_COLLECT_TIME, intr_p);
2251 }
2252
2253 out: /* Allow concurrent changes to flags to finish. */
2254 kpreempt_enable();
2255 }
2256
2257 static unsigned
add_sat(unsigned a,unsigned b)2258 add_sat(unsigned a, unsigned b)
2259 {
2260 unsigned c = a + b;
2261
2262 return (c < a ? UINT_MAX : c);
2263 }
2264
2265 /*
2266 * rnd_add_data_1(rs, buf, len, entropybits, count, flag)
2267 *
2268 * Internal subroutine to call either entropy_enter_intr, if we're
2269 * in interrupt context, or entropy_enter if not, and to count the
2270 * entropy in an rndsource.
2271 */
2272 static void
rnd_add_data_1(struct krndsource * rs,const void * buf,uint32_t len,uint32_t entropybits,bool count,uint32_t flag,bool intr_p)2273 rnd_add_data_1(struct krndsource *rs, const void *buf, uint32_t len,
2274 uint32_t entropybits, bool count, uint32_t flag, bool intr_p)
2275 {
2276 bool fullyused;
2277
2278 /*
2279 * For the interrupt-like path, use entropy_enter_intr and take
2280 * note of whether it consumed the full sample; otherwise, use
2281 * entropy_enter, which always consumes the full sample.
2282 */
2283 if (intr_p) {
2284 fullyused = entropy_enter_intr(buf, len, entropybits, count);
2285 } else {
2286 entropy_enter(buf, len, entropybits, count);
2287 fullyused = true;
2288 }
2289
2290 /*
2291 * If we used the full sample, note how many bits were
2292 * contributed from this source.
2293 */
2294 if (fullyused) {
2295 if (__predict_false(cold)) {
2296 const int s = splhigh();
2297 rs->total = add_sat(rs->total, entropybits);
2298 switch (flag) {
2299 case RND_FLAG_COLLECT_TIME:
2300 rs->time_delta.insamples =
2301 add_sat(rs->time_delta.insamples, 1);
2302 break;
2303 case RND_FLAG_COLLECT_VALUE:
2304 rs->value_delta.insamples =
2305 add_sat(rs->value_delta.insamples, 1);
2306 break;
2307 }
2308 splx(s);
2309 } else {
2310 struct rndsource_cpu *rc = percpu_getref(rs->state);
2311
2312 atomic_store_relaxed(&rc->rc_entropybits,
2313 add_sat(rc->rc_entropybits, entropybits));
2314 switch (flag) {
2315 case RND_FLAG_COLLECT_TIME:
2316 atomic_store_relaxed(&rc->rc_timesamples,
2317 add_sat(rc->rc_timesamples, 1));
2318 break;
2319 case RND_FLAG_COLLECT_VALUE:
2320 atomic_store_relaxed(&rc->rc_datasamples,
2321 add_sat(rc->rc_datasamples, 1));
2322 break;
2323 }
2324 percpu_putref(rs->state);
2325 }
2326 }
2327 }
2328
2329 /*
2330 * rnd_add_data_sync(rs, buf, len, entropybits)
2331 *
2332 * Same as rnd_add_data. Originally used in rndsource callbacks,
2333 * to break an unnecessary cycle; no longer really needed.
2334 */
2335 void
rnd_add_data_sync(struct krndsource * rs,const void * buf,uint32_t len,uint32_t entropybits)2336 rnd_add_data_sync(struct krndsource *rs, const void *buf, uint32_t len,
2337 uint32_t entropybits)
2338 {
2339
2340 rnd_add_data(rs, buf, len, entropybits);
2341 }
2342
2343 /*
2344 * rndsource_entropybits(rs)
2345 *
2346 * Return approximately the number of bits of entropy that have
2347 * been contributed via rs so far. Approximate if other CPUs may
2348 * be calling rnd_add_data concurrently.
2349 */
2350 static unsigned
rndsource_entropybits(struct krndsource * rs)2351 rndsource_entropybits(struct krndsource *rs)
2352 {
2353 unsigned nbits = rs->total;
2354
2355 KASSERT(!cold);
2356 KASSERT(rnd_sources_locked());
2357 percpu_foreach(rs->state, rndsource_entropybits_cpu, &nbits);
2358 return nbits;
2359 }
2360
2361 static void
rndsource_entropybits_cpu(void * ptr,void * cookie,struct cpu_info * ci)2362 rndsource_entropybits_cpu(void *ptr, void *cookie, struct cpu_info *ci)
2363 {
2364 struct rndsource_cpu *rc = ptr;
2365 unsigned *nbitsp = cookie;
2366 unsigned cpu_nbits;
2367
2368 cpu_nbits = atomic_load_relaxed(&rc->rc_entropybits);
2369 *nbitsp += MIN(UINT_MAX - *nbitsp, cpu_nbits);
2370 }
2371
2372 /*
2373 * rndsource_to_user(rs, urs)
2374 *
2375 * Copy a description of rs out to urs for userland.
2376 */
2377 static void
rndsource_to_user(struct krndsource * rs,rndsource_t * urs)2378 rndsource_to_user(struct krndsource *rs, rndsource_t *urs)
2379 {
2380
2381 KASSERT(!cold);
2382 KASSERT(rnd_sources_locked());
2383
2384 /* Avoid kernel memory disclosure. */
2385 memset(urs, 0, sizeof(*urs));
2386
2387 CTASSERT(sizeof(urs->name) == sizeof(rs->name));
2388 strlcpy(urs->name, rs->name, sizeof(urs->name));
2389 urs->total = rndsource_entropybits(rs);
2390 urs->type = rs->type;
2391 urs->flags = atomic_load_relaxed(&rs->flags);
2392 }
2393
2394 /*
2395 * rndsource_to_user_est(rs, urse)
2396 *
2397 * Copy a description of rs and estimation statistics out to urse
2398 * for userland.
2399 */
2400 static void
rndsource_to_user_est(struct krndsource * rs,rndsource_est_t * urse)2401 rndsource_to_user_est(struct krndsource *rs, rndsource_est_t *urse)
2402 {
2403
2404 KASSERT(!cold);
2405 KASSERT(rnd_sources_locked());
2406
2407 /* Avoid kernel memory disclosure. */
2408 memset(urse, 0, sizeof(*urse));
2409
2410 /* Copy out the rndsource description. */
2411 rndsource_to_user(rs, &urse->rt);
2412
2413 /* Gather the statistics. */
2414 urse->dt_samples = rs->time_delta.insamples;
2415 urse->dt_total = 0;
2416 urse->dv_samples = rs->value_delta.insamples;
2417 urse->dv_total = urse->rt.total;
2418 percpu_foreach(rs->state, rndsource_to_user_est_cpu, urse);
2419 }
2420
2421 static void
rndsource_to_user_est_cpu(void * ptr,void * cookie,struct cpu_info * ci)2422 rndsource_to_user_est_cpu(void *ptr, void *cookie, struct cpu_info *ci)
2423 {
2424 struct rndsource_cpu *rc = ptr;
2425 rndsource_est_t *urse = cookie;
2426
2427 urse->dt_samples = add_sat(urse->dt_samples,
2428 atomic_load_relaxed(&rc->rc_timesamples));
2429 urse->dv_samples = add_sat(urse->dv_samples,
2430 atomic_load_relaxed(&rc->rc_datasamples));
2431 }
2432
2433 /*
2434 * entropy_reset_xc(arg1, arg2)
2435 *
2436 * Reset the current CPU's pending entropy to zero.
2437 */
2438 static void
entropy_reset_xc(void * arg1 __unused,void * arg2 __unused)2439 entropy_reset_xc(void *arg1 __unused, void *arg2 __unused)
2440 {
2441 uint32_t extra = entropy_timer();
2442 struct entropy_cpu_lock lock;
2443 struct entropy_cpu *ec;
2444
2445 /*
2446 * With the per-CPU state locked, zero the pending count and
2447 * enter a cycle count for fun.
2448 */
2449 ec = entropy_cpu_get(&lock);
2450 ec->ec_bitspending = 0;
2451 ec->ec_samplespending = 0;
2452 entpool_enter(ec->ec_pool, &extra, sizeof extra);
2453 entropy_cpu_put(&lock, ec);
2454 }
2455
2456 /*
2457 * entropy_reset()
2458 *
2459 * Assume the entropy pool has been exposed, e.g. because the VM
2460 * has been cloned. Nix all the pending entropy and set the
2461 * needed to maximum.
2462 */
2463 void
entropy_reset(void)2464 entropy_reset(void)
2465 {
2466
2467 xc_broadcast(0, &entropy_reset_xc, NULL, NULL);
2468 mutex_enter(&E->lock);
2469 E->bitspending = 0;
2470 E->samplespending = 0;
2471 atomic_store_relaxed(&E->bitsneeded, MINENTROPYBITS);
2472 atomic_store_relaxed(&E->samplesneeded, MINSAMPLES);
2473 E->consolidate = false;
2474 mutex_exit(&E->lock);
2475 }
2476
2477 /*
2478 * entropy_ioctl(cmd, data)
2479 *
2480 * Handle various /dev/random ioctl queries.
2481 */
2482 int
entropy_ioctl(unsigned long cmd,void * data)2483 entropy_ioctl(unsigned long cmd, void *data)
2484 {
2485 struct krndsource *rs;
2486 bool privileged;
2487 int error;
2488
2489 KASSERT(!cold);
2490
2491 /* Verify user's authorization to perform the ioctl. */
2492 switch (cmd) {
2493 case RNDGETENTCNT:
2494 case RNDGETPOOLSTAT:
2495 case RNDGETSRCNUM:
2496 case RNDGETSRCNAME:
2497 case RNDGETESTNUM:
2498 case RNDGETESTNAME:
2499 error = kauth_authorize_device(kauth_cred_get(),
2500 KAUTH_DEVICE_RND_GETPRIV, NULL, NULL, NULL, NULL);
2501 break;
2502 case RNDCTL:
2503 error = kauth_authorize_device(kauth_cred_get(),
2504 KAUTH_DEVICE_RND_SETPRIV, NULL, NULL, NULL, NULL);
2505 break;
2506 case RNDADDDATA:
2507 error = kauth_authorize_device(kauth_cred_get(),
2508 KAUTH_DEVICE_RND_ADDDATA, NULL, NULL, NULL, NULL);
2509 /* Ascertain whether the user's inputs should be counted. */
2510 if (kauth_authorize_device(kauth_cred_get(),
2511 KAUTH_DEVICE_RND_ADDDATA_ESTIMATE,
2512 NULL, NULL, NULL, NULL) == 0)
2513 privileged = true;
2514 break;
2515 default: {
2516 /*
2517 * XXX Hack to avoid changing module ABI so this can be
2518 * pulled up. Later, we can just remove the argument.
2519 */
2520 static const struct fileops fops = {
2521 .fo_ioctl = rnd_system_ioctl,
2522 };
2523 struct file f = {
2524 .f_ops = &fops,
2525 };
2526 MODULE_HOOK_CALL(rnd_ioctl_50_hook, (&f, cmd, data),
2527 enosys(), error);
2528 #if defined(_LP64)
2529 if (error == ENOSYS)
2530 MODULE_HOOK_CALL(rnd_ioctl32_50_hook, (&f, cmd, data),
2531 enosys(), error);
2532 #endif
2533 if (error == ENOSYS)
2534 error = ENOTTY;
2535 break;
2536 }
2537 }
2538
2539 /* If anything went wrong with authorization, stop here. */
2540 if (error)
2541 return error;
2542
2543 /* Dispatch on the command. */
2544 switch (cmd) {
2545 case RNDGETENTCNT: { /* Get current entropy count in bits. */
2546 uint32_t *countp = data;
2547
2548 mutex_enter(&E->lock);
2549 *countp = MINENTROPYBITS - E->bitsneeded;
2550 mutex_exit(&E->lock);
2551
2552 break;
2553 }
2554 case RNDGETPOOLSTAT: { /* Get entropy pool statistics. */
2555 rndpoolstat_t *pstat = data;
2556
2557 mutex_enter(&E->lock);
2558
2559 /* parameters */
2560 pstat->poolsize = ENTPOOL_SIZE/sizeof(uint32_t); /* words */
2561 pstat->threshold = MINENTROPYBITS/NBBY; /* bytes */
2562 pstat->maxentropy = ENTROPY_CAPACITY*NBBY; /* bits */
2563
2564 /* state */
2565 pstat->added = 0; /* XXX total entropy_enter count */
2566 pstat->curentropy = MINENTROPYBITS - E->bitsneeded; /* bits */
2567 pstat->removed = 0; /* XXX total entropy_extract count */
2568 pstat->discarded = 0; /* XXX bits of entropy beyond capacity */
2569
2570 /*
2571 * This used to be bits of data fabricated in some
2572 * sense; we'll take it to mean number of samples,
2573 * excluding the bits of entropy from HWRNG or seed.
2574 */
2575 pstat->generated = MINSAMPLES - E->samplesneeded;
2576 pstat->generated -= MIN(pstat->generated, pstat->curentropy);
2577
2578 mutex_exit(&E->lock);
2579 break;
2580 }
2581 case RNDGETSRCNUM: { /* Get entropy sources by number. */
2582 rndstat_t *stat = data;
2583 uint32_t start = 0, i = 0;
2584
2585 /* Skip if none requested; fail if too many requested. */
2586 if (stat->count == 0)
2587 break;
2588 if (stat->count > RND_MAXSTATCOUNT)
2589 return EINVAL;
2590
2591 /*
2592 * Under the lock, find the first one, copy out as many
2593 * as requested, and report how many we copied out.
2594 */
2595 mutex_enter(&E->lock);
2596 error = rnd_lock_sources(ENTROPY_WAIT|ENTROPY_SIG);
2597 if (error) {
2598 mutex_exit(&E->lock);
2599 return error;
2600 }
2601 LIST_FOREACH(rs, &E->sources, list) {
2602 if (start++ == stat->start)
2603 break;
2604 }
2605 while (i < stat->count && rs != NULL) {
2606 mutex_exit(&E->lock);
2607 rndsource_to_user(rs, &stat->source[i++]);
2608 mutex_enter(&E->lock);
2609 rs = LIST_NEXT(rs, list);
2610 }
2611 KASSERT(i <= stat->count);
2612 stat->count = i;
2613 rnd_unlock_sources();
2614 mutex_exit(&E->lock);
2615 break;
2616 }
2617 case RNDGETESTNUM: { /* Get sources and estimates by number. */
2618 rndstat_est_t *estat = data;
2619 uint32_t start = 0, i = 0;
2620
2621 /* Skip if none requested; fail if too many requested. */
2622 if (estat->count == 0)
2623 break;
2624 if (estat->count > RND_MAXSTATCOUNT)
2625 return EINVAL;
2626
2627 /*
2628 * Under the lock, find the first one, copy out as many
2629 * as requested, and report how many we copied out.
2630 */
2631 mutex_enter(&E->lock);
2632 error = rnd_lock_sources(ENTROPY_WAIT|ENTROPY_SIG);
2633 if (error) {
2634 mutex_exit(&E->lock);
2635 return error;
2636 }
2637 LIST_FOREACH(rs, &E->sources, list) {
2638 if (start++ == estat->start)
2639 break;
2640 }
2641 while (i < estat->count && rs != NULL) {
2642 mutex_exit(&E->lock);
2643 rndsource_to_user_est(rs, &estat->source[i++]);
2644 mutex_enter(&E->lock);
2645 rs = LIST_NEXT(rs, list);
2646 }
2647 KASSERT(i <= estat->count);
2648 estat->count = i;
2649 rnd_unlock_sources();
2650 mutex_exit(&E->lock);
2651 break;
2652 }
2653 case RNDGETSRCNAME: { /* Get entropy sources by name. */
2654 rndstat_name_t *nstat = data;
2655 const size_t n = sizeof(rs->name);
2656
2657 CTASSERT(sizeof(rs->name) == sizeof(nstat->name));
2658
2659 /*
2660 * Under the lock, search by name. If found, copy it
2661 * out; if not found, fail with ENOENT.
2662 */
2663 mutex_enter(&E->lock);
2664 error = rnd_lock_sources(ENTROPY_WAIT|ENTROPY_SIG);
2665 if (error) {
2666 mutex_exit(&E->lock);
2667 return error;
2668 }
2669 LIST_FOREACH(rs, &E->sources, list) {
2670 if (strncmp(rs->name, nstat->name, n) == 0)
2671 break;
2672 }
2673 if (rs != NULL) {
2674 mutex_exit(&E->lock);
2675 rndsource_to_user(rs, &nstat->source);
2676 mutex_enter(&E->lock);
2677 } else {
2678 error = ENOENT;
2679 }
2680 rnd_unlock_sources();
2681 mutex_exit(&E->lock);
2682 break;
2683 }
2684 case RNDGETESTNAME: { /* Get sources and estimates by name. */
2685 rndstat_est_name_t *enstat = data;
2686 const size_t n = sizeof(rs->name);
2687
2688 CTASSERT(sizeof(rs->name) == sizeof(enstat->name));
2689
2690 /*
2691 * Under the lock, search by name. If found, copy it
2692 * out; if not found, fail with ENOENT.
2693 */
2694 mutex_enter(&E->lock);
2695 error = rnd_lock_sources(ENTROPY_WAIT|ENTROPY_SIG);
2696 if (error) {
2697 mutex_exit(&E->lock);
2698 return error;
2699 }
2700 LIST_FOREACH(rs, &E->sources, list) {
2701 if (strncmp(rs->name, enstat->name, n) == 0)
2702 break;
2703 }
2704 if (rs != NULL) {
2705 mutex_exit(&E->lock);
2706 rndsource_to_user_est(rs, &enstat->source);
2707 mutex_enter(&E->lock);
2708 } else {
2709 error = ENOENT;
2710 }
2711 rnd_unlock_sources();
2712 mutex_exit(&E->lock);
2713 break;
2714 }
2715 case RNDCTL: { /* Modify entropy source flags. */
2716 rndctl_t *rndctl = data;
2717 const size_t n = sizeof(rs->name);
2718 uint32_t resetflags = RND_FLAG_NO_ESTIMATE|RND_FLAG_NO_COLLECT;
2719 uint32_t flags;
2720 bool reset = false, request = false;
2721
2722 CTASSERT(sizeof(rs->name) == sizeof(rndctl->name));
2723
2724 /* Whitelist the flags that user can change. */
2725 rndctl->mask &= RND_FLAG_NO_ESTIMATE|RND_FLAG_NO_COLLECT;
2726
2727 /*
2728 * For each matching rndsource, either by type if
2729 * specified or by name if not, set the masked flags.
2730 */
2731 mutex_enter(&E->lock);
2732 LIST_FOREACH(rs, &E->sources, list) {
2733 if (rndctl->type != 0xff) {
2734 if (rs->type != rndctl->type)
2735 continue;
2736 } else if (rndctl->name[0] != '\0') {
2737 if (strncmp(rs->name, rndctl->name, n) != 0)
2738 continue;
2739 }
2740 flags = rs->flags & ~rndctl->mask;
2741 flags |= rndctl->flags & rndctl->mask;
2742 if ((rs->flags & resetflags) == 0 &&
2743 (flags & resetflags) != 0)
2744 reset = true;
2745 if ((rs->flags ^ flags) & resetflags)
2746 request = true;
2747 atomic_store_relaxed(&rs->flags, flags);
2748 }
2749 mutex_exit(&E->lock);
2750
2751 /*
2752 * If we disabled estimation or collection, nix all the
2753 * pending entropy and set needed to the maximum.
2754 */
2755 if (reset)
2756 entropy_reset();
2757
2758 /*
2759 * If we changed any of the estimation or collection
2760 * flags, request new samples from everyone -- either
2761 * to make up for what we just lost, or to get new
2762 * samples from what we just added.
2763 *
2764 * Failing on signal, while waiting for another process
2765 * to finish requesting entropy, is OK here even though
2766 * we have committed side effects, because this ioctl
2767 * command is idempotent, so repeating it is safe.
2768 */
2769 if (request)
2770 error = entropy_gather();
2771 break;
2772 }
2773 case RNDADDDATA: { /* Enter seed into entropy pool. */
2774 rnddata_t *rdata = data;
2775 unsigned entropybits = 0;
2776
2777 if (!atomic_load_relaxed(&entropy_collection))
2778 break; /* thanks but no thanks */
2779 if (rdata->len > MIN(sizeof(rdata->data), UINT32_MAX/NBBY))
2780 return EINVAL;
2781
2782 /*
2783 * This ioctl serves as the userland alternative a
2784 * bootloader-provided seed -- typically furnished by
2785 * /etc/rc.d/random_seed. We accept the user's entropy
2786 * claim only if
2787 *
2788 * (a) the user is privileged, and
2789 * (b) we have not entered a bootloader seed.
2790 *
2791 * under the assumption that the user may use this to
2792 * load a seed from disk that we have already loaded
2793 * from the bootloader, so we don't double-count it.
2794 */
2795 if (privileged && rdata->entropy && rdata->len) {
2796 mutex_enter(&E->lock);
2797 if (!E->seeded) {
2798 entropybits = MIN(rdata->entropy,
2799 MIN(rdata->len, ENTROPY_CAPACITY)*NBBY);
2800 E->seeded = true;
2801 }
2802 mutex_exit(&E->lock);
2803 }
2804
2805 /* Enter the data and consolidate entropy. */
2806 rnd_add_data(&seed_rndsource, rdata->data, rdata->len,
2807 entropybits);
2808 error = entropy_consolidate();
2809 break;
2810 }
2811 default:
2812 error = ENOTTY;
2813 }
2814
2815 /* Return any error that may have come up. */
2816 return error;
2817 }
2818
2819 /* Legacy entry points */
2820
2821 void
rnd_seed(void * seed,size_t len)2822 rnd_seed(void *seed, size_t len)
2823 {
2824
2825 if (len != sizeof(rndsave_t)) {
2826 printf("entropy: invalid seed length: %zu,"
2827 " expected sizeof(rndsave_t) = %zu\n",
2828 len, sizeof(rndsave_t));
2829 return;
2830 }
2831 entropy_seed(seed);
2832 }
2833
2834 void
rnd_init(void)2835 rnd_init(void)
2836 {
2837
2838 entropy_init();
2839 }
2840
2841 void
rnd_init_softint(void)2842 rnd_init_softint(void)
2843 {
2844
2845 entropy_init_late();
2846 entropy_bootrequest();
2847 }
2848
2849 int
rnd_system_ioctl(struct file * fp,unsigned long cmd,void * data)2850 rnd_system_ioctl(struct file *fp, unsigned long cmd, void *data)
2851 {
2852
2853 return entropy_ioctl(cmd, data);
2854 }
2855