1 /*-
2 * Copyright (c) 2002-2006 Sam Leffler. All rights reserved.
3 * Copyright (c) 2021 The FreeBSD Foundation
4 *
5 * Portions of this software were developed by Ararat River
6 * Consulting, LLC under sponsorship of the FreeBSD Foundation.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 /*
31 * Cryptographic Subsystem.
32 *
33 * This code is derived from the Openbsd Cryptographic Framework (OCF)
34 * that has the copyright shown below. Very little of the original
35 * code remains.
36 */
37
38 /*-
39 * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
40 *
41 * This code was written by Angelos D. Keromytis in Athens, Greece, in
42 * February 2000. Network Security Technologies Inc. (NSTI) kindly
43 * supported the development of this code.
44 *
45 * Copyright (c) 2000, 2001 Angelos D. Keromytis
46 *
47 * Permission to use, copy, and modify this software with or without fee
48 * is hereby granted, provided that this entire notice is included in
49 * all source code copies of any software which is or includes a copy or
50 * modification of this software.
51 *
52 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
53 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
54 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
55 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
56 * PURPOSE.
57 */
58
59 #include "opt_compat.h"
60 #include "opt_ddb.h"
61
62 #include <sys/param.h>
63 #include <sys/systm.h>
64 #include <sys/counter.h>
65 #include <sys/kernel.h>
66 #include <sys/kthread.h>
67 #include <sys/linker.h>
68 #include <sys/lock.h>
69 #include <sys/module.h>
70 #include <sys/mutex.h>
71 #include <sys/malloc.h>
72 #include <sys/mbuf.h>
73 #include <sys/proc.h>
74 #include <sys/refcount.h>
75 #include <sys/sdt.h>
76 #include <sys/smp.h>
77 #include <sys/sysctl.h>
78 #include <sys/taskqueue.h>
79 #include <sys/uio.h>
80
81 #include <ddb/ddb.h>
82
83 #include <machine/vmparam.h>
84 #include <vm/uma.h>
85
86 #include <crypto/intake.h>
87 #include <opencrypto/cryptodev.h>
88 #include <opencrypto/xform_auth.h>
89 #include <opencrypto/xform_enc.h>
90
91 #include <sys/kobj.h>
92 #include <sys/bus.h>
93 #include "cryptodev_if.h"
94
95 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
96 #include <machine/pcb.h>
97 #endif
98
99 SDT_PROVIDER_DEFINE(opencrypto);
100
101 /*
102 * Crypto drivers register themselves by allocating a slot in the
103 * crypto_drivers table with crypto_get_driverid() and then registering
104 * each asym algorithm they support with crypto_kregister().
105 */
106 static struct mtx crypto_drivers_mtx; /* lock on driver table */
107 #define CRYPTO_DRIVER_LOCK() mtx_lock(&crypto_drivers_mtx)
108 #define CRYPTO_DRIVER_UNLOCK() mtx_unlock(&crypto_drivers_mtx)
109 #define CRYPTO_DRIVER_ASSERT() mtx_assert(&crypto_drivers_mtx, MA_OWNED)
110
111 /*
112 * Crypto device/driver capabilities structure.
113 *
114 * Synchronization:
115 * (d) - protected by CRYPTO_DRIVER_LOCK()
116 * (q) - protected by CRYPTO_Q_LOCK()
117 * Not tagged fields are read-only.
118 */
119 struct cryptocap {
120 device_t cc_dev;
121 uint32_t cc_hid;
122 uint32_t cc_sessions; /* (d) # of sessions */
123 uint32_t cc_koperations; /* (d) # os asym operations */
124 uint8_t cc_kalg[CRK_ALGORITHM_MAX + 1];
125
126 int cc_flags; /* (d) flags */
127 #define CRYPTOCAP_F_CLEANUP 0x80000000 /* needs resource cleanup */
128 int cc_qblocked; /* (q) symmetric q blocked */
129 int cc_kqblocked; /* (q) asymmetric q blocked */
130 size_t cc_session_size;
131 volatile int cc_refs;
132 };
133
134 static struct cryptocap **crypto_drivers = NULL;
135 static int crypto_drivers_size = 0;
136
137 struct crypto_session {
138 struct cryptocap *cap;
139 struct crypto_session_params csp;
140 uint64_t id;
141 /* Driver softc follows. */
142 };
143
144 /*
145 * There are two queues for crypto requests; one for symmetric (e.g.
146 * cipher) operations and one for asymmetric (e.g. MOD)operations.
147 * A single mutex is used to lock access to both queues. We could
148 * have one per-queue but having one simplifies handling of block/unblock
149 * operations.
150 */
151 static int crp_sleep = 0;
152 static TAILQ_HEAD(cryptop_q ,cryptop) crp_q; /* request queues */
153 static TAILQ_HEAD(,cryptkop) crp_kq;
154 static struct mtx crypto_q_mtx;
155 #define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx)
156 #define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx)
157
158 SYSCTL_NODE(_kern, OID_AUTO, crypto, CTLFLAG_RW, 0,
159 "In-kernel cryptography");
160
161 /*
162 * Taskqueue used to dispatch the crypto requests
163 * that have the CRYPTO_F_ASYNC flag
164 */
165 static struct taskqueue *crypto_tq;
166
167 /*
168 * Crypto seq numbers are operated on with modular arithmetic
169 */
170 #define CRYPTO_SEQ_GT(a,b) ((int)((a)-(b)) > 0)
171
172 struct crypto_ret_worker {
173 struct mtx crypto_ret_mtx;
174
175 TAILQ_HEAD(,cryptop) crp_ordered_ret_q; /* ordered callback queue for symetric jobs */
176 TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queue for symetric jobs */
177 TAILQ_HEAD(,cryptkop) crp_ret_kq; /* callback queue for asym jobs */
178
179 uint32_t reorder_ops; /* total ordered sym jobs received */
180 uint32_t reorder_cur_seq; /* current sym job dispatched */
181
182 struct thread *td;
183 };
184 static struct crypto_ret_worker *crypto_ret_workers = NULL;
185
186 #define CRYPTO_RETW(i) (&crypto_ret_workers[i])
187 #define CRYPTO_RETW_ID(w) ((w) - crypto_ret_workers)
188 #define FOREACH_CRYPTO_RETW(w) \
189 for (w = crypto_ret_workers; w < crypto_ret_workers + crypto_workers_num; ++w)
190
191 #define CRYPTO_RETW_LOCK(w) mtx_lock(&w->crypto_ret_mtx)
192 #define CRYPTO_RETW_UNLOCK(w) mtx_unlock(&w->crypto_ret_mtx)
193 #define CRYPTO_RETW_EMPTY(w) \
194 (TAILQ_EMPTY(&w->crp_ret_q) && TAILQ_EMPTY(&w->crp_ret_kq) && TAILQ_EMPTY(&w->crp_ordered_ret_q))
195
196 static int crypto_workers_num = 0;
197 SYSCTL_INT(_kern_crypto, OID_AUTO, num_workers, CTLFLAG_RDTUN,
198 &crypto_workers_num, 0,
199 "Number of crypto workers used to dispatch crypto jobs");
200 #ifdef COMPAT_FREEBSD12
201 SYSCTL_INT(_kern, OID_AUTO, crypto_workers_num, CTLFLAG_RDTUN,
202 &crypto_workers_num, 0,
203 "Number of crypto workers used to dispatch crypto jobs");
204 #endif
205
206 static uma_zone_t cryptop_zone;
207
208 int crypto_userasymcrypto = 1;
209 SYSCTL_INT(_kern_crypto, OID_AUTO, asym_enable, CTLFLAG_RW,
210 &crypto_userasymcrypto, 0,
211 "Enable user-mode access to asymmetric crypto support");
212 #ifdef COMPAT_FREEBSD12
213 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW,
214 &crypto_userasymcrypto, 0,
215 "Enable/disable user-mode access to asymmetric crypto support");
216 #endif
217
218 int crypto_devallowsoft = 0;
219 SYSCTL_INT(_kern_crypto, OID_AUTO, allow_soft, CTLFLAG_RWTUN,
220 &crypto_devallowsoft, 0,
221 "Enable use of software crypto by /dev/crypto");
222 #ifdef COMPAT_FREEBSD12
223 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RWTUN,
224 &crypto_devallowsoft, 0,
225 "Enable/disable use of software crypto by /dev/crypto");
226 #endif
227
228 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
229
230 static void crypto_dispatch_thread(void *arg);
231 static struct thread *cryptotd;
232 static void crypto_ret_thread(void *arg);
233 static void crypto_destroy(void);
234 static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint);
235 static int crypto_kinvoke(struct cryptkop *krp);
236 static void crypto_task_invoke(void *ctx, int pending);
237 static void crypto_batch_enqueue(struct cryptop *crp);
238
239 static counter_u64_t cryptostats[sizeof(struct cryptostats) / sizeof(uint64_t)];
240 SYSCTL_COUNTER_U64_ARRAY(_kern_crypto, OID_AUTO, stats, CTLFLAG_RW,
241 cryptostats, nitems(cryptostats),
242 "Crypto system statistics");
243
244 #define CRYPTOSTAT_INC(stat) do { \
245 counter_u64_add( \
246 cryptostats[offsetof(struct cryptostats, stat) / sizeof(uint64_t)],\
247 1); \
248 } while (0)
249
250 static void
cryptostats_init(void * arg __unused)251 cryptostats_init(void *arg __unused)
252 {
253 COUNTER_ARRAY_ALLOC(cryptostats, nitems(cryptostats), M_WAITOK);
254 }
255 SYSINIT(cryptostats_init, SI_SUB_COUNTER, SI_ORDER_ANY, cryptostats_init, NULL);
256
257 static void
cryptostats_fini(void * arg __unused)258 cryptostats_fini(void *arg __unused)
259 {
260 COUNTER_ARRAY_FREE(cryptostats, nitems(cryptostats));
261 }
262 SYSUNINIT(cryptostats_fini, SI_SUB_COUNTER, SI_ORDER_ANY, cryptostats_fini,
263 NULL);
264
265 /* Try to avoid directly exposing the key buffer as a symbol */
266 static struct keybuf *keybuf;
267
268 static struct keybuf empty_keybuf = {
269 .kb_nents = 0
270 };
271
272 /* Obtain the key buffer from boot metadata */
273 static void
keybuf_init(void)274 keybuf_init(void)
275 {
276 caddr_t kmdp;
277
278 kmdp = preload_search_by_type("elf kernel");
279
280 if (kmdp == NULL)
281 kmdp = preload_search_by_type("elf64 kernel");
282
283 keybuf = (struct keybuf *)preload_search_info(kmdp,
284 MODINFO_METADATA | MODINFOMD_KEYBUF);
285
286 if (keybuf == NULL)
287 keybuf = &empty_keybuf;
288 }
289
290 /* It'd be nice if we could store these in some kind of secure memory... */
291 struct keybuf *
get_keybuf(void)292 get_keybuf(void)
293 {
294
295 return (keybuf);
296 }
297
298 static struct cryptocap *
cap_ref(struct cryptocap * cap)299 cap_ref(struct cryptocap *cap)
300 {
301
302 refcount_acquire(&cap->cc_refs);
303 return (cap);
304 }
305
306 static void
cap_rele(struct cryptocap * cap)307 cap_rele(struct cryptocap *cap)
308 {
309
310 if (refcount_release(&cap->cc_refs) == 0)
311 return;
312
313 KASSERT(cap->cc_sessions == 0,
314 ("freeing crypto driver with active sessions"));
315 KASSERT(cap->cc_koperations == 0,
316 ("freeing crypto driver with active key operations"));
317
318 free(cap, M_CRYPTO_DATA);
319 }
320
321 static int
crypto_init(void)322 crypto_init(void)
323 {
324 struct crypto_ret_worker *ret_worker;
325 struct proc *p;
326 int error;
327
328 mtx_init(&crypto_drivers_mtx, "crypto driver table", NULL, MTX_DEF);
329
330 TAILQ_INIT(&crp_q);
331 TAILQ_INIT(&crp_kq);
332 mtx_init(&crypto_q_mtx, "crypto op queues", NULL, MTX_DEF);
333
334 cryptop_zone = uma_zcreate("cryptop",
335 sizeof(struct cryptop), NULL, NULL, NULL, NULL,
336 UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
337
338 crypto_drivers_size = CRYPTO_DRIVERS_INITIAL;
339 crypto_drivers = malloc(crypto_drivers_size *
340 sizeof(struct cryptocap), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
341
342 if (crypto_workers_num < 1 || crypto_workers_num > mp_ncpus)
343 crypto_workers_num = mp_ncpus;
344
345 crypto_tq = taskqueue_create("crypto", M_WAITOK | M_ZERO,
346 taskqueue_thread_enqueue, &crypto_tq);
347
348 taskqueue_start_threads(&crypto_tq, crypto_workers_num, PRI_MIN_KERN,
349 "crypto");
350
351 p = NULL;
352 error = kproc_kthread_add(crypto_dispatch_thread, NULL, &p, &cryptotd,
353 0, 0, "crypto", "crypto");
354 if (error) {
355 printf("crypto_init: cannot start crypto thread; error %d",
356 error);
357 goto bad;
358 }
359
360 crypto_ret_workers = mallocarray(crypto_workers_num,
361 sizeof(struct crypto_ret_worker), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
362
363 FOREACH_CRYPTO_RETW(ret_worker) {
364 TAILQ_INIT(&ret_worker->crp_ordered_ret_q);
365 TAILQ_INIT(&ret_worker->crp_ret_q);
366 TAILQ_INIT(&ret_worker->crp_ret_kq);
367
368 ret_worker->reorder_ops = 0;
369 ret_worker->reorder_cur_seq = 0;
370
371 mtx_init(&ret_worker->crypto_ret_mtx, "crypto return queues",
372 NULL, MTX_DEF);
373
374 error = kthread_add(crypto_ret_thread, ret_worker, p,
375 &ret_worker->td, 0, 0, "crypto returns %td",
376 CRYPTO_RETW_ID(ret_worker));
377 if (error) {
378 printf("crypto_init: cannot start cryptoret thread; error %d",
379 error);
380 goto bad;
381 }
382 }
383
384 keybuf_init();
385
386 return 0;
387 bad:
388 crypto_destroy();
389 return error;
390 }
391
392 /*
393 * Signal a crypto thread to terminate. We use the driver
394 * table lock to synchronize the sleep/wakeups so that we
395 * are sure the threads have terminated before we release
396 * the data structures they use. See crypto_finis below
397 * for the other half of this song-and-dance.
398 */
399 static void
crypto_terminate(struct thread ** tdp,void * q)400 crypto_terminate(struct thread **tdp, void *q)
401 {
402 struct thread *td;
403
404 mtx_assert(&crypto_drivers_mtx, MA_OWNED);
405 td = *tdp;
406 *tdp = NULL;
407 if (td != NULL) {
408 wakeup_one(q);
409 mtx_sleep(td, &crypto_drivers_mtx, PWAIT, "crypto_destroy", 0);
410 }
411 }
412
413 static void
hmac_init_pad(const struct auth_hash * axf,const char * key,int klen,void * auth_ctx,uint8_t padval)414 hmac_init_pad(const struct auth_hash *axf, const char *key, int klen,
415 void *auth_ctx, uint8_t padval)
416 {
417 uint8_t hmac_key[HMAC_MAX_BLOCK_LEN];
418 u_int i;
419
420 KASSERT(axf->blocksize <= sizeof(hmac_key),
421 ("Invalid HMAC block size %d", axf->blocksize));
422
423 /*
424 * If the key is larger than the block size, use the digest of
425 * the key as the key instead.
426 */
427 memset(hmac_key, 0, sizeof(hmac_key));
428 if (klen > axf->blocksize) {
429 axf->Init(auth_ctx);
430 axf->Update(auth_ctx, key, klen);
431 axf->Final(hmac_key, auth_ctx);
432 klen = axf->hashsize;
433 } else
434 memcpy(hmac_key, key, klen);
435
436 for (i = 0; i < axf->blocksize; i++)
437 hmac_key[i] ^= padval;
438
439 axf->Init(auth_ctx);
440 axf->Update(auth_ctx, hmac_key, axf->blocksize);
441 explicit_bzero(hmac_key, sizeof(hmac_key));
442 }
443
444 void
hmac_init_ipad(const struct auth_hash * axf,const char * key,int klen,void * auth_ctx)445 hmac_init_ipad(const struct auth_hash *axf, const char *key, int klen,
446 void *auth_ctx)
447 {
448
449 hmac_init_pad(axf, key, klen, auth_ctx, HMAC_IPAD_VAL);
450 }
451
452 void
hmac_init_opad(const struct auth_hash * axf,const char * key,int klen,void * auth_ctx)453 hmac_init_opad(const struct auth_hash *axf, const char *key, int klen,
454 void *auth_ctx)
455 {
456
457 hmac_init_pad(axf, key, klen, auth_ctx, HMAC_OPAD_VAL);
458 }
459
460 static void
crypto_destroy(void)461 crypto_destroy(void)
462 {
463 struct crypto_ret_worker *ret_worker;
464 int i;
465
466 /*
467 * Terminate any crypto threads.
468 */
469 if (crypto_tq != NULL)
470 taskqueue_drain_all(crypto_tq);
471 CRYPTO_DRIVER_LOCK();
472 crypto_terminate(&cryptotd, &crp_q);
473 FOREACH_CRYPTO_RETW(ret_worker)
474 crypto_terminate(&ret_worker->td, &ret_worker->crp_ret_q);
475 CRYPTO_DRIVER_UNLOCK();
476
477 /* XXX flush queues??? */
478
479 /*
480 * Reclaim dynamically allocated resources.
481 */
482 for (i = 0; i < crypto_drivers_size; i++) {
483 if (crypto_drivers[i] != NULL)
484 cap_rele(crypto_drivers[i]);
485 }
486 free(crypto_drivers, M_CRYPTO_DATA);
487
488 if (cryptop_zone != NULL)
489 uma_zdestroy(cryptop_zone);
490 mtx_destroy(&crypto_q_mtx);
491 FOREACH_CRYPTO_RETW(ret_worker)
492 mtx_destroy(&ret_worker->crypto_ret_mtx);
493 free(crypto_ret_workers, M_CRYPTO_DATA);
494 if (crypto_tq != NULL)
495 taskqueue_free(crypto_tq);
496 mtx_destroy(&crypto_drivers_mtx);
497 }
498
499 uint32_t
crypto_ses2hid(crypto_session_t crypto_session)500 crypto_ses2hid(crypto_session_t crypto_session)
501 {
502 return (crypto_session->cap->cc_hid);
503 }
504
505 uint32_t
crypto_ses2caps(crypto_session_t crypto_session)506 crypto_ses2caps(crypto_session_t crypto_session)
507 {
508 return (crypto_session->cap->cc_flags & 0xff000000);
509 }
510
511 void *
crypto_get_driver_session(crypto_session_t crypto_session)512 crypto_get_driver_session(crypto_session_t crypto_session)
513 {
514 return (crypto_session + 1);
515 }
516
517 const struct crypto_session_params *
crypto_get_params(crypto_session_t crypto_session)518 crypto_get_params(crypto_session_t crypto_session)
519 {
520 return (&crypto_session->csp);
521 }
522
523 struct auth_hash *
crypto_auth_hash(const struct crypto_session_params * csp)524 crypto_auth_hash(const struct crypto_session_params *csp)
525 {
526
527 switch (csp->csp_auth_alg) {
528 case CRYPTO_SHA1_HMAC:
529 return (&auth_hash_hmac_sha1);
530 case CRYPTO_SHA2_224_HMAC:
531 return (&auth_hash_hmac_sha2_224);
532 case CRYPTO_SHA2_256_HMAC:
533 return (&auth_hash_hmac_sha2_256);
534 case CRYPTO_SHA2_384_HMAC:
535 return (&auth_hash_hmac_sha2_384);
536 case CRYPTO_SHA2_512_HMAC:
537 return (&auth_hash_hmac_sha2_512);
538 case CRYPTO_NULL_HMAC:
539 return (&auth_hash_null);
540 case CRYPTO_RIPEMD160_HMAC:
541 return (&auth_hash_hmac_ripemd_160);
542 case CRYPTO_SHA1:
543 return (&auth_hash_sha1);
544 case CRYPTO_SHA2_224:
545 return (&auth_hash_sha2_224);
546 case CRYPTO_SHA2_256:
547 return (&auth_hash_sha2_256);
548 case CRYPTO_SHA2_384:
549 return (&auth_hash_sha2_384);
550 case CRYPTO_SHA2_512:
551 return (&auth_hash_sha2_512);
552 case CRYPTO_AES_NIST_GMAC:
553 switch (csp->csp_auth_klen) {
554 case 128 / 8:
555 return (&auth_hash_nist_gmac_aes_128);
556 case 192 / 8:
557 return (&auth_hash_nist_gmac_aes_192);
558 case 256 / 8:
559 return (&auth_hash_nist_gmac_aes_256);
560 default:
561 return (NULL);
562 }
563 case CRYPTO_BLAKE2B:
564 return (&auth_hash_blake2b);
565 case CRYPTO_BLAKE2S:
566 return (&auth_hash_blake2s);
567 case CRYPTO_POLY1305:
568 return (&auth_hash_poly1305);
569 case CRYPTO_AES_CCM_CBC_MAC:
570 switch (csp->csp_auth_klen) {
571 case 128 / 8:
572 return (&auth_hash_ccm_cbc_mac_128);
573 case 192 / 8:
574 return (&auth_hash_ccm_cbc_mac_192);
575 case 256 / 8:
576 return (&auth_hash_ccm_cbc_mac_256);
577 default:
578 return (NULL);
579 }
580 default:
581 return (NULL);
582 }
583 }
584
585 struct enc_xform *
crypto_cipher(const struct crypto_session_params * csp)586 crypto_cipher(const struct crypto_session_params *csp)
587 {
588
589 switch (csp->csp_cipher_alg) {
590 case CRYPTO_RIJNDAEL128_CBC:
591 return (&enc_xform_rijndael128);
592 case CRYPTO_AES_XTS:
593 return (&enc_xform_aes_xts);
594 case CRYPTO_AES_ICM:
595 return (&enc_xform_aes_icm);
596 case CRYPTO_AES_NIST_GCM_16:
597 return (&enc_xform_aes_nist_gcm);
598 case CRYPTO_CAMELLIA_CBC:
599 return (&enc_xform_camellia);
600 case CRYPTO_NULL_CBC:
601 return (&enc_xform_null);
602 case CRYPTO_CHACHA20:
603 return (&enc_xform_chacha20);
604 case CRYPTO_AES_CCM_16:
605 return (&enc_xform_ccm);
606 case CRYPTO_CHACHA20_POLY1305:
607 return (&enc_xform_chacha20_poly1305);
608 default:
609 return (NULL);
610 }
611 }
612
613 static struct cryptocap *
crypto_checkdriver(uint32_t hid)614 crypto_checkdriver(uint32_t hid)
615 {
616
617 return (hid >= crypto_drivers_size ? NULL : crypto_drivers[hid]);
618 }
619
620 /*
621 * Select a driver for a new session that supports the specified
622 * algorithms and, optionally, is constrained according to the flags.
623 */
624 static struct cryptocap *
crypto_select_driver(const struct crypto_session_params * csp,int flags)625 crypto_select_driver(const struct crypto_session_params *csp, int flags)
626 {
627 struct cryptocap *cap, *best;
628 int best_match, error, hid;
629
630 CRYPTO_DRIVER_ASSERT();
631
632 best = NULL;
633 for (hid = 0; hid < crypto_drivers_size; hid++) {
634 /*
635 * If there is no driver for this slot, or the driver
636 * is not appropriate (hardware or software based on
637 * match), then skip.
638 */
639 cap = crypto_drivers[hid];
640 if (cap == NULL ||
641 (cap->cc_flags & flags) == 0)
642 continue;
643
644 error = CRYPTODEV_PROBESESSION(cap->cc_dev, csp);
645 if (error >= 0)
646 continue;
647
648 /*
649 * Use the driver with the highest probe value.
650 * Hardware drivers use a higher probe value than
651 * software. In case of a tie, prefer the driver with
652 * the fewest active sessions.
653 */
654 if (best == NULL || error > best_match ||
655 (error == best_match &&
656 cap->cc_sessions < best->cc_sessions)) {
657 best = cap;
658 best_match = error;
659 }
660 }
661 return best;
662 }
663
664 static enum alg_type {
665 ALG_NONE = 0,
666 ALG_CIPHER,
667 ALG_DIGEST,
668 ALG_KEYED_DIGEST,
669 ALG_COMPRESSION,
670 ALG_AEAD
671 } alg_types[] = {
672 [CRYPTO_SHA1_HMAC] = ALG_KEYED_DIGEST,
673 [CRYPTO_RIPEMD160_HMAC] = ALG_KEYED_DIGEST,
674 [CRYPTO_AES_CBC] = ALG_CIPHER,
675 [CRYPTO_SHA1] = ALG_DIGEST,
676 [CRYPTO_NULL_HMAC] = ALG_DIGEST,
677 [CRYPTO_NULL_CBC] = ALG_CIPHER,
678 [CRYPTO_DEFLATE_COMP] = ALG_COMPRESSION,
679 [CRYPTO_SHA2_256_HMAC] = ALG_KEYED_DIGEST,
680 [CRYPTO_SHA2_384_HMAC] = ALG_KEYED_DIGEST,
681 [CRYPTO_SHA2_512_HMAC] = ALG_KEYED_DIGEST,
682 [CRYPTO_CAMELLIA_CBC] = ALG_CIPHER,
683 [CRYPTO_AES_XTS] = ALG_CIPHER,
684 [CRYPTO_AES_ICM] = ALG_CIPHER,
685 [CRYPTO_AES_NIST_GMAC] = ALG_KEYED_DIGEST,
686 [CRYPTO_AES_NIST_GCM_16] = ALG_AEAD,
687 [CRYPTO_BLAKE2B] = ALG_KEYED_DIGEST,
688 [CRYPTO_BLAKE2S] = ALG_KEYED_DIGEST,
689 [CRYPTO_CHACHA20] = ALG_CIPHER,
690 [CRYPTO_SHA2_224_HMAC] = ALG_KEYED_DIGEST,
691 [CRYPTO_RIPEMD160] = ALG_DIGEST,
692 [CRYPTO_SHA2_224] = ALG_DIGEST,
693 [CRYPTO_SHA2_256] = ALG_DIGEST,
694 [CRYPTO_SHA2_384] = ALG_DIGEST,
695 [CRYPTO_SHA2_512] = ALG_DIGEST,
696 [CRYPTO_POLY1305] = ALG_KEYED_DIGEST,
697 [CRYPTO_AES_CCM_CBC_MAC] = ALG_KEYED_DIGEST,
698 [CRYPTO_AES_CCM_16] = ALG_AEAD,
699 [CRYPTO_CHACHA20_POLY1305] = ALG_AEAD,
700 };
701
702 static enum alg_type
alg_type(int alg)703 alg_type(int alg)
704 {
705
706 if (alg < nitems(alg_types))
707 return (alg_types[alg]);
708 return (ALG_NONE);
709 }
710
711 static bool
alg_is_compression(int alg)712 alg_is_compression(int alg)
713 {
714
715 return (alg_type(alg) == ALG_COMPRESSION);
716 }
717
718 static bool
alg_is_cipher(int alg)719 alg_is_cipher(int alg)
720 {
721
722 return (alg_type(alg) == ALG_CIPHER);
723 }
724
725 static bool
alg_is_digest(int alg)726 alg_is_digest(int alg)
727 {
728
729 return (alg_type(alg) == ALG_DIGEST ||
730 alg_type(alg) == ALG_KEYED_DIGEST);
731 }
732
733 static bool
alg_is_keyed_digest(int alg)734 alg_is_keyed_digest(int alg)
735 {
736
737 return (alg_type(alg) == ALG_KEYED_DIGEST);
738 }
739
740 static bool
alg_is_aead(int alg)741 alg_is_aead(int alg)
742 {
743
744 return (alg_type(alg) == ALG_AEAD);
745 }
746
747 static bool
ccm_tag_length_valid(int len)748 ccm_tag_length_valid(int len)
749 {
750 /* RFC 3610 */
751 switch (len) {
752 case 4:
753 case 6:
754 case 8:
755 case 10:
756 case 12:
757 case 14:
758 case 16:
759 return (true);
760 default:
761 return (false);
762 }
763 }
764
765 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN)
766
767 /* Various sanity checks on crypto session parameters. */
768 static bool
check_csp(const struct crypto_session_params * csp)769 check_csp(const struct crypto_session_params *csp)
770 {
771 struct auth_hash *axf;
772
773 /* Mode-independent checks. */
774 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0)
775 return (false);
776 if (csp->csp_ivlen < 0 || csp->csp_cipher_klen < 0 ||
777 csp->csp_auth_klen < 0 || csp->csp_auth_mlen < 0)
778 return (false);
779 if (csp->csp_auth_key != NULL && csp->csp_auth_klen == 0)
780 return (false);
781 if (csp->csp_cipher_key != NULL && csp->csp_cipher_klen == 0)
782 return (false);
783
784 switch (csp->csp_mode) {
785 case CSP_MODE_COMPRESS:
786 if (!alg_is_compression(csp->csp_cipher_alg))
787 return (false);
788 if (csp->csp_flags & CSP_F_SEPARATE_OUTPUT)
789 return (false);
790 if (csp->csp_flags & CSP_F_SEPARATE_AAD)
791 return (false);
792 if (csp->csp_cipher_klen != 0 || csp->csp_ivlen != 0 ||
793 csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 ||
794 csp->csp_auth_mlen != 0)
795 return (false);
796 break;
797 case CSP_MODE_CIPHER:
798 if (!alg_is_cipher(csp->csp_cipher_alg))
799 return (false);
800 if (csp->csp_flags & CSP_F_SEPARATE_AAD)
801 return (false);
802 if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) {
803 if (csp->csp_cipher_klen == 0)
804 return (false);
805 if (csp->csp_ivlen == 0)
806 return (false);
807 }
808 if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
809 return (false);
810 if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 ||
811 csp->csp_auth_mlen != 0)
812 return (false);
813 break;
814 case CSP_MODE_DIGEST:
815 if (csp->csp_cipher_alg != 0 || csp->csp_cipher_klen != 0)
816 return (false);
817
818 if (csp->csp_flags & CSP_F_SEPARATE_AAD)
819 return (false);
820
821 /* IV is optional for digests (e.g. GMAC). */
822 switch (csp->csp_auth_alg) {
823 case CRYPTO_AES_CCM_CBC_MAC:
824 if (csp->csp_ivlen < 7 || csp->csp_ivlen > 13)
825 return (false);
826 break;
827 case CRYPTO_AES_NIST_GMAC:
828 if (csp->csp_ivlen != AES_GCM_IV_LEN)
829 return (false);
830 break;
831 default:
832 if (csp->csp_ivlen != 0)
833 return (false);
834 break;
835 }
836
837 if (!alg_is_digest(csp->csp_auth_alg))
838 return (false);
839
840 /* Key is optional for BLAKE2 digests. */
841 if (csp->csp_auth_alg == CRYPTO_BLAKE2B ||
842 csp->csp_auth_alg == CRYPTO_BLAKE2S)
843 ;
844 else if (alg_is_keyed_digest(csp->csp_auth_alg)) {
845 if (csp->csp_auth_klen == 0)
846 return (false);
847 } else {
848 if (csp->csp_auth_klen != 0)
849 return (false);
850 }
851 if (csp->csp_auth_mlen != 0) {
852 axf = crypto_auth_hash(csp);
853 if (axf == NULL || csp->csp_auth_mlen > axf->hashsize)
854 return (false);
855
856 if (csp->csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC &&
857 !ccm_tag_length_valid(csp->csp_auth_mlen))
858 return (false);
859 }
860 break;
861 case CSP_MODE_AEAD:
862 if (!alg_is_aead(csp->csp_cipher_alg))
863 return (false);
864 if (csp->csp_cipher_klen == 0)
865 return (false);
866 if (csp->csp_ivlen == 0 ||
867 csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
868 return (false);
869 if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0)
870 return (false);
871
872 switch (csp->csp_cipher_alg) {
873 case CRYPTO_AES_CCM_16:
874 if (csp->csp_auth_mlen != 0 &&
875 !ccm_tag_length_valid(csp->csp_auth_mlen))
876 return (false);
877
878 if (csp->csp_ivlen < 7 || csp->csp_ivlen > 13)
879 return (false);
880 break;
881 case CRYPTO_AES_NIST_GCM_16:
882 if (csp->csp_auth_mlen > AES_GMAC_HASH_LEN)
883 return (false);
884
885 if (csp->csp_ivlen != AES_GCM_IV_LEN)
886 return (false);
887 break;
888 case CRYPTO_CHACHA20_POLY1305:
889 if (csp->csp_ivlen != 8 && csp->csp_ivlen != 12)
890 return (false);
891 if (csp->csp_auth_mlen > POLY1305_HASH_LEN)
892 return (false);
893 break;
894 }
895 break;
896 case CSP_MODE_ETA:
897 if (!alg_is_cipher(csp->csp_cipher_alg))
898 return (false);
899 if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) {
900 if (csp->csp_cipher_klen == 0)
901 return (false);
902 if (csp->csp_ivlen == 0)
903 return (false);
904 }
905 if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
906 return (false);
907 if (!alg_is_digest(csp->csp_auth_alg))
908 return (false);
909
910 /* Key is optional for BLAKE2 digests. */
911 if (csp->csp_auth_alg == CRYPTO_BLAKE2B ||
912 csp->csp_auth_alg == CRYPTO_BLAKE2S)
913 ;
914 else if (alg_is_keyed_digest(csp->csp_auth_alg)) {
915 if (csp->csp_auth_klen == 0)
916 return (false);
917 } else {
918 if (csp->csp_auth_klen != 0)
919 return (false);
920 }
921 if (csp->csp_auth_mlen != 0) {
922 axf = crypto_auth_hash(csp);
923 if (axf == NULL || csp->csp_auth_mlen > axf->hashsize)
924 return (false);
925 }
926 break;
927 default:
928 return (false);
929 }
930
931 return (true);
932 }
933
934 /*
935 * Delete a session after it has been detached from its driver.
936 */
937 static void
crypto_deletesession(crypto_session_t cses)938 crypto_deletesession(crypto_session_t cses)
939 {
940 struct cryptocap *cap;
941
942 cap = cses->cap;
943
944 zfree(cses, M_CRYPTO_DATA);
945
946 CRYPTO_DRIVER_LOCK();
947 cap->cc_sessions--;
948 if (cap->cc_sessions == 0 && cap->cc_flags & CRYPTOCAP_F_CLEANUP)
949 wakeup(cap);
950 CRYPTO_DRIVER_UNLOCK();
951 cap_rele(cap);
952 }
953
954 /*
955 * Create a new session. The crid argument specifies a crypto
956 * driver to use or constraints on a driver to select (hardware
957 * only, software only, either). Whatever driver is selected
958 * must be capable of the requested crypto algorithms.
959 */
960 int
crypto_newsession(crypto_session_t * cses,const struct crypto_session_params * csp,int crid)961 crypto_newsession(crypto_session_t *cses,
962 const struct crypto_session_params *csp, int crid)
963 {
964 static uint64_t sessid = 0;
965 crypto_session_t res;
966 struct cryptocap *cap;
967 int err;
968
969 if (!check_csp(csp))
970 return (EINVAL);
971
972 res = NULL;
973
974 CRYPTO_DRIVER_LOCK();
975 if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
976 /*
977 * Use specified driver; verify it is capable.
978 */
979 cap = crypto_checkdriver(crid);
980 if (cap != NULL && CRYPTODEV_PROBESESSION(cap->cc_dev, csp) > 0)
981 cap = NULL;
982 } else {
983 /*
984 * No requested driver; select based on crid flags.
985 */
986 cap = crypto_select_driver(csp, crid);
987 }
988 if (cap == NULL) {
989 CRYPTO_DRIVER_UNLOCK();
990 CRYPTDEB("no driver");
991 return (EOPNOTSUPP);
992 }
993 cap_ref(cap);
994 cap->cc_sessions++;
995 CRYPTO_DRIVER_UNLOCK();
996
997 /* Allocate a single block for the generic session and driver softc. */
998 res = malloc(sizeof(*res) + cap->cc_session_size, M_CRYPTO_DATA,
999 M_WAITOK | M_ZERO);
1000 res->cap = cap;
1001 res->csp = *csp;
1002 res->id = atomic_fetchadd_64(&sessid, 1);
1003
1004 /* Call the driver initialization routine. */
1005 err = CRYPTODEV_NEWSESSION(cap->cc_dev, res, csp);
1006 if (err != 0) {
1007 CRYPTDEB("dev newsession failed: %d", err);
1008 crypto_deletesession(res);
1009 return (err);
1010 }
1011
1012 *cses = res;
1013 return (0);
1014 }
1015
1016 /*
1017 * Delete an existing session (or a reserved session on an unregistered
1018 * driver).
1019 */
1020 void
crypto_freesession(crypto_session_t cses)1021 crypto_freesession(crypto_session_t cses)
1022 {
1023 struct cryptocap *cap;
1024
1025 if (cses == NULL)
1026 return;
1027
1028 cap = cses->cap;
1029
1030 /* Call the driver cleanup routine, if available. */
1031 CRYPTODEV_FREESESSION(cap->cc_dev, cses);
1032
1033 crypto_deletesession(cses);
1034 }
1035
1036 /*
1037 * Return a new driver id. Registers a driver with the system so that
1038 * it can be probed by subsequent sessions.
1039 */
1040 int32_t
crypto_get_driverid(device_t dev,size_t sessionsize,int flags)1041 crypto_get_driverid(device_t dev, size_t sessionsize, int flags)
1042 {
1043 struct cryptocap *cap, **newdrv;
1044 int i;
1045
1046 if ((flags & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
1047 device_printf(dev,
1048 "no flags specified when registering driver\n");
1049 return -1;
1050 }
1051
1052 cap = malloc(sizeof(*cap), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
1053 cap->cc_dev = dev;
1054 cap->cc_session_size = sessionsize;
1055 cap->cc_flags = flags;
1056 refcount_init(&cap->cc_refs, 1);
1057
1058 CRYPTO_DRIVER_LOCK();
1059 for (;;) {
1060 for (i = 0; i < crypto_drivers_size; i++) {
1061 if (crypto_drivers[i] == NULL)
1062 break;
1063 }
1064
1065 if (i < crypto_drivers_size)
1066 break;
1067
1068 /* Out of entries, allocate some more. */
1069
1070 if (2 * crypto_drivers_size <= crypto_drivers_size) {
1071 CRYPTO_DRIVER_UNLOCK();
1072 printf("crypto: driver count wraparound!\n");
1073 cap_rele(cap);
1074 return (-1);
1075 }
1076 CRYPTO_DRIVER_UNLOCK();
1077
1078 newdrv = malloc(2 * crypto_drivers_size *
1079 sizeof(*crypto_drivers), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
1080
1081 CRYPTO_DRIVER_LOCK();
1082 memcpy(newdrv, crypto_drivers,
1083 crypto_drivers_size * sizeof(*crypto_drivers));
1084
1085 crypto_drivers_size *= 2;
1086
1087 free(crypto_drivers, M_CRYPTO_DATA);
1088 crypto_drivers = newdrv;
1089 }
1090
1091 cap->cc_hid = i;
1092 crypto_drivers[i] = cap;
1093 CRYPTO_DRIVER_UNLOCK();
1094
1095 if (bootverbose)
1096 printf("crypto: assign %s driver id %u, flags 0x%x\n",
1097 device_get_nameunit(dev), i, flags);
1098
1099 return i;
1100 }
1101
1102 /*
1103 * Lookup a driver by name. We match against the full device
1104 * name and unit, and against just the name. The latter gives
1105 * us a simple widlcarding by device name. On success return the
1106 * driver/hardware identifier; otherwise return -1.
1107 */
1108 int
crypto_find_driver(const char * match)1109 crypto_find_driver(const char *match)
1110 {
1111 struct cryptocap *cap;
1112 int i, len = strlen(match);
1113
1114 CRYPTO_DRIVER_LOCK();
1115 for (i = 0; i < crypto_drivers_size; i++) {
1116 if (crypto_drivers[i] == NULL)
1117 continue;
1118 cap = crypto_drivers[i];
1119 if (strncmp(match, device_get_nameunit(cap->cc_dev), len) == 0 ||
1120 strncmp(match, device_get_name(cap->cc_dev), len) == 0) {
1121 CRYPTO_DRIVER_UNLOCK();
1122 return (i);
1123 }
1124 }
1125 CRYPTO_DRIVER_UNLOCK();
1126 return (-1);
1127 }
1128
1129 /*
1130 * Return the device_t for the specified driver or NULL
1131 * if the driver identifier is invalid.
1132 */
1133 device_t
crypto_find_device_byhid(int hid)1134 crypto_find_device_byhid(int hid)
1135 {
1136 struct cryptocap *cap;
1137 device_t dev;
1138
1139 dev = NULL;
1140 CRYPTO_DRIVER_LOCK();
1141 cap = crypto_checkdriver(hid);
1142 if (cap != NULL)
1143 dev = cap->cc_dev;
1144 CRYPTO_DRIVER_UNLOCK();
1145 return (dev);
1146 }
1147
1148 /*
1149 * Return the device/driver capabilities.
1150 */
1151 int
crypto_getcaps(int hid)1152 crypto_getcaps(int hid)
1153 {
1154 struct cryptocap *cap;
1155 int flags;
1156
1157 flags = 0;
1158 CRYPTO_DRIVER_LOCK();
1159 cap = crypto_checkdriver(hid);
1160 if (cap != NULL)
1161 flags = cap->cc_flags;
1162 CRYPTO_DRIVER_UNLOCK();
1163 return (flags);
1164 }
1165
1166 /*
1167 * Register support for a key-related algorithm. This routine
1168 * is called once for each algorithm supported a driver.
1169 */
1170 int
crypto_kregister(uint32_t driverid,int kalg,uint32_t flags)1171 crypto_kregister(uint32_t driverid, int kalg, uint32_t flags)
1172 {
1173 struct cryptocap *cap;
1174 int err;
1175
1176 CRYPTO_DRIVER_LOCK();
1177
1178 cap = crypto_checkdriver(driverid);
1179 if (cap != NULL &&
1180 (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) {
1181 /*
1182 * XXX Do some performance testing to determine placing.
1183 * XXX We probably need an auxiliary data structure that
1184 * XXX describes relative performances.
1185 */
1186
1187 cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
1188 if (bootverbose)
1189 printf("crypto: %s registers key alg %u flags %u\n"
1190 , device_get_nameunit(cap->cc_dev)
1191 , kalg
1192 , flags
1193 );
1194 gone_in_dev(cap->cc_dev, 14, "asymmetric crypto");
1195 err = 0;
1196 } else
1197 err = EINVAL;
1198
1199 CRYPTO_DRIVER_UNLOCK();
1200 return err;
1201 }
1202
1203 /*
1204 * Unregister all algorithms associated with a crypto driver.
1205 * If there are pending sessions using it, leave enough information
1206 * around so that subsequent calls using those sessions will
1207 * correctly detect the driver has been unregistered and reroute
1208 * requests.
1209 */
1210 int
crypto_unregister_all(uint32_t driverid)1211 crypto_unregister_all(uint32_t driverid)
1212 {
1213 struct cryptocap *cap;
1214
1215 CRYPTO_DRIVER_LOCK();
1216 cap = crypto_checkdriver(driverid);
1217 if (cap == NULL) {
1218 CRYPTO_DRIVER_UNLOCK();
1219 return (EINVAL);
1220 }
1221
1222 cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
1223 crypto_drivers[driverid] = NULL;
1224
1225 /*
1226 * XXX: This doesn't do anything to kick sessions that
1227 * have no pending operations.
1228 */
1229 while (cap->cc_sessions != 0 || cap->cc_koperations != 0)
1230 mtx_sleep(cap, &crypto_drivers_mtx, 0, "cryunreg", 0);
1231 CRYPTO_DRIVER_UNLOCK();
1232 cap_rele(cap);
1233
1234 return (0);
1235 }
1236
1237 /*
1238 * Clear blockage on a driver. The what parameter indicates whether
1239 * the driver is now ready for cryptop's and/or cryptokop's.
1240 */
1241 int
crypto_unblock(uint32_t driverid,int what)1242 crypto_unblock(uint32_t driverid, int what)
1243 {
1244 struct cryptocap *cap;
1245 int err;
1246
1247 CRYPTO_Q_LOCK();
1248 cap = crypto_checkdriver(driverid);
1249 if (cap != NULL) {
1250 if (what & CRYPTO_SYMQ)
1251 cap->cc_qblocked = 0;
1252 if (what & CRYPTO_ASYMQ)
1253 cap->cc_kqblocked = 0;
1254 if (crp_sleep)
1255 wakeup_one(&crp_q);
1256 err = 0;
1257 } else
1258 err = EINVAL;
1259 CRYPTO_Q_UNLOCK();
1260
1261 return err;
1262 }
1263
1264 size_t
crypto_buffer_len(struct crypto_buffer * cb)1265 crypto_buffer_len(struct crypto_buffer *cb)
1266 {
1267 switch (cb->cb_type) {
1268 case CRYPTO_BUF_CONTIG:
1269 return (cb->cb_buf_len);
1270 case CRYPTO_BUF_MBUF:
1271 if (cb->cb_mbuf->m_flags & M_PKTHDR)
1272 return (cb->cb_mbuf->m_pkthdr.len);
1273 return (m_length(cb->cb_mbuf, NULL));
1274 case CRYPTO_BUF_SINGLE_MBUF:
1275 return (cb->cb_mbuf->m_len);
1276 case CRYPTO_BUF_VMPAGE:
1277 return (cb->cb_vm_page_len);
1278 case CRYPTO_BUF_UIO:
1279 return (cb->cb_uio->uio_resid);
1280 default:
1281 return (0);
1282 }
1283 }
1284
1285 #ifdef INVARIANTS
1286 /* Various sanity checks on crypto requests. */
1287 static void
cb_sanity(struct crypto_buffer * cb,const char * name)1288 cb_sanity(struct crypto_buffer *cb, const char *name)
1289 {
1290 KASSERT(cb->cb_type > CRYPTO_BUF_NONE && cb->cb_type <= CRYPTO_BUF_LAST,
1291 ("incoming crp with invalid %s buffer type", name));
1292 switch (cb->cb_type) {
1293 case CRYPTO_BUF_CONTIG:
1294 KASSERT(cb->cb_buf_len >= 0,
1295 ("incoming crp with -ve %s buffer length", name));
1296 break;
1297 case CRYPTO_BUF_VMPAGE:
1298 KASSERT(CRYPTO_HAS_VMPAGE,
1299 ("incoming crp uses dmap on supported arch"));
1300 KASSERT(cb->cb_vm_page_len >= 0,
1301 ("incoming crp with -ve %s buffer length", name));
1302 KASSERT(cb->cb_vm_page_offset >= 0,
1303 ("incoming crp with -ve %s buffer offset", name));
1304 KASSERT(cb->cb_vm_page_offset < PAGE_SIZE,
1305 ("incoming crp with %s buffer offset greater than page size"
1306 , name));
1307 break;
1308 default:
1309 break;
1310 }
1311 }
1312
1313 static void
crp_sanity(struct cryptop * crp)1314 crp_sanity(struct cryptop *crp)
1315 {
1316 struct crypto_session_params *csp;
1317 struct crypto_buffer *out;
1318 size_t ilen, len, olen;
1319
1320 KASSERT(crp->crp_session != NULL, ("incoming crp without a session"));
1321 KASSERT(crp->crp_obuf.cb_type >= CRYPTO_BUF_NONE &&
1322 crp->crp_obuf.cb_type <= CRYPTO_BUF_LAST,
1323 ("incoming crp with invalid output buffer type"));
1324 KASSERT(crp->crp_etype == 0, ("incoming crp with error"));
1325 KASSERT(!(crp->crp_flags & CRYPTO_F_DONE),
1326 ("incoming crp already done"));
1327
1328 csp = &crp->crp_session->csp;
1329 cb_sanity(&crp->crp_buf, "input");
1330 ilen = crypto_buffer_len(&crp->crp_buf);
1331 olen = ilen;
1332 out = NULL;
1333 if (csp->csp_flags & CSP_F_SEPARATE_OUTPUT) {
1334 if (crp->crp_obuf.cb_type != CRYPTO_BUF_NONE) {
1335 cb_sanity(&crp->crp_obuf, "output");
1336 out = &crp->crp_obuf;
1337 olen = crypto_buffer_len(out);
1338 }
1339 } else
1340 KASSERT(crp->crp_obuf.cb_type == CRYPTO_BUF_NONE,
1341 ("incoming crp with separate output buffer "
1342 "but no session support"));
1343
1344 switch (csp->csp_mode) {
1345 case CSP_MODE_COMPRESS:
1346 KASSERT(crp->crp_op == CRYPTO_OP_COMPRESS ||
1347 crp->crp_op == CRYPTO_OP_DECOMPRESS,
1348 ("invalid compression op %x", crp->crp_op));
1349 break;
1350 case CSP_MODE_CIPHER:
1351 KASSERT(crp->crp_op == CRYPTO_OP_ENCRYPT ||
1352 crp->crp_op == CRYPTO_OP_DECRYPT,
1353 ("invalid cipher op %x", crp->crp_op));
1354 break;
1355 case CSP_MODE_DIGEST:
1356 KASSERT(crp->crp_op == CRYPTO_OP_COMPUTE_DIGEST ||
1357 crp->crp_op == CRYPTO_OP_VERIFY_DIGEST,
1358 ("invalid digest op %x", crp->crp_op));
1359 break;
1360 case CSP_MODE_AEAD:
1361 KASSERT(crp->crp_op ==
1362 (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) ||
1363 crp->crp_op ==
1364 (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST),
1365 ("invalid AEAD op %x", crp->crp_op));
1366 KASSERT(crp->crp_flags & CRYPTO_F_IV_SEPARATE,
1367 ("AEAD without a separate IV"));
1368 break;
1369 case CSP_MODE_ETA:
1370 KASSERT(crp->crp_op ==
1371 (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) ||
1372 crp->crp_op ==
1373 (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST),
1374 ("invalid ETA op %x", crp->crp_op));
1375 break;
1376 }
1377 if (csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) {
1378 if (crp->crp_aad == NULL) {
1379 KASSERT(crp->crp_aad_start == 0 ||
1380 crp->crp_aad_start < ilen,
1381 ("invalid AAD start"));
1382 KASSERT(crp->crp_aad_length != 0 ||
1383 crp->crp_aad_start == 0,
1384 ("AAD with zero length and non-zero start"));
1385 KASSERT(crp->crp_aad_length == 0 ||
1386 crp->crp_aad_start + crp->crp_aad_length <= ilen,
1387 ("AAD outside input length"));
1388 } else {
1389 KASSERT(csp->csp_flags & CSP_F_SEPARATE_AAD,
1390 ("session doesn't support separate AAD buffer"));
1391 KASSERT(crp->crp_aad_start == 0,
1392 ("separate AAD buffer with non-zero AAD start"));
1393 KASSERT(crp->crp_aad_length != 0,
1394 ("separate AAD buffer with zero length"));
1395 }
1396 } else {
1397 KASSERT(crp->crp_aad == NULL && crp->crp_aad_start == 0 &&
1398 crp->crp_aad_length == 0,
1399 ("AAD region in request not supporting AAD"));
1400 }
1401 if (csp->csp_ivlen == 0) {
1402 KASSERT((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0,
1403 ("IV_SEPARATE set when IV isn't used"));
1404 KASSERT(crp->crp_iv_start == 0,
1405 ("crp_iv_start set when IV isn't used"));
1406 } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) {
1407 KASSERT(crp->crp_iv_start == 0,
1408 ("IV_SEPARATE used with non-zero IV start"));
1409 } else {
1410 KASSERT(crp->crp_iv_start < ilen,
1411 ("invalid IV start"));
1412 KASSERT(crp->crp_iv_start + csp->csp_ivlen <= ilen,
1413 ("IV outside buffer length"));
1414 }
1415 /* XXX: payload_start of 0 should always be < ilen? */
1416 KASSERT(crp->crp_payload_start == 0 ||
1417 crp->crp_payload_start < ilen,
1418 ("invalid payload start"));
1419 KASSERT(crp->crp_payload_start + crp->crp_payload_length <=
1420 ilen, ("payload outside input buffer"));
1421 if (out == NULL) {
1422 KASSERT(crp->crp_payload_output_start == 0,
1423 ("payload output start non-zero without output buffer"));
1424 } else if (csp->csp_mode == CSP_MODE_DIGEST) {
1425 KASSERT(!(crp->crp_op & CRYPTO_OP_VERIFY_DIGEST),
1426 ("digest verify with separate output buffer"));
1427 KASSERT(crp->crp_payload_output_start == 0,
1428 ("digest operation with non-zero payload output start"));
1429 } else {
1430 KASSERT(crp->crp_payload_output_start == 0 ||
1431 crp->crp_payload_output_start < olen,
1432 ("invalid payload output start"));
1433 KASSERT(crp->crp_payload_output_start +
1434 crp->crp_payload_length <= olen,
1435 ("payload outside output buffer"));
1436 }
1437 if (csp->csp_mode == CSP_MODE_DIGEST ||
1438 csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) {
1439 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST)
1440 len = ilen;
1441 else
1442 len = olen;
1443 KASSERT(crp->crp_digest_start == 0 ||
1444 crp->crp_digest_start < len,
1445 ("invalid digest start"));
1446 /* XXX: For the mlen == 0 case this check isn't perfect. */
1447 KASSERT(crp->crp_digest_start + csp->csp_auth_mlen <= len,
1448 ("digest outside buffer"));
1449 } else {
1450 KASSERT(crp->crp_digest_start == 0,
1451 ("non-zero digest start for request without a digest"));
1452 }
1453 if (csp->csp_cipher_klen != 0)
1454 KASSERT(csp->csp_cipher_key != NULL ||
1455 crp->crp_cipher_key != NULL,
1456 ("cipher request without a key"));
1457 if (csp->csp_auth_klen != 0)
1458 KASSERT(csp->csp_auth_key != NULL || crp->crp_auth_key != NULL,
1459 ("auth request without a key"));
1460 KASSERT(crp->crp_callback != NULL, ("incoming crp without callback"));
1461 }
1462 #endif
1463
1464 /*
1465 * Add a crypto request to a queue, to be processed by the kernel thread.
1466 */
1467 int
crypto_dispatch(struct cryptop * crp)1468 crypto_dispatch(struct cryptop *crp)
1469 {
1470 struct cryptocap *cap;
1471 int result;
1472
1473 #ifdef INVARIANTS
1474 crp_sanity(crp);
1475 #endif
1476
1477 CRYPTOSTAT_INC(cs_ops);
1478
1479 crp->crp_retw_id = crp->crp_session->id % crypto_workers_num;
1480
1481 if (CRYPTOP_ASYNC(crp)) {
1482 if (crp->crp_flags & CRYPTO_F_ASYNC_KEEPORDER) {
1483 struct crypto_ret_worker *ret_worker;
1484
1485 ret_worker = CRYPTO_RETW(crp->crp_retw_id);
1486
1487 CRYPTO_RETW_LOCK(ret_worker);
1488 crp->crp_seq = ret_worker->reorder_ops++;
1489 CRYPTO_RETW_UNLOCK(ret_worker);
1490 }
1491
1492 TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp);
1493 taskqueue_enqueue(crypto_tq, &crp->crp_task);
1494 return (0);
1495 }
1496
1497 if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) {
1498 /*
1499 * Caller marked the request to be processed
1500 * immediately; dispatch it directly to the
1501 * driver unless the driver is currently blocked.
1502 */
1503 cap = crp->crp_session->cap;
1504 if (!cap->cc_qblocked) {
1505 result = crypto_invoke(cap, crp, 0);
1506 if (result != ERESTART)
1507 return (result);
1508 /*
1509 * The driver ran out of resources, put the request on
1510 * the queue.
1511 */
1512 }
1513 }
1514 crypto_batch_enqueue(crp);
1515 return 0;
1516 }
1517
1518 void
crypto_batch_enqueue(struct cryptop * crp)1519 crypto_batch_enqueue(struct cryptop *crp)
1520 {
1521
1522 CRYPTO_Q_LOCK();
1523 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
1524 if (crp_sleep)
1525 wakeup_one(&crp_q);
1526 CRYPTO_Q_UNLOCK();
1527 }
1528
1529 /*
1530 * Add an asymetric crypto request to a queue,
1531 * to be processed by the kernel thread.
1532 */
1533 int
crypto_kdispatch(struct cryptkop * krp)1534 crypto_kdispatch(struct cryptkop *krp)
1535 {
1536 int error;
1537
1538 CRYPTOSTAT_INC(cs_kops);
1539
1540 krp->krp_cap = NULL;
1541 error = crypto_kinvoke(krp);
1542 if (error == ERESTART) {
1543 CRYPTO_Q_LOCK();
1544 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
1545 if (crp_sleep)
1546 wakeup_one(&crp_q);
1547 CRYPTO_Q_UNLOCK();
1548 error = 0;
1549 }
1550 return error;
1551 }
1552
1553 /*
1554 * Verify a driver is suitable for the specified operation.
1555 */
1556 static __inline int
kdriver_suitable(const struct cryptocap * cap,const struct cryptkop * krp)1557 kdriver_suitable(const struct cryptocap *cap, const struct cryptkop *krp)
1558 {
1559 return (cap->cc_kalg[krp->krp_op] & CRYPTO_ALG_FLAG_SUPPORTED) != 0;
1560 }
1561
1562 /*
1563 * Select a driver for an asym operation. The driver must
1564 * support the necessary algorithm. The caller can constrain
1565 * which device is selected with the flags parameter. The
1566 * algorithm we use here is pretty stupid; just use the first
1567 * driver that supports the algorithms we need. If there are
1568 * multiple suitable drivers we choose the driver with the
1569 * fewest active operations. We prefer hardware-backed
1570 * drivers to software ones when either may be used.
1571 */
1572 static struct cryptocap *
crypto_select_kdriver(const struct cryptkop * krp,int flags)1573 crypto_select_kdriver(const struct cryptkop *krp, int flags)
1574 {
1575 struct cryptocap *cap, *best;
1576 int match, hid;
1577
1578 CRYPTO_DRIVER_ASSERT();
1579
1580 /*
1581 * Look first for hardware crypto devices if permitted.
1582 */
1583 if (flags & CRYPTOCAP_F_HARDWARE)
1584 match = CRYPTOCAP_F_HARDWARE;
1585 else
1586 match = CRYPTOCAP_F_SOFTWARE;
1587 best = NULL;
1588 again:
1589 for (hid = 0; hid < crypto_drivers_size; hid++) {
1590 /*
1591 * If there is no driver for this slot, or the driver
1592 * is not appropriate (hardware or software based on
1593 * match), then skip.
1594 */
1595 cap = crypto_drivers[hid];
1596 if (cap == NULL ||
1597 (cap->cc_flags & match) == 0)
1598 continue;
1599
1600 /* verify all the algorithms are supported. */
1601 if (kdriver_suitable(cap, krp)) {
1602 if (best == NULL ||
1603 cap->cc_koperations < best->cc_koperations)
1604 best = cap;
1605 }
1606 }
1607 if (best != NULL)
1608 return best;
1609 if (match == CRYPTOCAP_F_HARDWARE && (flags & CRYPTOCAP_F_SOFTWARE)) {
1610 /* sort of an Algol 68-style for loop */
1611 match = CRYPTOCAP_F_SOFTWARE;
1612 goto again;
1613 }
1614 return best;
1615 }
1616
1617 /*
1618 * Choose a driver for an asymmetric crypto request.
1619 */
1620 static struct cryptocap *
crypto_lookup_kdriver(struct cryptkop * krp)1621 crypto_lookup_kdriver(struct cryptkop *krp)
1622 {
1623 struct cryptocap *cap;
1624 uint32_t crid;
1625
1626 /* If this request is requeued, it might already have a driver. */
1627 cap = krp->krp_cap;
1628 if (cap != NULL)
1629 return (cap);
1630
1631 /* Use krp_crid to choose a driver. */
1632 crid = krp->krp_crid;
1633 if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
1634 cap = crypto_checkdriver(crid);
1635 if (cap != NULL) {
1636 /*
1637 * Driver present, it must support the
1638 * necessary algorithm and, if s/w drivers are
1639 * excluded, it must be registered as
1640 * hardware-backed.
1641 */
1642 if (!kdriver_suitable(cap, krp) ||
1643 (!crypto_devallowsoft &&
1644 (cap->cc_flags & CRYPTOCAP_F_HARDWARE) == 0))
1645 cap = NULL;
1646 }
1647 } else {
1648 /*
1649 * No requested driver; select based on crid flags.
1650 */
1651 if (!crypto_devallowsoft) /* NB: disallow s/w drivers */
1652 crid &= ~CRYPTOCAP_F_SOFTWARE;
1653 cap = crypto_select_kdriver(krp, crid);
1654 }
1655
1656 if (cap != NULL) {
1657 krp->krp_cap = cap_ref(cap);
1658 krp->krp_hid = cap->cc_hid;
1659 }
1660 return (cap);
1661 }
1662
1663 /*
1664 * Dispatch an asymmetric crypto request.
1665 */
1666 static int
crypto_kinvoke(struct cryptkop * krp)1667 crypto_kinvoke(struct cryptkop *krp)
1668 {
1669 struct cryptocap *cap = NULL;
1670 int error;
1671
1672 KASSERT(krp != NULL, ("%s: krp == NULL", __func__));
1673 KASSERT(krp->krp_callback != NULL,
1674 ("%s: krp->crp_callback == NULL", __func__));
1675
1676 CRYPTO_DRIVER_LOCK();
1677 cap = crypto_lookup_kdriver(krp);
1678 if (cap == NULL) {
1679 CRYPTO_DRIVER_UNLOCK();
1680 krp->krp_status = ENODEV;
1681 crypto_kdone(krp);
1682 return (0);
1683 }
1684
1685 /*
1686 * If the device is blocked, return ERESTART to requeue it.
1687 */
1688 if (cap->cc_kqblocked) {
1689 /*
1690 * XXX: Previously this set krp_status to ERESTART and
1691 * invoked crypto_kdone but the caller would still
1692 * requeue it.
1693 */
1694 CRYPTO_DRIVER_UNLOCK();
1695 return (ERESTART);
1696 }
1697
1698 cap->cc_koperations++;
1699 CRYPTO_DRIVER_UNLOCK();
1700 error = CRYPTODEV_KPROCESS(cap->cc_dev, krp, 0);
1701 if (error == ERESTART) {
1702 CRYPTO_DRIVER_LOCK();
1703 cap->cc_koperations--;
1704 CRYPTO_DRIVER_UNLOCK();
1705 return (error);
1706 }
1707
1708 KASSERT(error == 0, ("error %d returned from crypto_kprocess", error));
1709 return (0);
1710 }
1711
1712 static void
crypto_task_invoke(void * ctx,int pending)1713 crypto_task_invoke(void *ctx, int pending)
1714 {
1715 struct cryptocap *cap;
1716 struct cryptop *crp;
1717 int result;
1718
1719 crp = (struct cryptop *)ctx;
1720 cap = crp->crp_session->cap;
1721 result = crypto_invoke(cap, crp, 0);
1722 if (result == ERESTART)
1723 crypto_batch_enqueue(crp);
1724 }
1725
1726 /*
1727 * Dispatch a crypto request to the appropriate crypto devices.
1728 */
1729 static int
crypto_invoke(struct cryptocap * cap,struct cryptop * crp,int hint)1730 crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint)
1731 {
1732 int error;
1733
1734 KASSERT(crp != NULL, ("%s: crp == NULL", __func__));
1735 KASSERT(crp->crp_callback != NULL,
1736 ("%s: crp->crp_callback == NULL", __func__));
1737 KASSERT(crp->crp_session != NULL,
1738 ("%s: crp->crp_session == NULL", __func__));
1739
1740 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
1741 struct crypto_session_params csp;
1742 crypto_session_t nses;
1743
1744 /*
1745 * Driver has unregistered; migrate the session and return
1746 * an error to the caller so they'll resubmit the op.
1747 *
1748 * XXX: What if there are more already queued requests for this
1749 * session?
1750 *
1751 * XXX: Real solution is to make sessions refcounted
1752 * and force callers to hold a reference when
1753 * assigning to crp_session. Could maybe change
1754 * crypto_getreq to accept a session pointer to make
1755 * that work. Alternatively, we could abandon the
1756 * notion of rewriting crp_session in requests forcing
1757 * the caller to deal with allocating a new session.
1758 * Perhaps provide a method to allow a crp's session to
1759 * be swapped that callers could use.
1760 */
1761 csp = crp->crp_session->csp;
1762 crypto_freesession(crp->crp_session);
1763
1764 /*
1765 * XXX: Key pointers may no longer be valid. If we
1766 * really want to support this we need to define the
1767 * KPI such that 'csp' is required to be valid for the
1768 * duration of a session by the caller perhaps.
1769 *
1770 * XXX: If the keys have been changed this will reuse
1771 * the old keys. This probably suggests making
1772 * rekeying more explicit and updating the key
1773 * pointers in 'csp' when the keys change.
1774 */
1775 if (crypto_newsession(&nses, &csp,
1776 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE) == 0)
1777 crp->crp_session = nses;
1778
1779 crp->crp_etype = EAGAIN;
1780 crypto_done(crp);
1781 error = 0;
1782 } else {
1783 /*
1784 * Invoke the driver to process the request. Errors are
1785 * signaled by setting crp_etype before invoking the completion
1786 * callback.
1787 */
1788 error = CRYPTODEV_PROCESS(cap->cc_dev, crp, hint);
1789 KASSERT(error == 0 || error == ERESTART,
1790 ("%s: invalid error %d from CRYPTODEV_PROCESS",
1791 __func__, error));
1792 }
1793 return (error);
1794 }
1795
1796 void
crypto_destroyreq(struct cryptop * crp)1797 crypto_destroyreq(struct cryptop *crp)
1798 {
1799 #ifdef DIAGNOSTIC
1800 {
1801 struct cryptop *crp2;
1802 struct crypto_ret_worker *ret_worker;
1803
1804 CRYPTO_Q_LOCK();
1805 TAILQ_FOREACH(crp2, &crp_q, crp_next) {
1806 KASSERT(crp2 != crp,
1807 ("Freeing cryptop from the crypto queue (%p).",
1808 crp));
1809 }
1810 CRYPTO_Q_UNLOCK();
1811
1812 FOREACH_CRYPTO_RETW(ret_worker) {
1813 CRYPTO_RETW_LOCK(ret_worker);
1814 TAILQ_FOREACH(crp2, &ret_worker->crp_ret_q, crp_next) {
1815 KASSERT(crp2 != crp,
1816 ("Freeing cryptop from the return queue (%p).",
1817 crp));
1818 }
1819 CRYPTO_RETW_UNLOCK(ret_worker);
1820 }
1821 }
1822 #endif
1823 }
1824
1825 void
crypto_freereq(struct cryptop * crp)1826 crypto_freereq(struct cryptop *crp)
1827 {
1828 if (crp == NULL)
1829 return;
1830
1831 crypto_destroyreq(crp);
1832 uma_zfree(cryptop_zone, crp);
1833 }
1834
1835 static void
_crypto_initreq(struct cryptop * crp,crypto_session_t cses)1836 _crypto_initreq(struct cryptop *crp, crypto_session_t cses)
1837 {
1838 crp->crp_session = cses;
1839 }
1840
1841 void
crypto_initreq(struct cryptop * crp,crypto_session_t cses)1842 crypto_initreq(struct cryptop *crp, crypto_session_t cses)
1843 {
1844 memset(crp, 0, sizeof(*crp));
1845 _crypto_initreq(crp, cses);
1846 }
1847
1848 struct cryptop *
crypto_getreq(crypto_session_t cses,int how)1849 crypto_getreq(crypto_session_t cses, int how)
1850 {
1851 struct cryptop *crp;
1852
1853 MPASS(how == M_WAITOK || how == M_NOWAIT);
1854 crp = uma_zalloc(cryptop_zone, how | M_ZERO);
1855 if (crp != NULL)
1856 _crypto_initreq(crp, cses);
1857 return (crp);
1858 }
1859
1860 /*
1861 * Clone a crypto request, but associate it with the specified session
1862 * rather than inheriting the session from the original request. The
1863 * fields describing the request buffers are copied, but not the
1864 * opaque field or callback function.
1865 */
1866 struct cryptop *
crypto_clonereq(struct cryptop * crp,crypto_session_t cses,int how)1867 crypto_clonereq(struct cryptop *crp, crypto_session_t cses, int how)
1868 {
1869 struct cryptop *new;
1870
1871 MPASS((crp->crp_flags & CRYPTO_F_DONE) == 0);
1872 new = crypto_getreq(cses, how);
1873 if (new == NULL)
1874 return (NULL);
1875
1876 memcpy(&new->crp_startcopy, &crp->crp_startcopy,
1877 __rangeof(struct cryptop, crp_startcopy, crp_endcopy));
1878 return (new);
1879 }
1880
1881 /*
1882 * Invoke the callback on behalf of the driver.
1883 */
1884 void
crypto_done(struct cryptop * crp)1885 crypto_done(struct cryptop *crp)
1886 {
1887 KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0,
1888 ("crypto_done: op already done, flags 0x%x", crp->crp_flags));
1889 crp->crp_flags |= CRYPTO_F_DONE;
1890 if (crp->crp_etype != 0)
1891 CRYPTOSTAT_INC(cs_errs);
1892
1893 /*
1894 * CBIMM means unconditionally do the callback immediately;
1895 * CBIFSYNC means do the callback immediately only if the
1896 * operation was done synchronously. Both are used to avoid
1897 * doing extraneous context switches; the latter is mostly
1898 * used with the software crypto driver.
1899 */
1900 if (!CRYPTOP_ASYNC_KEEPORDER(crp) &&
1901 ((crp->crp_flags & CRYPTO_F_CBIMM) ||
1902 ((crp->crp_flags & CRYPTO_F_CBIFSYNC) &&
1903 (crypto_ses2caps(crp->crp_session) & CRYPTOCAP_F_SYNC)))) {
1904 /*
1905 * Do the callback directly. This is ok when the
1906 * callback routine does very little (e.g. the
1907 * /dev/crypto callback method just does a wakeup).
1908 */
1909 crp->crp_callback(crp);
1910 } else {
1911 struct crypto_ret_worker *ret_worker;
1912 bool wake;
1913
1914 ret_worker = CRYPTO_RETW(crp->crp_retw_id);
1915 wake = false;
1916
1917 /*
1918 * Normal case; queue the callback for the thread.
1919 */
1920 CRYPTO_RETW_LOCK(ret_worker);
1921 if (CRYPTOP_ASYNC_KEEPORDER(crp)) {
1922 struct cryptop *tmp;
1923
1924 TAILQ_FOREACH_REVERSE(tmp, &ret_worker->crp_ordered_ret_q,
1925 cryptop_q, crp_next) {
1926 if (CRYPTO_SEQ_GT(crp->crp_seq, tmp->crp_seq)) {
1927 TAILQ_INSERT_AFTER(&ret_worker->crp_ordered_ret_q,
1928 tmp, crp, crp_next);
1929 break;
1930 }
1931 }
1932 if (tmp == NULL) {
1933 TAILQ_INSERT_HEAD(&ret_worker->crp_ordered_ret_q,
1934 crp, crp_next);
1935 }
1936
1937 if (crp->crp_seq == ret_worker->reorder_cur_seq)
1938 wake = true;
1939 }
1940 else {
1941 if (CRYPTO_RETW_EMPTY(ret_worker))
1942 wake = true;
1943
1944 TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp, crp_next);
1945 }
1946
1947 if (wake)
1948 wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */
1949 CRYPTO_RETW_UNLOCK(ret_worker);
1950 }
1951 }
1952
1953 /*
1954 * Invoke the callback on behalf of the driver.
1955 */
1956 void
crypto_kdone(struct cryptkop * krp)1957 crypto_kdone(struct cryptkop *krp)
1958 {
1959 struct crypto_ret_worker *ret_worker;
1960 struct cryptocap *cap;
1961
1962 if (krp->krp_status != 0)
1963 CRYPTOSTAT_INC(cs_kerrs);
1964 cap = krp->krp_cap;
1965 if (cap != NULL) {
1966 CRYPTO_DRIVER_LOCK();
1967 KASSERT(cap->cc_koperations > 0, ("cc_koperations == 0"));
1968 cap->cc_koperations--;
1969 if (cap->cc_koperations == 0 &&
1970 cap->cc_flags & CRYPTOCAP_F_CLEANUP)
1971 wakeup(cap);
1972 CRYPTO_DRIVER_UNLOCK();
1973 krp->krp_cap = NULL;
1974 cap_rele(cap);
1975 }
1976
1977 ret_worker = CRYPTO_RETW(0);
1978
1979 CRYPTO_RETW_LOCK(ret_worker);
1980 if (CRYPTO_RETW_EMPTY(ret_worker))
1981 wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */
1982 TAILQ_INSERT_TAIL(&ret_worker->crp_ret_kq, krp, krp_next);
1983 CRYPTO_RETW_UNLOCK(ret_worker);
1984 }
1985
1986 int
crypto_getfeat(int * featp)1987 crypto_getfeat(int *featp)
1988 {
1989 int hid, kalg, feat = 0;
1990
1991 CRYPTO_DRIVER_LOCK();
1992 for (hid = 0; hid < crypto_drivers_size; hid++) {
1993 const struct cryptocap *cap = crypto_drivers[hid];
1994
1995 if (cap == NULL ||
1996 ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) &&
1997 !crypto_devallowsoft)) {
1998 continue;
1999 }
2000 for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++)
2001 if (cap->cc_kalg[kalg] & CRYPTO_ALG_FLAG_SUPPORTED)
2002 feat |= 1 << kalg;
2003 }
2004 CRYPTO_DRIVER_UNLOCK();
2005 *featp = feat;
2006 return (0);
2007 }
2008
2009 /*
2010 * Terminate a thread at module unload. The process that
2011 * initiated this is waiting for us to signal that we're gone;
2012 * wake it up and exit. We use the driver table lock to insure
2013 * we don't do the wakeup before they're waiting. There is no
2014 * race here because the waiter sleeps on the proc lock for the
2015 * thread so it gets notified at the right time because of an
2016 * extra wakeup that's done in exit1().
2017 */
2018 static void
crypto_finis(void * chan)2019 crypto_finis(void *chan)
2020 {
2021 CRYPTO_DRIVER_LOCK();
2022 wakeup_one(chan);
2023 CRYPTO_DRIVER_UNLOCK();
2024 kthread_exit();
2025 }
2026
2027 /*
2028 * Crypto thread, dispatches crypto requests.
2029 */
2030 static void
crypto_dispatch_thread(void * arg __unused)2031 crypto_dispatch_thread(void *arg __unused)
2032 {
2033 struct cryptop *crp, *submit;
2034 struct cryptkop *krp;
2035 struct cryptocap *cap;
2036 int result, hint;
2037
2038 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
2039 fpu_kern_thread(FPU_KERN_NORMAL);
2040 #endif
2041
2042 CRYPTO_Q_LOCK();
2043 for (;;) {
2044 /*
2045 * Find the first element in the queue that can be
2046 * processed and look-ahead to see if multiple ops
2047 * are ready for the same driver.
2048 */
2049 submit = NULL;
2050 hint = 0;
2051 TAILQ_FOREACH(crp, &crp_q, crp_next) {
2052 cap = crp->crp_session->cap;
2053 /*
2054 * Driver cannot disappeared when there is an active
2055 * session.
2056 */
2057 KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
2058 __func__, __LINE__));
2059 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
2060 /* Op needs to be migrated, process it. */
2061 if (submit == NULL)
2062 submit = crp;
2063 break;
2064 }
2065 if (!cap->cc_qblocked) {
2066 if (submit != NULL) {
2067 /*
2068 * We stop on finding another op,
2069 * regardless whether its for the same
2070 * driver or not. We could keep
2071 * searching the queue but it might be
2072 * better to just use a per-driver
2073 * queue instead.
2074 */
2075 if (submit->crp_session->cap == cap)
2076 hint = CRYPTO_HINT_MORE;
2077 break;
2078 } else {
2079 submit = crp;
2080 if ((submit->crp_flags & CRYPTO_F_BATCH) == 0)
2081 break;
2082 /* keep scanning for more are q'd */
2083 }
2084 }
2085 }
2086 if (submit != NULL) {
2087 TAILQ_REMOVE(&crp_q, submit, crp_next);
2088 cap = submit->crp_session->cap;
2089 KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
2090 __func__, __LINE__));
2091 CRYPTO_Q_UNLOCK();
2092 result = crypto_invoke(cap, submit, hint);
2093 CRYPTO_Q_LOCK();
2094 if (result == ERESTART) {
2095 /*
2096 * The driver ran out of resources, mark the
2097 * driver ``blocked'' for cryptop's and put
2098 * the request back in the queue. It would
2099 * best to put the request back where we got
2100 * it but that's hard so for now we put it
2101 * at the front. This should be ok; putting
2102 * it at the end does not work.
2103 */
2104 cap->cc_qblocked = 1;
2105 TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
2106 CRYPTOSTAT_INC(cs_blocks);
2107 }
2108 }
2109
2110 /* As above, but for key ops */
2111 TAILQ_FOREACH(krp, &crp_kq, krp_next) {
2112 cap = krp->krp_cap;
2113 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
2114 /*
2115 * Operation needs to be migrated,
2116 * clear krp_cap so a new driver is
2117 * selected.
2118 */
2119 krp->krp_cap = NULL;
2120 cap_rele(cap);
2121 break;
2122 }
2123 if (!cap->cc_kqblocked)
2124 break;
2125 }
2126 if (krp != NULL) {
2127 TAILQ_REMOVE(&crp_kq, krp, krp_next);
2128 CRYPTO_Q_UNLOCK();
2129 result = crypto_kinvoke(krp);
2130 CRYPTO_Q_LOCK();
2131 if (result == ERESTART) {
2132 /*
2133 * The driver ran out of resources, mark the
2134 * driver ``blocked'' for cryptkop's and put
2135 * the request back in the queue. It would
2136 * best to put the request back where we got
2137 * it but that's hard so for now we put it
2138 * at the front. This should be ok; putting
2139 * it at the end does not work.
2140 */
2141 krp->krp_cap->cc_kqblocked = 1;
2142 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
2143 CRYPTOSTAT_INC(cs_kblocks);
2144 }
2145 }
2146
2147 if (submit == NULL && krp == NULL) {
2148 /*
2149 * Nothing more to be processed. Sleep until we're
2150 * woken because there are more ops to process.
2151 * This happens either by submission or by a driver
2152 * becoming unblocked and notifying us through
2153 * crypto_unblock. Note that when we wakeup we
2154 * start processing each queue again from the
2155 * front. It's not clear that it's important to
2156 * preserve this ordering since ops may finish
2157 * out of order if dispatched to different devices
2158 * and some become blocked while others do not.
2159 */
2160 crp_sleep = 1;
2161 msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0);
2162 crp_sleep = 0;
2163 if (cryptotd == NULL)
2164 break;
2165 CRYPTOSTAT_INC(cs_intrs);
2166 }
2167 }
2168 CRYPTO_Q_UNLOCK();
2169
2170 crypto_finis(&crp_q);
2171 }
2172
2173 /*
2174 * Crypto returns thread, does callbacks for processed crypto requests.
2175 * Callbacks are done here, rather than in the crypto drivers, because
2176 * callbacks typically are expensive and would slow interrupt handling.
2177 */
2178 static void
crypto_ret_thread(void * arg)2179 crypto_ret_thread(void *arg)
2180 {
2181 struct crypto_ret_worker *ret_worker = arg;
2182 struct cryptop *crpt;
2183 struct cryptkop *krpt;
2184
2185 CRYPTO_RETW_LOCK(ret_worker);
2186 for (;;) {
2187 /* Harvest return q's for completed ops */
2188 crpt = TAILQ_FIRST(&ret_worker->crp_ordered_ret_q);
2189 if (crpt != NULL) {
2190 if (crpt->crp_seq == ret_worker->reorder_cur_seq) {
2191 TAILQ_REMOVE(&ret_worker->crp_ordered_ret_q, crpt, crp_next);
2192 ret_worker->reorder_cur_seq++;
2193 } else {
2194 crpt = NULL;
2195 }
2196 }
2197
2198 if (crpt == NULL) {
2199 crpt = TAILQ_FIRST(&ret_worker->crp_ret_q);
2200 if (crpt != NULL)
2201 TAILQ_REMOVE(&ret_worker->crp_ret_q, crpt, crp_next);
2202 }
2203
2204 krpt = TAILQ_FIRST(&ret_worker->crp_ret_kq);
2205 if (krpt != NULL)
2206 TAILQ_REMOVE(&ret_worker->crp_ret_kq, krpt, krp_next);
2207
2208 if (crpt != NULL || krpt != NULL) {
2209 CRYPTO_RETW_UNLOCK(ret_worker);
2210 /*
2211 * Run callbacks unlocked.
2212 */
2213 if (crpt != NULL)
2214 crpt->crp_callback(crpt);
2215 if (krpt != NULL)
2216 krpt->krp_callback(krpt);
2217 CRYPTO_RETW_LOCK(ret_worker);
2218 } else {
2219 /*
2220 * Nothing more to be processed. Sleep until we're
2221 * woken because there are more returns to process.
2222 */
2223 msleep(&ret_worker->crp_ret_q, &ret_worker->crypto_ret_mtx, PWAIT,
2224 "crypto_ret_wait", 0);
2225 if (ret_worker->td == NULL)
2226 break;
2227 CRYPTOSTAT_INC(cs_rets);
2228 }
2229 }
2230 CRYPTO_RETW_UNLOCK(ret_worker);
2231
2232 crypto_finis(&ret_worker->crp_ret_q);
2233 }
2234
2235 #ifdef DDB
2236 static void
db_show_drivers(void)2237 db_show_drivers(void)
2238 {
2239 int hid;
2240
2241 db_printf("%12s %4s %4s %8s %2s %2s\n"
2242 , "Device"
2243 , "Ses"
2244 , "Kops"
2245 , "Flags"
2246 , "QB"
2247 , "KB"
2248 );
2249 for (hid = 0; hid < crypto_drivers_size; hid++) {
2250 const struct cryptocap *cap = crypto_drivers[hid];
2251 if (cap == NULL)
2252 continue;
2253 db_printf("%-12s %4u %4u %08x %2u %2u\n"
2254 , device_get_nameunit(cap->cc_dev)
2255 , cap->cc_sessions
2256 , cap->cc_koperations
2257 , cap->cc_flags
2258 , cap->cc_qblocked
2259 , cap->cc_kqblocked
2260 );
2261 }
2262 }
2263
DB_SHOW_COMMAND(crypto,db_show_crypto)2264 DB_SHOW_COMMAND(crypto, db_show_crypto)
2265 {
2266 struct cryptop *crp;
2267 struct crypto_ret_worker *ret_worker;
2268
2269 db_show_drivers();
2270 db_printf("\n");
2271
2272 db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n",
2273 "HID", "Caps", "Ilen", "Olen", "Etype", "Flags",
2274 "Device", "Callback");
2275 TAILQ_FOREACH(crp, &crp_q, crp_next) {
2276 db_printf("%4u %08x %4u %4u %04x %8p %8p\n"
2277 , crp->crp_session->cap->cc_hid
2278 , (int) crypto_ses2caps(crp->crp_session)
2279 , crp->crp_olen
2280 , crp->crp_etype
2281 , crp->crp_flags
2282 , device_get_nameunit(crp->crp_session->cap->cc_dev)
2283 , crp->crp_callback
2284 );
2285 }
2286 FOREACH_CRYPTO_RETW(ret_worker) {
2287 db_printf("\n%8s %4s %4s %4s %8s\n",
2288 "ret_worker", "HID", "Etype", "Flags", "Callback");
2289 if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
2290 TAILQ_FOREACH(crp, &ret_worker->crp_ret_q, crp_next) {
2291 db_printf("%8td %4u %4u %04x %8p\n"
2292 , CRYPTO_RETW_ID(ret_worker)
2293 , crp->crp_session->cap->cc_hid
2294 , crp->crp_etype
2295 , crp->crp_flags
2296 , crp->crp_callback
2297 );
2298 }
2299 }
2300 }
2301 }
2302
DB_SHOW_COMMAND(kcrypto,db_show_kcrypto)2303 DB_SHOW_COMMAND(kcrypto, db_show_kcrypto)
2304 {
2305 struct cryptkop *krp;
2306 struct crypto_ret_worker *ret_worker;
2307
2308 db_show_drivers();
2309 db_printf("\n");
2310
2311 db_printf("%4s %5s %4s %4s %8s %4s %8s\n",
2312 "Op", "Status", "#IP", "#OP", "CRID", "HID", "Callback");
2313 TAILQ_FOREACH(krp, &crp_kq, krp_next) {
2314 db_printf("%4u %5u %4u %4u %08x %4u %8p\n"
2315 , krp->krp_op
2316 , krp->krp_status
2317 , krp->krp_iparams, krp->krp_oparams
2318 , krp->krp_crid, krp->krp_hid
2319 , krp->krp_callback
2320 );
2321 }
2322
2323 ret_worker = CRYPTO_RETW(0);
2324 if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
2325 db_printf("%4s %5s %8s %4s %8s\n",
2326 "Op", "Status", "CRID", "HID", "Callback");
2327 TAILQ_FOREACH(krp, &ret_worker->crp_ret_kq, krp_next) {
2328 db_printf("%4u %5u %08x %4u %8p\n"
2329 , krp->krp_op
2330 , krp->krp_status
2331 , krp->krp_crid, krp->krp_hid
2332 , krp->krp_callback
2333 );
2334 }
2335 }
2336 }
2337 #endif
2338
2339 int crypto_modevent(module_t mod, int type, void *unused);
2340
2341 /*
2342 * Initialization code, both for static and dynamic loading.
2343 * Note this is not invoked with the usual MODULE_DECLARE
2344 * mechanism but instead is listed as a dependency by the
2345 * cryptosoft driver. This guarantees proper ordering of
2346 * calls on module load/unload.
2347 */
2348 int
crypto_modevent(module_t mod,int type,void * unused)2349 crypto_modevent(module_t mod, int type, void *unused)
2350 {
2351 int error = EINVAL;
2352
2353 switch (type) {
2354 case MOD_LOAD:
2355 error = crypto_init();
2356 if (error == 0 && bootverbose)
2357 printf("crypto: <crypto core>\n");
2358 break;
2359 case MOD_UNLOAD:
2360 /*XXX disallow if active sessions */
2361 error = 0;
2362 crypto_destroy();
2363 return 0;
2364 }
2365 return error;
2366 }
2367 MODULE_VERSION(crypto, 1);
2368 MODULE_DEPEND(crypto, zlib, 1, 1, 1);
2369