1 /*-
2 * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3 * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
4 * Copyright (c) 2014-2021 The FreeBSD Foundation
5 * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
6 * All rights reserved.
7 *
8 * Portions of this software were developed by John-Mark Gurney
9 * under sponsorship of the FreeBSD Foundation and
10 * Rubicon Communications, LLC (Netgate).
11 *
12 * Portions of this software were developed by Ararat River
13 * Consulting, LLC under sponsorship of the FreeBSD Foundation.
14 *
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
17 * are met:
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 * notice, this list of conditions and the following disclaimer in the
22 * documentation and/or other materials provided with the distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 */
36
37 #include <sys/cdefs.h>
38 #include <sys/param.h>
39 #include <sys/bus.h>
40 #include <sys/kernel.h>
41 #include <sys/kobj.h>
42 #include <sys/libkern.h>
43 #include <sys/lock.h>
44 #include <sys/malloc.h>
45 #include <sys/mbuf.h>
46 #include <sys/module.h>
47 #include <sys/mutex.h>
48 #include <sys/smp.h>
49 #include <sys/systm.h>
50 #include <sys/uio.h>
51
52 #include <crypto/aesni/aesni.h>
53 #include <crypto/aesni/sha_sse.h>
54 #include <crypto/sha1.h>
55 #include <crypto/sha2/sha224.h>
56 #include <crypto/sha2/sha256.h>
57
58 #include <opencrypto/cryptodev.h>
59 #include <opencrypto/gmac.h>
60 #include <cryptodev_if.h>
61
62 #include <machine/md_var.h>
63 #include <machine/specialreg.h>
64 #include <machine/fpu.h>
65
66 static struct mtx_padalign *ctx_mtx;
67 static struct fpu_kern_ctx **ctx_fpu;
68
69 struct aesni_softc {
70 int32_t cid;
71 bool has_aes;
72 bool has_sha;
73 };
74
75 #define ACQUIRE_CTX(i, ctx) \
76 do { \
77 (i) = PCPU_GET(cpuid); \
78 mtx_lock(&ctx_mtx[(i)]); \
79 (ctx) = ctx_fpu[(i)]; \
80 } while (0)
81 #define RELEASE_CTX(i, ctx) \
82 do { \
83 mtx_unlock(&ctx_mtx[(i)]); \
84 (i) = -1; \
85 (ctx) = NULL; \
86 } while (0)
87
88 static int aesni_cipher_setup(struct aesni_session *ses,
89 const struct crypto_session_params *csp);
90 static int aesni_cipher_process(struct aesni_session *ses, struct cryptop *crp);
91 static int aesni_cipher_crypt(struct aesni_session *ses, struct cryptop *crp,
92 const struct crypto_session_params *csp);
93 static int aesni_cipher_mac(struct aesni_session *ses, struct cryptop *crp,
94 const struct crypto_session_params *csp);
95
96 MALLOC_DEFINE(M_AESNI, "aesni_data", "AESNI Data");
97
98 static void
aesni_identify(driver_t * drv,device_t parent)99 aesni_identify(driver_t *drv, device_t parent)
100 {
101
102 /* NB: order 10 is so we get attached after h/w devices */
103 if (device_find_child(parent, "aesni", -1) == NULL &&
104 BUS_ADD_CHILD(parent, 10, "aesni", -1) == 0)
105 panic("aesni: could not attach");
106 }
107
108 static void
detect_cpu_features(bool * has_aes,bool * has_sha)109 detect_cpu_features(bool *has_aes, bool *has_sha)
110 {
111
112 *has_aes = ((cpu_feature2 & CPUID2_AESNI) != 0 &&
113 (cpu_feature2 & CPUID2_SSE41) != 0);
114 *has_sha = ((cpu_stdext_feature & CPUID_STDEXT_SHA) != 0 &&
115 (cpu_feature2 & CPUID2_SSSE3) != 0);
116 }
117
118 static int
aesni_probe(device_t dev)119 aesni_probe(device_t dev)
120 {
121 bool has_aes, has_sha;
122
123 detect_cpu_features(&has_aes, &has_sha);
124 if (!has_aes && !has_sha) {
125 device_printf(dev, "No AES or SHA support.\n");
126 return (EINVAL);
127 } else if (has_aes && has_sha)
128 device_set_desc(dev,
129 "AES-CBC,AES-CCM,AES-GCM,AES-ICM,AES-XTS,SHA1,SHA256");
130 else if (has_aes)
131 device_set_desc(dev,
132 "AES-CBC,AES-CCM,AES-GCM,AES-ICM,AES-XTS");
133 else
134 device_set_desc(dev, "SHA1,SHA256");
135
136 return (0);
137 }
138
139 static void
aesni_cleanctx(void)140 aesni_cleanctx(void)
141 {
142 int i;
143
144 /* XXX - no way to return driverid */
145 CPU_FOREACH(i) {
146 if (ctx_fpu[i] != NULL) {
147 mtx_destroy(&ctx_mtx[i]);
148 fpu_kern_free_ctx(ctx_fpu[i]);
149 }
150 ctx_fpu[i] = NULL;
151 }
152 free(ctx_mtx, M_AESNI);
153 ctx_mtx = NULL;
154 free(ctx_fpu, M_AESNI);
155 ctx_fpu = NULL;
156 }
157
158 static int
aesni_attach(device_t dev)159 aesni_attach(device_t dev)
160 {
161 struct aesni_softc *sc;
162 int i;
163
164 sc = device_get_softc(dev);
165
166 sc->cid = crypto_get_driverid(dev, sizeof(struct aesni_session),
167 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC |
168 CRYPTOCAP_F_ACCEL_SOFTWARE);
169 if (sc->cid < 0) {
170 device_printf(dev, "Could not get crypto driver id.\n");
171 return (ENOMEM);
172 }
173
174 ctx_mtx = malloc(sizeof *ctx_mtx * (mp_maxid + 1), M_AESNI,
175 M_WAITOK|M_ZERO);
176 ctx_fpu = malloc(sizeof *ctx_fpu * (mp_maxid + 1), M_AESNI,
177 M_WAITOK|M_ZERO);
178
179 CPU_FOREACH(i) {
180 #ifdef __amd64__
181 ctx_fpu[i] = fpu_kern_alloc_ctx_domain(
182 pcpu_find(i)->pc_domain, FPU_KERN_NORMAL);
183 #else
184 ctx_fpu[i] = fpu_kern_alloc_ctx(FPU_KERN_NORMAL);
185 #endif
186 mtx_init(&ctx_mtx[i], "anifpumtx", NULL, MTX_DEF|MTX_NEW);
187 }
188
189 detect_cpu_features(&sc->has_aes, &sc->has_sha);
190 return (0);
191 }
192
193 static int
aesni_detach(device_t dev)194 aesni_detach(device_t dev)
195 {
196 struct aesni_softc *sc;
197
198 sc = device_get_softc(dev);
199
200 crypto_unregister_all(sc->cid);
201
202 aesni_cleanctx();
203
204 return (0);
205 }
206
207 static bool
aesni_auth_supported(struct aesni_softc * sc,const struct crypto_session_params * csp)208 aesni_auth_supported(struct aesni_softc *sc,
209 const struct crypto_session_params *csp)
210 {
211
212 if (!sc->has_sha)
213 return (false);
214
215 switch (csp->csp_auth_alg) {
216 case CRYPTO_SHA1:
217 case CRYPTO_SHA2_224:
218 case CRYPTO_SHA2_256:
219 case CRYPTO_SHA1_HMAC:
220 case CRYPTO_SHA2_224_HMAC:
221 case CRYPTO_SHA2_256_HMAC:
222 break;
223 default:
224 return (false);
225 }
226
227 return (true);
228 }
229
230 static bool
aesni_cipher_supported(struct aesni_softc * sc,const struct crypto_session_params * csp)231 aesni_cipher_supported(struct aesni_softc *sc,
232 const struct crypto_session_params *csp)
233 {
234
235 if (!sc->has_aes)
236 return (false);
237
238 switch (csp->csp_cipher_alg) {
239 case CRYPTO_AES_CBC:
240 case CRYPTO_AES_ICM:
241 switch (csp->csp_cipher_klen * 8) {
242 case 128:
243 case 192:
244 case 256:
245 break;
246 default:
247 CRYPTDEB("invalid CBC/ICM key length");
248 return (false);
249 }
250 if (csp->csp_ivlen != AES_BLOCK_LEN)
251 return (false);
252 break;
253 case CRYPTO_AES_XTS:
254 switch (csp->csp_cipher_klen * 8) {
255 case 256:
256 case 512:
257 break;
258 default:
259 CRYPTDEB("invalid XTS key length");
260 return (false);
261 }
262 if (csp->csp_ivlen != AES_XTS_IV_LEN)
263 return (false);
264 break;
265 default:
266 return (false);
267 }
268
269 return (true);
270 }
271
272 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN)
273
274 static int
aesni_probesession(device_t dev,const struct crypto_session_params * csp)275 aesni_probesession(device_t dev, const struct crypto_session_params *csp)
276 {
277 struct aesni_softc *sc;
278
279 sc = device_get_softc(dev);
280 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0)
281 return (EINVAL);
282 switch (csp->csp_mode) {
283 case CSP_MODE_DIGEST:
284 if (!aesni_auth_supported(sc, csp))
285 return (EINVAL);
286 break;
287 case CSP_MODE_CIPHER:
288 if (!aesni_cipher_supported(sc, csp))
289 return (EINVAL);
290 break;
291 case CSP_MODE_AEAD:
292 switch (csp->csp_cipher_alg) {
293 case CRYPTO_AES_NIST_GCM_16:
294 switch (csp->csp_cipher_klen * 8) {
295 case 128:
296 case 192:
297 case 256:
298 break;
299 default:
300 CRYPTDEB("invalid GCM key length");
301 return (EINVAL);
302 }
303 if (csp->csp_auth_mlen != 0 &&
304 csp->csp_auth_mlen != GMAC_DIGEST_LEN)
305 return (EINVAL);
306 if (!sc->has_aes)
307 return (EINVAL);
308 break;
309 case CRYPTO_AES_CCM_16:
310 switch (csp->csp_cipher_klen * 8) {
311 case 128:
312 case 192:
313 case 256:
314 break;
315 default:
316 CRYPTDEB("invalid CCM key length");
317 return (EINVAL);
318 }
319 if (!sc->has_aes)
320 return (EINVAL);
321 break;
322 default:
323 return (EINVAL);
324 }
325 break;
326 case CSP_MODE_ETA:
327 if (!aesni_auth_supported(sc, csp) ||
328 !aesni_cipher_supported(sc, csp))
329 return (EINVAL);
330 break;
331 default:
332 return (EINVAL);
333 }
334
335 return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
336 }
337
338 static int
aesni_newsession(device_t dev,crypto_session_t cses,const struct crypto_session_params * csp)339 aesni_newsession(device_t dev, crypto_session_t cses,
340 const struct crypto_session_params *csp)
341 {
342 struct aesni_softc *sc;
343 struct aesni_session *ses;
344 int error;
345
346 sc = device_get_softc(dev);
347
348 ses = crypto_get_driver_session(cses);
349
350 switch (csp->csp_mode) {
351 case CSP_MODE_DIGEST:
352 case CSP_MODE_CIPHER:
353 case CSP_MODE_AEAD:
354 case CSP_MODE_ETA:
355 break;
356 default:
357 return (EINVAL);
358 }
359 error = aesni_cipher_setup(ses, csp);
360 if (error != 0) {
361 CRYPTDEB("setup failed");
362 return (error);
363 }
364
365 return (0);
366 }
367
368 static int
aesni_process(device_t dev,struct cryptop * crp,int hint __unused)369 aesni_process(device_t dev, struct cryptop *crp, int hint __unused)
370 {
371 struct aesni_session *ses;
372 int error;
373
374 ses = crypto_get_driver_session(crp->crp_session);
375
376 error = aesni_cipher_process(ses, crp);
377
378 crp->crp_etype = error;
379 crypto_done(crp);
380 return (0);
381 }
382
383 static uint8_t *
aesni_cipher_alloc(struct cryptop * crp,int start,int length,bool * allocated)384 aesni_cipher_alloc(struct cryptop *crp, int start, int length, bool *allocated)
385 {
386 uint8_t *addr;
387
388 addr = crypto_contiguous_subsegment(crp, start, length);
389 if (addr != NULL) {
390 *allocated = false;
391 return (addr);
392 }
393 addr = malloc(length, M_AESNI, M_NOWAIT);
394 if (addr != NULL) {
395 *allocated = true;
396 crypto_copydata(crp, start, length, addr);
397 } else
398 *allocated = false;
399 return (addr);
400 }
401
402 static device_method_t aesni_methods[] = {
403 DEVMETHOD(device_identify, aesni_identify),
404 DEVMETHOD(device_probe, aesni_probe),
405 DEVMETHOD(device_attach, aesni_attach),
406 DEVMETHOD(device_detach, aesni_detach),
407
408 DEVMETHOD(cryptodev_probesession, aesni_probesession),
409 DEVMETHOD(cryptodev_newsession, aesni_newsession),
410 DEVMETHOD(cryptodev_process, aesni_process),
411
412 DEVMETHOD_END
413 };
414
415 static driver_t aesni_driver = {
416 "aesni",
417 aesni_methods,
418 sizeof(struct aesni_softc),
419 };
420 static devclass_t aesni_devclass;
421
422 DRIVER_MODULE(aesni, nexus, aesni_driver, aesni_devclass, 0, 0);
423 MODULE_VERSION(aesni, 1);
424 MODULE_DEPEND(aesni, crypto, 1, 1, 1);
425
426 static int
intel_sha1_update(void * vctx,const void * vdata,u_int datalen)427 intel_sha1_update(void *vctx, const void *vdata, u_int datalen)
428 {
429 struct sha1_ctxt *ctx = vctx;
430 const char *data = vdata;
431 size_t gaplen;
432 size_t gapstart;
433 size_t off;
434 size_t copysiz;
435 u_int blocks;
436
437 off = 0;
438 /* Do any aligned blocks without redundant copying. */
439 if (datalen >= 64 && ctx->count % 64 == 0) {
440 blocks = datalen / 64;
441 ctx->c.b64[0] += blocks * 64 * 8;
442 intel_sha1_step(ctx->h.b32, data + off, blocks);
443 off += blocks * 64;
444 }
445
446 while (off < datalen) {
447 gapstart = ctx->count % 64;
448 gaplen = 64 - gapstart;
449
450 copysiz = (gaplen < datalen - off) ? gaplen : datalen - off;
451 bcopy(&data[off], &ctx->m.b8[gapstart], copysiz);
452 ctx->count += copysiz;
453 ctx->count %= 64;
454 ctx->c.b64[0] += copysiz * 8;
455 if (ctx->count % 64 == 0)
456 intel_sha1_step(ctx->h.b32, (void *)ctx->m.b8, 1);
457 off += copysiz;
458 }
459
460 return (0);
461 }
462
463 static void
SHA1_Init_fn(void * ctx)464 SHA1_Init_fn(void *ctx)
465 {
466 sha1_init(ctx);
467 }
468
469 static void
SHA1_Finalize_fn(void * digest,void * ctx)470 SHA1_Finalize_fn(void *digest, void *ctx)
471 {
472 sha1_result(ctx, digest);
473 }
474
475 static int
intel_sha256_update(void * vctx,const void * vdata,u_int len)476 intel_sha256_update(void *vctx, const void *vdata, u_int len)
477 {
478 SHA256_CTX *ctx = vctx;
479 uint64_t bitlen;
480 uint32_t r;
481 u_int blocks;
482 const unsigned char *src = vdata;
483
484 /* Number of bytes left in the buffer from previous updates */
485 r = (ctx->count >> 3) & 0x3f;
486
487 /* Convert the length into a number of bits */
488 bitlen = len << 3;
489
490 /* Update number of bits */
491 ctx->count += bitlen;
492
493 /* Handle the case where we don't need to perform any transforms */
494 if (len < 64 - r) {
495 memcpy(&ctx->buf[r], src, len);
496 return (0);
497 }
498
499 /* Finish the current block */
500 memcpy(&ctx->buf[r], src, 64 - r);
501 intel_sha256_step(ctx->state, ctx->buf, 1);
502 src += 64 - r;
503 len -= 64 - r;
504
505 /* Perform complete blocks */
506 if (len >= 64) {
507 blocks = len / 64;
508 intel_sha256_step(ctx->state, src, blocks);
509 src += blocks * 64;
510 len -= blocks * 64;
511 }
512
513 /* Copy left over data into buffer */
514 memcpy(ctx->buf, src, len);
515
516 return (0);
517 }
518
519 static void
SHA224_Init_fn(void * ctx)520 SHA224_Init_fn(void *ctx)
521 {
522 SHA224_Init(ctx);
523 }
524
525 static void
SHA224_Finalize_fn(void * digest,void * ctx)526 SHA224_Finalize_fn(void *digest, void *ctx)
527 {
528 SHA224_Final(digest, ctx);
529 }
530
531 static void
SHA256_Init_fn(void * ctx)532 SHA256_Init_fn(void *ctx)
533 {
534 SHA256_Init(ctx);
535 }
536
537 static void
SHA256_Finalize_fn(void * digest,void * ctx)538 SHA256_Finalize_fn(void *digest, void *ctx)
539 {
540 SHA256_Final(digest, ctx);
541 }
542
543 static int
aesni_authprepare(struct aesni_session * ses,int klen)544 aesni_authprepare(struct aesni_session *ses, int klen)
545 {
546
547 if (klen > SHA1_BLOCK_LEN)
548 return (EINVAL);
549 if ((ses->hmac && klen == 0) || (!ses->hmac && klen != 0))
550 return (EINVAL);
551 return (0);
552 }
553
554 static int
aesni_cipher_setup(struct aesni_session * ses,const struct crypto_session_params * csp)555 aesni_cipher_setup(struct aesni_session *ses,
556 const struct crypto_session_params *csp)
557 {
558 struct fpu_kern_ctx *ctx;
559 uint8_t *schedbase;
560 int kt, ctxidx, error;
561
562 schedbase = (uint8_t *)roundup2((uintptr_t)ses->schedules,
563 AES_SCHED_ALIGN);
564 ses->enc_schedule = schedbase;
565 ses->dec_schedule = schedbase + AES_SCHED_LEN;
566 ses->xts_schedule = schedbase + AES_SCHED_LEN * 2;
567
568 switch (csp->csp_auth_alg) {
569 case CRYPTO_SHA1_HMAC:
570 ses->hmac = true;
571 /* FALLTHROUGH */
572 case CRYPTO_SHA1:
573 ses->hash_len = SHA1_HASH_LEN;
574 ses->hash_init = SHA1_Init_fn;
575 ses->hash_update = intel_sha1_update;
576 ses->hash_finalize = SHA1_Finalize_fn;
577 break;
578 case CRYPTO_SHA2_224_HMAC:
579 ses->hmac = true;
580 /* FALLTHROUGH */
581 case CRYPTO_SHA2_224:
582 ses->hash_len = SHA2_224_HASH_LEN;
583 ses->hash_init = SHA224_Init_fn;
584 ses->hash_update = intel_sha256_update;
585 ses->hash_finalize = SHA224_Finalize_fn;
586 break;
587 case CRYPTO_SHA2_256_HMAC:
588 ses->hmac = true;
589 /* FALLTHROUGH */
590 case CRYPTO_SHA2_256:
591 ses->hash_len = SHA2_256_HASH_LEN;
592 ses->hash_init = SHA256_Init_fn;
593 ses->hash_update = intel_sha256_update;
594 ses->hash_finalize = SHA256_Finalize_fn;
595 break;
596 }
597
598 if (ses->hash_len != 0) {
599 if (csp->csp_auth_mlen == 0)
600 ses->mlen = ses->hash_len;
601 else
602 ses->mlen = csp->csp_auth_mlen;
603
604 error = aesni_authprepare(ses, csp->csp_auth_klen);
605 if (error != 0)
606 return (error);
607 } else if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16) {
608 if (csp->csp_auth_mlen == 0)
609 ses->mlen = AES_CBC_MAC_HASH_LEN;
610 else
611 ses->mlen = csp->csp_auth_mlen;
612 }
613
614 kt = is_fpu_kern_thread(0) || (csp->csp_cipher_alg == 0);
615 if (!kt) {
616 ACQUIRE_CTX(ctxidx, ctx);
617 fpu_kern_enter(curthread, ctx,
618 FPU_KERN_NORMAL | FPU_KERN_KTHR);
619 }
620
621 error = 0;
622 if (csp->csp_cipher_key != NULL)
623 aesni_cipher_setup_common(ses, csp, csp->csp_cipher_key,
624 csp->csp_cipher_klen);
625
626 if (!kt) {
627 fpu_kern_leave(curthread, ctx);
628 RELEASE_CTX(ctxidx, ctx);
629 }
630 return (error);
631 }
632
633 static int
aesni_cipher_process(struct aesni_session * ses,struct cryptop * crp)634 aesni_cipher_process(struct aesni_session *ses, struct cryptop *crp)
635 {
636 const struct crypto_session_params *csp;
637 struct fpu_kern_ctx *ctx;
638 int error, ctxidx;
639 bool kt;
640
641 csp = crypto_get_params(crp->crp_session);
642 switch (csp->csp_cipher_alg) {
643 case CRYPTO_AES_CCM_16:
644 if (crp->crp_payload_length > ccm_max_payload_length(csp))
645 return (EMSGSIZE);
646 /* FALLTHROUGH */
647 case CRYPTO_AES_ICM:
648 case CRYPTO_AES_NIST_GCM_16:
649 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
650 return (EINVAL);
651 break;
652 case CRYPTO_AES_CBC:
653 case CRYPTO_AES_XTS:
654 /* CBC & XTS can only handle full blocks for now */
655 if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0)
656 return (EINVAL);
657 break;
658 }
659
660 ctx = NULL;
661 ctxidx = 0;
662 error = 0;
663 kt = is_fpu_kern_thread(0);
664 if (!kt) {
665 ACQUIRE_CTX(ctxidx, ctx);
666 fpu_kern_enter(curthread, ctx,
667 FPU_KERN_NORMAL | FPU_KERN_KTHR);
668 }
669
670 /* Do work */
671 if (csp->csp_mode == CSP_MODE_ETA) {
672 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
673 error = aesni_cipher_crypt(ses, crp, csp);
674 if (error == 0)
675 error = aesni_cipher_mac(ses, crp, csp);
676 } else {
677 error = aesni_cipher_mac(ses, crp, csp);
678 if (error == 0)
679 error = aesni_cipher_crypt(ses, crp, csp);
680 }
681 } else if (csp->csp_mode == CSP_MODE_DIGEST)
682 error = aesni_cipher_mac(ses, crp, csp);
683 else
684 error = aesni_cipher_crypt(ses, crp, csp);
685
686 if (!kt) {
687 fpu_kern_leave(curthread, ctx);
688 RELEASE_CTX(ctxidx, ctx);
689 }
690 return (error);
691 }
692
693 static int
aesni_cipher_crypt(struct aesni_session * ses,struct cryptop * crp,const struct crypto_session_params * csp)694 aesni_cipher_crypt(struct aesni_session *ses, struct cryptop *crp,
695 const struct crypto_session_params *csp)
696 {
697 uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN];
698 uint8_t *authbuf, *buf, *outbuf;
699 int error;
700 bool encflag, allocated, authallocated, outallocated, outcopy;
701
702 if (crp->crp_payload_length == 0) {
703 buf = NULL;
704 allocated = false;
705 } else {
706 buf = aesni_cipher_alloc(crp, crp->crp_payload_start,
707 crp->crp_payload_length, &allocated);
708 if (buf == NULL)
709 return (ENOMEM);
710 }
711
712 outallocated = false;
713 authallocated = false;
714 authbuf = NULL;
715 if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16 ||
716 csp->csp_cipher_alg == CRYPTO_AES_CCM_16) {
717 if (crp->crp_aad_length == 0) {
718 authbuf = NULL;
719 } else if (crp->crp_aad != NULL) {
720 authbuf = crp->crp_aad;
721 } else {
722 authbuf = aesni_cipher_alloc(crp, crp->crp_aad_start,
723 crp->crp_aad_length, &authallocated);
724 if (authbuf == NULL) {
725 error = ENOMEM;
726 goto out;
727 }
728 }
729 }
730
731 if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && crp->crp_payload_length > 0) {
732 outbuf = crypto_buffer_contiguous_subsegment(&crp->crp_obuf,
733 crp->crp_payload_output_start, crp->crp_payload_length);
734 if (outbuf == NULL) {
735 outcopy = true;
736 if (allocated)
737 outbuf = buf;
738 else {
739 outbuf = malloc(crp->crp_payload_length,
740 M_AESNI, M_NOWAIT);
741 if (outbuf == NULL) {
742 error = ENOMEM;
743 goto out;
744 }
745 outallocated = true;
746 }
747 } else
748 outcopy = false;
749 } else {
750 outbuf = buf;
751 outcopy = allocated;
752 }
753
754 error = 0;
755 encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
756 if (crp->crp_cipher_key != NULL)
757 aesni_cipher_setup_common(ses, csp, crp->crp_cipher_key,
758 csp->csp_cipher_klen);
759
760 crypto_read_iv(crp, iv);
761
762 switch (csp->csp_cipher_alg) {
763 case CRYPTO_AES_CBC:
764 if (encflag)
765 aesni_encrypt_cbc(ses->rounds, ses->enc_schedule,
766 crp->crp_payload_length, buf, outbuf, iv);
767 else {
768 if (buf != outbuf)
769 memcpy(outbuf, buf, crp->crp_payload_length);
770 aesni_decrypt_cbc(ses->rounds, ses->dec_schedule,
771 crp->crp_payload_length, outbuf, iv);
772 }
773 break;
774 case CRYPTO_AES_ICM:
775 /* encryption & decryption are the same */
776 aesni_encrypt_icm(ses->rounds, ses->enc_schedule,
777 crp->crp_payload_length, buf, outbuf, iv);
778 break;
779 case CRYPTO_AES_XTS:
780 if (encflag)
781 aesni_encrypt_xts(ses->rounds, ses->enc_schedule,
782 ses->xts_schedule, crp->crp_payload_length, buf,
783 outbuf, iv);
784 else
785 aesni_decrypt_xts(ses->rounds, ses->dec_schedule,
786 ses->xts_schedule, crp->crp_payload_length, buf,
787 outbuf, iv);
788 break;
789 case CRYPTO_AES_NIST_GCM_16:
790 if (encflag) {
791 memset(tag, 0, sizeof(tag));
792 AES_GCM_encrypt(buf, outbuf, authbuf, iv, tag,
793 crp->crp_payload_length, crp->crp_aad_length,
794 csp->csp_ivlen, ses->enc_schedule, ses->rounds);
795 crypto_copyback(crp, crp->crp_digest_start, sizeof(tag),
796 tag);
797 } else {
798 crypto_copydata(crp, crp->crp_digest_start, sizeof(tag),
799 tag);
800 if (!AES_GCM_decrypt(buf, outbuf, authbuf, iv, tag,
801 crp->crp_payload_length, crp->crp_aad_length,
802 csp->csp_ivlen, ses->enc_schedule, ses->rounds))
803 error = EBADMSG;
804 }
805 break;
806 case CRYPTO_AES_CCM_16:
807 if (encflag) {
808 memset(tag, 0, sizeof(tag));
809 AES_CCM_encrypt(buf, outbuf, authbuf, iv, tag,
810 crp->crp_payload_length, crp->crp_aad_length,
811 csp->csp_ivlen, ses->mlen, ses->enc_schedule,
812 ses->rounds);
813 crypto_copyback(crp, crp->crp_digest_start, ses->mlen,
814 tag);
815 } else {
816 crypto_copydata(crp, crp->crp_digest_start, ses->mlen,
817 tag);
818 if (!AES_CCM_decrypt(buf, outbuf, authbuf, iv, tag,
819 crp->crp_payload_length, crp->crp_aad_length,
820 csp->csp_ivlen, ses->mlen, ses->enc_schedule,
821 ses->rounds))
822 error = EBADMSG;
823 }
824 break;
825 }
826 if (outcopy && error == 0)
827 crypto_copyback(crp, CRYPTO_HAS_OUTPUT_BUFFER(crp) ?
828 crp->crp_payload_output_start : crp->crp_payload_start,
829 crp->crp_payload_length, outbuf);
830
831 out:
832 if (allocated)
833 zfree(buf, M_AESNI);
834 if (authallocated)
835 zfree(authbuf, M_AESNI);
836 if (outallocated)
837 zfree(outbuf, M_AESNI);
838 explicit_bzero(iv, sizeof(iv));
839 explicit_bzero(tag, sizeof(tag));
840 return (error);
841 }
842
843 static int
aesni_cipher_mac(struct aesni_session * ses,struct cryptop * crp,const struct crypto_session_params * csp)844 aesni_cipher_mac(struct aesni_session *ses, struct cryptop *crp,
845 const struct crypto_session_params *csp)
846 {
847 union {
848 struct SHA256Context sha2 __aligned(16);
849 struct sha1_ctxt sha1 __aligned(16);
850 } sctx;
851 uint32_t res[SHA2_256_HASH_LEN / sizeof(uint32_t)];
852 const uint8_t *key;
853 int i, keylen;
854
855 if (crp->crp_auth_key != NULL)
856 key = crp->crp_auth_key;
857 else
858 key = csp->csp_auth_key;
859 keylen = csp->csp_auth_klen;
860
861 if (ses->hmac) {
862 uint8_t hmac_key[SHA1_BLOCK_LEN] __aligned(16);
863
864 /* Inner hash: (K ^ IPAD) || data */
865 ses->hash_init(&sctx);
866 for (i = 0; i < keylen; i++)
867 hmac_key[i] = key[i] ^ HMAC_IPAD_VAL;
868 for (i = keylen; i < sizeof(hmac_key); i++)
869 hmac_key[i] = 0 ^ HMAC_IPAD_VAL;
870 ses->hash_update(&sctx, hmac_key, sizeof(hmac_key));
871
872 if (crp->crp_aad != NULL)
873 ses->hash_update(&sctx, crp->crp_aad,
874 crp->crp_aad_length);
875 else
876 crypto_apply(crp, crp->crp_aad_start,
877 crp->crp_aad_length, ses->hash_update, &sctx);
878 if (CRYPTO_HAS_OUTPUT_BUFFER(crp) &&
879 CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
880 crypto_apply_buf(&crp->crp_obuf,
881 crp->crp_payload_output_start,
882 crp->crp_payload_length,
883 ses->hash_update, &sctx);
884 else
885 crypto_apply(crp, crp->crp_payload_start,
886 crp->crp_payload_length, ses->hash_update, &sctx);
887
888 if (csp->csp_flags & CSP_F_ESN)
889 ses->hash_update(&sctx, crp->crp_esn, 4);
890
891 ses->hash_finalize(res, &sctx);
892
893 /* Outer hash: (K ^ OPAD) || inner hash */
894 ses->hash_init(&sctx);
895 for (i = 0; i < keylen; i++)
896 hmac_key[i] = key[i] ^ HMAC_OPAD_VAL;
897 for (i = keylen; i < sizeof(hmac_key); i++)
898 hmac_key[i] = 0 ^ HMAC_OPAD_VAL;
899 ses->hash_update(&sctx, hmac_key, sizeof(hmac_key));
900 ses->hash_update(&sctx, res, ses->hash_len);
901 ses->hash_finalize(res, &sctx);
902 explicit_bzero(hmac_key, sizeof(hmac_key));
903 } else {
904 ses->hash_init(&sctx);
905
906 if (crp->crp_aad != NULL)
907 ses->hash_update(&sctx, crp->crp_aad,
908 crp->crp_aad_length);
909 else
910 crypto_apply(crp, crp->crp_aad_start,
911 crp->crp_aad_length, ses->hash_update, &sctx);
912 if (CRYPTO_HAS_OUTPUT_BUFFER(crp) &&
913 CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
914 crypto_apply_buf(&crp->crp_obuf,
915 crp->crp_payload_output_start,
916 crp->crp_payload_length,
917 ses->hash_update, &sctx);
918 else
919 crypto_apply(crp, crp->crp_payload_start,
920 crp->crp_payload_length,
921 ses->hash_update, &sctx);
922
923 ses->hash_finalize(res, &sctx);
924 }
925
926 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
927 uint32_t res2[SHA2_256_HASH_LEN / sizeof(uint32_t)];
928
929 crypto_copydata(crp, crp->crp_digest_start, ses->mlen, res2);
930 if (timingsafe_bcmp(res, res2, ses->mlen) != 0)
931 return (EBADMSG);
932 explicit_bzero(res2, sizeof(res2));
933 } else
934 crypto_copyback(crp, crp->crp_digest_start, ses->mlen, res);
935 explicit_bzero(res, sizeof(res));
936 return (0);
937 }
938