1 /*-
2 * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3 * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
4 * Copyright (c) 2014,2016 The FreeBSD Foundation
5 * Copyright (c) 2020 Ampere Computing
6 * All rights reserved.
7 *
8 * Portions of this software were developed by John-Mark Gurney
9 * under sponsorship of the FreeBSD Foundation and
10 * Rubicon Communications, LLC (Netgate).
11 *
12 * This software was developed by Andrew Turner under
13 * sponsorship from the FreeBSD Foundation.
14 *
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
17 * are met:
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 * notice, this list of conditions and the following disclaimer in the
22 * documentation and/or other materials provided with the distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 */
36
37 /*
38 * This is based on the aesni code.
39 */
40
41 #include <sys/cdefs.h>
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/kernel.h>
45 #include <sys/bus.h>
46 #include <sys/endian.h>
47 #include <sys/lock.h>
48 #include <sys/malloc.h>
49 #include <sys/mbuf.h>
50 #include <sys/module.h>
51 #include <sys/mutex.h>
52 #include <sys/queue.h>
53 #include <sys/rwlock.h>
54 #include <sys/smp.h>
55 #include <sys/uio.h>
56
57 #include <machine/vfp.h>
58
59 #include <opencrypto/cryptodev.h>
60 #include <opencrypto/gmac.h>
61 #include <cryptodev_if.h>
62 #include <crypto/armv8/armv8_crypto.h>
63 #include <crypto/rijndael/rijndael.h>
64
65 struct armv8_crypto_softc {
66 int dieing;
67 int32_t cid;
68 struct rwlock lock;
69 bool has_pmul;
70 };
71
72 static struct mtx *ctx_mtx;
73 static struct fpu_kern_ctx **ctx_vfp;
74
75 #define AQUIRE_CTX(i, ctx) \
76 do { \
77 (i) = PCPU_GET(cpuid); \
78 mtx_lock(&ctx_mtx[(i)]); \
79 (ctx) = ctx_vfp[(i)]; \
80 } while (0)
81 #define RELEASE_CTX(i, ctx) \
82 do { \
83 mtx_unlock(&ctx_mtx[(i)]); \
84 (i) = -1; \
85 (ctx) = NULL; \
86 } while (0)
87
88 static int armv8_crypto_cipher_process(struct armv8_crypto_session *,
89 struct cryptop *);
90
91 MALLOC_DEFINE(M_ARMV8_CRYPTO, "armv8_crypto", "ARMv8 Crypto Data");
92
93 static void
armv8_crypto_identify(driver_t * drv,device_t parent)94 armv8_crypto_identify(driver_t *drv, device_t parent)
95 {
96
97 /* NB: order 10 is so we get attached after h/w devices */
98 if (device_find_child(parent, "armv8crypto", -1) == NULL &&
99 BUS_ADD_CHILD(parent, 10, "armv8crypto", -1) == 0)
100 panic("ARMv8 crypto: could not attach");
101 }
102
103 static int
armv8_crypto_probe(device_t dev)104 armv8_crypto_probe(device_t dev)
105 {
106 uint64_t reg;
107 int ret = ENXIO;
108
109 reg = READ_SPECIALREG(id_aa64isar0_el1);
110
111 switch (ID_AA64ISAR0_AES_VAL(reg)) {
112 case ID_AA64ISAR0_AES_BASE:
113 ret = 0;
114 device_set_desc(dev, "AES-CBC,AES-XTS");
115 break;
116 case ID_AA64ISAR0_AES_PMULL:
117 ret = 0;
118 device_set_desc(dev, "AES-CBC,AES-XTS,AES-GCM");
119 break;
120 default:
121 break;
122 case ID_AA64ISAR0_AES_NONE:
123 device_printf(dev, "CPU lacks AES instructions\n");
124 break;
125 }
126
127 /* TODO: Check more fields as we support more features */
128
129 return (ret);
130 }
131
132 static int
armv8_crypto_attach(device_t dev)133 armv8_crypto_attach(device_t dev)
134 {
135 struct armv8_crypto_softc *sc;
136 uint64_t reg;
137 int i;
138
139 sc = device_get_softc(dev);
140 sc->dieing = 0;
141
142 reg = READ_SPECIALREG(id_aa64isar0_el1);
143
144 if (ID_AA64ISAR0_AES_VAL(reg) == ID_AA64ISAR0_AES_PMULL)
145 sc->has_pmul = true;
146
147 sc->cid = crypto_get_driverid(dev, sizeof(struct armv8_crypto_session),
148 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | CRYPTOCAP_F_ACCEL_SOFTWARE);
149 if (sc->cid < 0) {
150 device_printf(dev, "Could not get crypto driver id.\n");
151 return (ENOMEM);
152 }
153
154 rw_init(&sc->lock, "armv8crypto");
155
156 ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_ARMV8_CRYPTO,
157 M_WAITOK|M_ZERO);
158 ctx_vfp = malloc(sizeof(*ctx_vfp) * (mp_maxid + 1), M_ARMV8_CRYPTO,
159 M_WAITOK|M_ZERO);
160
161 CPU_FOREACH(i) {
162 ctx_vfp[i] = fpu_kern_alloc_ctx(0);
163 mtx_init(&ctx_mtx[i], "armv8cryptoctx", NULL, MTX_DEF|MTX_NEW);
164 }
165
166 return (0);
167 }
168
169 static int
armv8_crypto_detach(device_t dev)170 armv8_crypto_detach(device_t dev)
171 {
172 struct armv8_crypto_softc *sc;
173 int i;
174
175 sc = device_get_softc(dev);
176
177 rw_wlock(&sc->lock);
178 sc->dieing = 1;
179 rw_wunlock(&sc->lock);
180 crypto_unregister_all(sc->cid);
181
182 rw_destroy(&sc->lock);
183
184 CPU_FOREACH(i) {
185 if (ctx_vfp[i] != NULL) {
186 mtx_destroy(&ctx_mtx[i]);
187 fpu_kern_free_ctx(ctx_vfp[i]);
188 }
189 ctx_vfp[i] = NULL;
190 }
191 free(ctx_mtx, M_ARMV8_CRYPTO);
192 ctx_mtx = NULL;
193 free(ctx_vfp, M_ARMV8_CRYPTO);
194 ctx_vfp = NULL;
195
196 return (0);
197 }
198
199 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)
200
201 static int
armv8_crypto_probesession(device_t dev,const struct crypto_session_params * csp)202 armv8_crypto_probesession(device_t dev,
203 const struct crypto_session_params *csp)
204 {
205 struct armv8_crypto_softc *sc;
206
207 sc = device_get_softc(dev);
208
209 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0)
210 return (EINVAL);
211
212 switch (csp->csp_mode) {
213 case CSP_MODE_AEAD:
214 switch (csp->csp_cipher_alg) {
215 case CRYPTO_AES_NIST_GCM_16:
216 if (!sc->has_pmul)
217 return (EINVAL);
218 if (csp->csp_auth_mlen != 0 &&
219 csp->csp_auth_mlen != GMAC_DIGEST_LEN)
220 return (EINVAL);
221 switch (csp->csp_cipher_klen * 8) {
222 case 128:
223 case 192:
224 case 256:
225 break;
226 default:
227 return (EINVAL);
228 }
229 break;
230 default:
231 return (EINVAL);
232 }
233 break;
234 case CSP_MODE_CIPHER:
235 switch (csp->csp_cipher_alg) {
236 case CRYPTO_AES_CBC:
237 if (csp->csp_ivlen != AES_BLOCK_LEN)
238 return (EINVAL);
239 switch (csp->csp_cipher_klen * 8) {
240 case 128:
241 case 192:
242 case 256:
243 break;
244 default:
245 return (EINVAL);
246 }
247 break;
248 case CRYPTO_AES_XTS:
249 if (csp->csp_ivlen != AES_XTS_IV_LEN)
250 return (EINVAL);
251 switch (csp->csp_cipher_klen * 8) {
252 case 256:
253 case 512:
254 break;
255 default:
256 return (EINVAL);
257 }
258 break;
259 default:
260 return (EINVAL);
261 }
262 break;
263 default:
264 return (EINVAL);
265 }
266 return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
267 }
268
269 static int
armv8_crypto_cipher_setup(struct armv8_crypto_session * ses,const struct crypto_session_params * csp,const uint8_t * key,int keylen)270 armv8_crypto_cipher_setup(struct armv8_crypto_session *ses,
271 const struct crypto_session_params *csp, const uint8_t *key, int keylen)
272 {
273 __uint128_val_t H;
274 struct fpu_kern_ctx *ctx;
275 int kt, i;
276
277 if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
278 keylen /= 2;
279
280 switch (keylen * 8) {
281 case 128:
282 case 192:
283 case 256:
284 break;
285 default:
286 return (EINVAL);
287 }
288
289 kt = is_fpu_kern_thread(0);
290 if (!kt) {
291 AQUIRE_CTX(i, ctx);
292 fpu_kern_enter(curthread, ctx,
293 FPU_KERN_NORMAL | FPU_KERN_KTHR);
294 }
295
296 aes_v8_set_encrypt_key(key,
297 keylen * 8, &ses->enc_schedule);
298
299 if ((csp->csp_cipher_alg == CRYPTO_AES_XTS) ||
300 (csp->csp_cipher_alg == CRYPTO_AES_CBC))
301 aes_v8_set_decrypt_key(key,
302 keylen * 8, &ses->dec_schedule);
303
304 if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
305 aes_v8_set_encrypt_key(key + keylen, keylen * 8, &ses->xts_schedule);
306
307 if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
308 memset(H.c, 0, sizeof(H.c));
309 aes_v8_encrypt(H.c, H.c, &ses->enc_schedule);
310 H.u[0] = bswap64(H.u[0]);
311 H.u[1] = bswap64(H.u[1]);
312 gcm_init_v8(ses->Htable, H.u);
313 }
314
315 if (!kt) {
316 fpu_kern_leave(curthread, ctx);
317 RELEASE_CTX(i, ctx);
318 }
319
320 return (0);
321 }
322
323 static int
armv8_crypto_newsession(device_t dev,crypto_session_t cses,const struct crypto_session_params * csp)324 armv8_crypto_newsession(device_t dev, crypto_session_t cses,
325 const struct crypto_session_params *csp)
326 {
327 struct armv8_crypto_softc *sc;
328 struct armv8_crypto_session *ses;
329 int error;
330
331 sc = device_get_softc(dev);
332 rw_wlock(&sc->lock);
333 if (sc->dieing) {
334 rw_wunlock(&sc->lock);
335 return (EINVAL);
336 }
337
338 ses = crypto_get_driver_session(cses);
339 error = armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key,
340 csp->csp_cipher_klen);
341 rw_wunlock(&sc->lock);
342 return (error);
343 }
344
345 static int
armv8_crypto_process(device_t dev,struct cryptop * crp,int hint __unused)346 armv8_crypto_process(device_t dev, struct cryptop *crp, int hint __unused)
347 {
348 struct armv8_crypto_session *ses;
349
350 ses = crypto_get_driver_session(crp->crp_session);
351 crp->crp_etype = armv8_crypto_cipher_process(ses, crp);
352 crypto_done(crp);
353 return (0);
354 }
355
356 static uint8_t *
armv8_crypto_cipher_alloc(struct cryptop * crp,int start,int length,int * allocated)357 armv8_crypto_cipher_alloc(struct cryptop *crp, int start, int length, int *allocated)
358 {
359 uint8_t *addr;
360
361 addr = crypto_contiguous_subsegment(crp, start, length);
362 if (addr != NULL) {
363 *allocated = 0;
364 return (addr);
365 }
366 addr = malloc(crp->crp_payload_length, M_ARMV8_CRYPTO, M_NOWAIT);
367 if (addr != NULL) {
368 *allocated = 1;
369 crypto_copydata(crp, start, length, addr);
370 } else
371 *allocated = 0;
372 return (addr);
373 }
374
375 static int
armv8_crypto_cipher_process(struct armv8_crypto_session * ses,struct cryptop * crp)376 armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
377 struct cryptop *crp)
378 {
379 struct crypto_buffer_cursor fromc, toc;
380 const struct crypto_session_params *csp;
381 struct fpu_kern_ctx *ctx;
382 uint8_t *authbuf;
383 uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN];
384 int authallocated, i;
385 int encflag;
386 int kt;
387 int error;
388
389 csp = crypto_get_params(crp->crp_session);
390 encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
391
392 authallocated = 0;
393 authbuf = NULL;
394 kt = 1;
395
396 if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
397 if (crp->crp_aad != NULL)
398 authbuf = crp->crp_aad;
399 else
400 authbuf = armv8_crypto_cipher_alloc(crp, crp->crp_aad_start,
401 crp->crp_aad_length, &authallocated);
402 if (authbuf == NULL) {
403 error = ENOMEM;
404 goto out;
405 }
406 }
407 crypto_cursor_init(&fromc, &crp->crp_buf);
408 crypto_cursor_advance(&fromc, crp->crp_payload_start);
409 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
410 crypto_cursor_init(&toc, &crp->crp_obuf);
411 crypto_cursor_advance(&toc, crp->crp_payload_output_start);
412 } else {
413 crypto_cursor_copy(&fromc, &toc);
414 }
415
416 kt = is_fpu_kern_thread(0);
417 if (!kt) {
418 AQUIRE_CTX(i, ctx);
419 fpu_kern_enter(curthread, ctx,
420 FPU_KERN_NORMAL | FPU_KERN_KTHR);
421 }
422
423 if (crp->crp_cipher_key != NULL) {
424 armv8_crypto_cipher_setup(ses, csp, crp->crp_cipher_key,
425 csp->csp_cipher_klen);
426 }
427
428 crypto_read_iv(crp, iv);
429
430 switch (csp->csp_cipher_alg) {
431 case CRYPTO_AES_CBC:
432 if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) {
433 error = EINVAL;
434 goto out;
435 }
436 if (encflag)
437 armv8_aes_encrypt_cbc(&ses->enc_schedule,
438 crp->crp_payload_length, &fromc, &toc, iv);
439 else
440 armv8_aes_decrypt_cbc(&ses->dec_schedule,
441 crp->crp_payload_length, &fromc, &toc, iv);
442 break;
443 case CRYPTO_AES_XTS:
444 if (encflag)
445 armv8_aes_encrypt_xts(&ses->enc_schedule,
446 &ses->xts_schedule.aes_key, crp->crp_payload_length,
447 &fromc, &toc, iv);
448 else
449 armv8_aes_decrypt_xts(&ses->dec_schedule,
450 &ses->xts_schedule.aes_key, crp->crp_payload_length,
451 &fromc, &toc, iv);
452 break;
453 case CRYPTO_AES_NIST_GCM_16:
454 if (encflag) {
455 memset(tag, 0, sizeof(tag));
456 armv8_aes_encrypt_gcm(&ses->enc_schedule,
457 crp->crp_payload_length, &fromc, &toc,
458 crp->crp_aad_length, authbuf, tag, iv, ses->Htable);
459 crypto_copyback(crp, crp->crp_digest_start, sizeof(tag),
460 tag);
461 } else {
462 crypto_copydata(crp, crp->crp_digest_start, sizeof(tag),
463 tag);
464 error = armv8_aes_decrypt_gcm(&ses->enc_schedule,
465 crp->crp_payload_length, &fromc, &toc,
466 crp->crp_aad_length, authbuf, tag, iv, ses->Htable);
467 if (error != 0)
468 goto out;
469 }
470 break;
471 }
472
473 error = 0;
474 out:
475 if (!kt) {
476 fpu_kern_leave(curthread, ctx);
477 RELEASE_CTX(i, ctx);
478 }
479
480 if (authallocated)
481 zfree(authbuf, M_ARMV8_CRYPTO);
482 explicit_bzero(iv, sizeof(iv));
483 explicit_bzero(tag, sizeof(tag));
484
485 return (error);
486 }
487
488 static device_method_t armv8_crypto_methods[] = {
489 DEVMETHOD(device_identify, armv8_crypto_identify),
490 DEVMETHOD(device_probe, armv8_crypto_probe),
491 DEVMETHOD(device_attach, armv8_crypto_attach),
492 DEVMETHOD(device_detach, armv8_crypto_detach),
493
494 DEVMETHOD(cryptodev_probesession, armv8_crypto_probesession),
495 DEVMETHOD(cryptodev_newsession, armv8_crypto_newsession),
496 DEVMETHOD(cryptodev_process, armv8_crypto_process),
497
498 DEVMETHOD_END,
499 };
500
501 static DEFINE_CLASS_0(armv8crypto, armv8_crypto_driver, armv8_crypto_methods,
502 sizeof(struct armv8_crypto_softc));
503 static devclass_t armv8_crypto_devclass;
504
505 DRIVER_MODULE(armv8crypto, nexus, armv8_crypto_driver, armv8_crypto_devclass,
506 0, 0);
507