1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (C) 2008-2009 Semihalf, Piotr Ziecik
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
19 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
22 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
23 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
24 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 /*
29 * Freescale integrated Security Engine (SEC) driver. Currently SEC 2.0 and
30 * 3.0 are supported.
31 */
32
33 #include <sys/cdefs.h>
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/bus.h>
37 #include <sys/endian.h>
38 #include <sys/kernel.h>
39 #include <sys/lock.h>
40 #include <sys/malloc.h>
41 #include <sys/mbuf.h>
42 #include <sys/module.h>
43 #include <sys/mutex.h>
44 #include <sys/random.h>
45 #include <sys/rman.h>
46
47 #include <machine/_inttypes.h>
48 #include <machine/bus.h>
49 #include <machine/resource.h>
50
51 #include <opencrypto/cryptodev.h>
52 #include <opencrypto/xform_auth.h>
53 #include "cryptodev_if.h"
54
55 #include <dev/ofw/ofw_bus_subr.h>
56 #include <dev/sec/sec.h>
57
58 static int sec_probe(device_t dev);
59 static int sec_attach(device_t dev);
60 static int sec_detach(device_t dev);
61 static int sec_suspend(device_t dev);
62 static int sec_resume(device_t dev);
63 static int sec_shutdown(device_t dev);
64 static void sec_primary_intr(void *arg);
65 static void sec_secondary_intr(void *arg);
66 static int sec_setup_intr(struct sec_softc *sc, struct resource **ires,
67 void **ihand, int *irid, driver_intr_t handler, const char *iname);
68 static void sec_release_intr(struct sec_softc *sc, struct resource *ires,
69 void *ihand, int irid, const char *iname);
70 static int sec_controller_reset(struct sec_softc *sc);
71 static int sec_channel_reset(struct sec_softc *sc, int channel, int full);
72 static int sec_init(struct sec_softc *sc);
73 static int sec_alloc_dma_mem(struct sec_softc *sc,
74 struct sec_dma_mem *dma_mem, bus_size_t size);
75 static int sec_desc_map_dma(struct sec_softc *sc,
76 struct sec_dma_mem *dma_mem, struct cryptop *crp, bus_size_t size,
77 struct sec_desc_map_info *sdmi);
78 static void sec_free_dma_mem(struct sec_dma_mem *dma_mem);
79 static void sec_enqueue(struct sec_softc *sc);
80 static int sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc,
81 int channel);
82 static int sec_eu_channel(struct sec_softc *sc, int eu);
83 static int sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc,
84 u_int n, struct cryptop *crp, bus_size_t doffset, bus_size_t dsize);
85 static int sec_make_pointer_direct(struct sec_softc *sc,
86 struct sec_desc *desc, u_int n, bus_addr_t data, bus_size_t dsize);
87 static int sec_probesession(device_t dev,
88 const struct crypto_session_params *csp);
89 static int sec_newsession(device_t dev, crypto_session_t cses,
90 const struct crypto_session_params *csp);
91 static int sec_process(device_t dev, struct cryptop *crp, int hint);
92 static int sec_build_common_ns_desc(struct sec_softc *sc,
93 struct sec_desc *desc, const struct crypto_session_params *csp,
94 struct cryptop *crp);
95 static int sec_build_common_s_desc(struct sec_softc *sc,
96 struct sec_desc *desc, const struct crypto_session_params *csp,
97 struct cryptop *crp);
98
99 static struct sec_desc *sec_find_desc(struct sec_softc *sc, bus_addr_t paddr);
100
101 /* AESU */
102 static bool sec_aesu_newsession(const struct crypto_session_params *csp);
103 static int sec_aesu_make_desc(struct sec_softc *sc,
104 const struct crypto_session_params *csp, struct sec_desc *desc,
105 struct cryptop *crp);
106
107 /* MDEU */
108 static bool sec_mdeu_can_handle(u_int alg);
109 static int sec_mdeu_config(const struct crypto_session_params *csp,
110 u_int *eu, u_int *mode, u_int *hashlen);
111 static bool sec_mdeu_newsession(const struct crypto_session_params *csp);
112 static int sec_mdeu_make_desc(struct sec_softc *sc,
113 const struct crypto_session_params *csp, struct sec_desc *desc,
114 struct cryptop *crp);
115
116 static device_method_t sec_methods[] = {
117 /* Device interface */
118 DEVMETHOD(device_probe, sec_probe),
119 DEVMETHOD(device_attach, sec_attach),
120 DEVMETHOD(device_detach, sec_detach),
121
122 DEVMETHOD(device_suspend, sec_suspend),
123 DEVMETHOD(device_resume, sec_resume),
124 DEVMETHOD(device_shutdown, sec_shutdown),
125
126 /* Crypto methods */
127 DEVMETHOD(cryptodev_probesession, sec_probesession),
128 DEVMETHOD(cryptodev_newsession, sec_newsession),
129 DEVMETHOD(cryptodev_process, sec_process),
130
131 DEVMETHOD_END
132 };
133 static driver_t sec_driver = {
134 "sec",
135 sec_methods,
136 sizeof(struct sec_softc),
137 };
138
139 static devclass_t sec_devclass;
140 DRIVER_MODULE(sec, simplebus, sec_driver, sec_devclass, 0, 0);
141 MODULE_DEPEND(sec, crypto, 1, 1, 1);
142
143 static struct sec_eu_methods sec_eus[] = {
144 {
145 sec_aesu_newsession,
146 sec_aesu_make_desc,
147 },
148 {
149 sec_mdeu_newsession,
150 sec_mdeu_make_desc,
151 },
152 { NULL, NULL }
153 };
154
155 static inline void
sec_sync_dma_mem(struct sec_dma_mem * dma_mem,bus_dmasync_op_t op)156 sec_sync_dma_mem(struct sec_dma_mem *dma_mem, bus_dmasync_op_t op)
157 {
158
159 /* Sync only if dma memory is valid */
160 if (dma_mem->dma_vaddr != NULL)
161 bus_dmamap_sync(dma_mem->dma_tag, dma_mem->dma_map, op);
162 }
163
164 static inline void *
sec_get_pointer_data(struct sec_desc * desc,u_int n)165 sec_get_pointer_data(struct sec_desc *desc, u_int n)
166 {
167
168 return (desc->sd_ptr_dmem[n].dma_vaddr);
169 }
170
171 static int
sec_probe(device_t dev)172 sec_probe(device_t dev)
173 {
174 struct sec_softc *sc;
175 uint64_t id;
176
177 if (!ofw_bus_status_okay(dev))
178 return (ENXIO);
179
180 if (!ofw_bus_is_compatible(dev, "fsl,sec2.0"))
181 return (ENXIO);
182
183 sc = device_get_softc(dev);
184
185 sc->sc_rrid = 0;
186 sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid,
187 RF_ACTIVE);
188
189 if (sc->sc_rres == NULL)
190 return (ENXIO);
191
192 sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres);
193 sc->sc_bas.bst = rman_get_bustag(sc->sc_rres);
194
195 id = SEC_READ(sc, SEC_ID);
196
197 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres);
198
199 switch (id) {
200 case SEC_20_ID:
201 device_set_desc(dev, "Freescale Security Engine 2.0");
202 sc->sc_version = 2;
203 break;
204 case SEC_30_ID:
205 device_set_desc(dev, "Freescale Security Engine 3.0");
206 sc->sc_version = 3;
207 break;
208 case SEC_31_ID:
209 device_set_desc(dev, "Freescale Security Engine 3.1");
210 sc->sc_version = 3;
211 break;
212 default:
213 device_printf(dev, "unknown SEC ID 0x%016"PRIx64"!\n", id);
214 return (ENXIO);
215 }
216
217 return (0);
218 }
219
220 static int
sec_attach(device_t dev)221 sec_attach(device_t dev)
222 {
223 struct sec_softc *sc;
224 struct sec_hw_lt *lt;
225 int error = 0;
226 int i;
227
228 sc = device_get_softc(dev);
229 sc->sc_dev = dev;
230 sc->sc_blocked = 0;
231 sc->sc_shutdown = 0;
232
233 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct sec_session),
234 CRYPTOCAP_F_HARDWARE);
235 if (sc->sc_cid < 0) {
236 device_printf(dev, "could not get crypto driver ID!\n");
237 return (ENXIO);
238 }
239
240 /* Init locks */
241 mtx_init(&sc->sc_controller_lock, device_get_nameunit(dev),
242 "SEC Controller lock", MTX_DEF);
243 mtx_init(&sc->sc_descriptors_lock, device_get_nameunit(dev),
244 "SEC Descriptors lock", MTX_DEF);
245
246 /* Allocate I/O memory for SEC registers */
247 sc->sc_rrid = 0;
248 sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid,
249 RF_ACTIVE);
250
251 if (sc->sc_rres == NULL) {
252 device_printf(dev, "could not allocate I/O memory!\n");
253 goto fail1;
254 }
255
256 sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres);
257 sc->sc_bas.bst = rman_get_bustag(sc->sc_rres);
258
259 /* Setup interrupts */
260 sc->sc_pri_irid = 0;
261 error = sec_setup_intr(sc, &sc->sc_pri_ires, &sc->sc_pri_ihand,
262 &sc->sc_pri_irid, sec_primary_intr, "primary");
263
264 if (error)
265 goto fail2;
266
267 if (sc->sc_version == 3) {
268 sc->sc_sec_irid = 1;
269 error = sec_setup_intr(sc, &sc->sc_sec_ires, &sc->sc_sec_ihand,
270 &sc->sc_sec_irid, sec_secondary_intr, "secondary");
271
272 if (error)
273 goto fail3;
274 }
275
276 /* Alloc DMA memory for descriptors and link tables */
277 error = sec_alloc_dma_mem(sc, &(sc->sc_desc_dmem),
278 SEC_DESCRIPTORS * sizeof(struct sec_hw_desc));
279
280 if (error)
281 goto fail4;
282
283 error = sec_alloc_dma_mem(sc, &(sc->sc_lt_dmem),
284 (SEC_LT_ENTRIES + 1) * sizeof(struct sec_hw_lt));
285
286 if (error)
287 goto fail5;
288
289 /* Fill in descriptors and link tables */
290 for (i = 0; i < SEC_DESCRIPTORS; i++) {
291 sc->sc_desc[i].sd_desc =
292 (struct sec_hw_desc*)(sc->sc_desc_dmem.dma_vaddr) + i;
293 sc->sc_desc[i].sd_desc_paddr = sc->sc_desc_dmem.dma_paddr +
294 (i * sizeof(struct sec_hw_desc));
295 }
296
297 for (i = 0; i < SEC_LT_ENTRIES + 1; i++) {
298 sc->sc_lt[i].sl_lt =
299 (struct sec_hw_lt*)(sc->sc_lt_dmem.dma_vaddr) + i;
300 sc->sc_lt[i].sl_lt_paddr = sc->sc_lt_dmem.dma_paddr +
301 (i * sizeof(struct sec_hw_lt));
302 }
303
304 /* Last entry in link table is used to create a circle */
305 lt = sc->sc_lt[SEC_LT_ENTRIES].sl_lt;
306 lt->shl_length = 0;
307 lt->shl_r = 0;
308 lt->shl_n = 1;
309 lt->shl_ptr = sc->sc_lt[0].sl_lt_paddr;
310
311 /* Init descriptor and link table queues pointers */
312 SEC_CNT_INIT(sc, sc_free_desc_get_cnt, SEC_DESCRIPTORS);
313 SEC_CNT_INIT(sc, sc_free_desc_put_cnt, SEC_DESCRIPTORS);
314 SEC_CNT_INIT(sc, sc_ready_desc_get_cnt, SEC_DESCRIPTORS);
315 SEC_CNT_INIT(sc, sc_ready_desc_put_cnt, SEC_DESCRIPTORS);
316 SEC_CNT_INIT(sc, sc_queued_desc_get_cnt, SEC_DESCRIPTORS);
317 SEC_CNT_INIT(sc, sc_queued_desc_put_cnt, SEC_DESCRIPTORS);
318 SEC_CNT_INIT(sc, sc_lt_alloc_cnt, SEC_LT_ENTRIES);
319 SEC_CNT_INIT(sc, sc_lt_free_cnt, SEC_LT_ENTRIES);
320
321 /* Create masks for fast checks */
322 sc->sc_int_error_mask = 0;
323 for (i = 0; i < SEC_CHANNELS; i++)
324 sc->sc_int_error_mask |= (~0ULL & SEC_INT_CH_ERR(i));
325
326 switch (sc->sc_version) {
327 case 2:
328 sc->sc_channel_idle_mask =
329 (SEC_CHAN_CSR2_FFLVL_M << SEC_CHAN_CSR2_FFLVL_S) |
330 (SEC_CHAN_CSR2_MSTATE_M << SEC_CHAN_CSR2_MSTATE_S) |
331 (SEC_CHAN_CSR2_PSTATE_M << SEC_CHAN_CSR2_PSTATE_S) |
332 (SEC_CHAN_CSR2_GSTATE_M << SEC_CHAN_CSR2_GSTATE_S);
333 break;
334 case 3:
335 sc->sc_channel_idle_mask =
336 (SEC_CHAN_CSR3_FFLVL_M << SEC_CHAN_CSR3_FFLVL_S) |
337 (SEC_CHAN_CSR3_MSTATE_M << SEC_CHAN_CSR3_MSTATE_S) |
338 (SEC_CHAN_CSR3_PSTATE_M << SEC_CHAN_CSR3_PSTATE_S) |
339 (SEC_CHAN_CSR3_GSTATE_M << SEC_CHAN_CSR3_GSTATE_S);
340 break;
341 }
342
343 /* Init hardware */
344 error = sec_init(sc);
345
346 if (error)
347 goto fail6;
348
349 return (0);
350
351 fail6:
352 sec_free_dma_mem(&(sc->sc_lt_dmem));
353 fail5:
354 sec_free_dma_mem(&(sc->sc_desc_dmem));
355 fail4:
356 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand,
357 sc->sc_sec_irid, "secondary");
358 fail3:
359 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand,
360 sc->sc_pri_irid, "primary");
361 fail2:
362 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres);
363 fail1:
364 mtx_destroy(&sc->sc_controller_lock);
365 mtx_destroy(&sc->sc_descriptors_lock);
366
367 return (ENXIO);
368 }
369
370 static int
sec_detach(device_t dev)371 sec_detach(device_t dev)
372 {
373 struct sec_softc *sc = device_get_softc(dev);
374 int i, error, timeout = SEC_TIMEOUT;
375
376 /* Prepare driver to shutdown */
377 SEC_LOCK(sc, descriptors);
378 sc->sc_shutdown = 1;
379 SEC_UNLOCK(sc, descriptors);
380
381 /* Wait until all queued processing finishes */
382 while (1) {
383 SEC_LOCK(sc, descriptors);
384 i = SEC_READY_DESC_CNT(sc) + SEC_QUEUED_DESC_CNT(sc);
385 SEC_UNLOCK(sc, descriptors);
386
387 if (i == 0)
388 break;
389
390 if (timeout < 0) {
391 device_printf(dev, "queue flush timeout!\n");
392
393 /* DMA can be still active - stop it */
394 for (i = 0; i < SEC_CHANNELS; i++)
395 sec_channel_reset(sc, i, 1);
396
397 break;
398 }
399
400 timeout -= 1000;
401 DELAY(1000);
402 }
403
404 /* Disable interrupts */
405 SEC_WRITE(sc, SEC_IER, 0);
406
407 /* Unregister from OCF */
408 crypto_unregister_all(sc->sc_cid);
409
410 /* Free DMA memory */
411 for (i = 0; i < SEC_DESCRIPTORS; i++)
412 SEC_DESC_FREE_POINTERS(&(sc->sc_desc[i]));
413
414 sec_free_dma_mem(&(sc->sc_lt_dmem));
415 sec_free_dma_mem(&(sc->sc_desc_dmem));
416
417 /* Release interrupts */
418 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand,
419 sc->sc_pri_irid, "primary");
420 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand,
421 sc->sc_sec_irid, "secondary");
422
423 /* Release memory */
424 if (sc->sc_rres) {
425 error = bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid,
426 sc->sc_rres);
427 if (error)
428 device_printf(dev, "bus_release_resource() failed for"
429 " I/O memory, error %d\n", error);
430
431 sc->sc_rres = NULL;
432 }
433
434 mtx_destroy(&sc->sc_controller_lock);
435 mtx_destroy(&sc->sc_descriptors_lock);
436
437 return (0);
438 }
439
440 static int
sec_suspend(device_t dev)441 sec_suspend(device_t dev)
442 {
443
444 return (0);
445 }
446
447 static int
sec_resume(device_t dev)448 sec_resume(device_t dev)
449 {
450
451 return (0);
452 }
453
454 static int
sec_shutdown(device_t dev)455 sec_shutdown(device_t dev)
456 {
457
458 return (0);
459 }
460
461 static int
sec_setup_intr(struct sec_softc * sc,struct resource ** ires,void ** ihand,int * irid,driver_intr_t handler,const char * iname)462 sec_setup_intr(struct sec_softc *sc, struct resource **ires, void **ihand,
463 int *irid, driver_intr_t handler, const char *iname)
464 {
465 int error;
466
467 (*ires) = bus_alloc_resource_any(sc->sc_dev, SYS_RES_IRQ, irid,
468 RF_ACTIVE);
469
470 if ((*ires) == NULL) {
471 device_printf(sc->sc_dev, "could not allocate %s IRQ\n", iname);
472 return (ENXIO);
473 }
474
475 error = bus_setup_intr(sc->sc_dev, *ires, INTR_MPSAFE | INTR_TYPE_NET,
476 NULL, handler, sc, ihand);
477
478 if (error) {
479 device_printf(sc->sc_dev, "failed to set up %s IRQ\n", iname);
480 if (bus_release_resource(sc->sc_dev, SYS_RES_IRQ, *irid, *ires))
481 device_printf(sc->sc_dev, "could not release %s IRQ\n",
482 iname);
483
484 (*ires) = NULL;
485 return (error);
486 }
487
488 return (0);
489 }
490
491 static void
sec_release_intr(struct sec_softc * sc,struct resource * ires,void * ihand,int irid,const char * iname)492 sec_release_intr(struct sec_softc *sc, struct resource *ires, void *ihand,
493 int irid, const char *iname)
494 {
495 int error;
496
497 if (ires == NULL)
498 return;
499
500 error = bus_teardown_intr(sc->sc_dev, ires, ihand);
501 if (error)
502 device_printf(sc->sc_dev, "bus_teardown_intr() failed for %s"
503 " IRQ, error %d\n", iname, error);
504
505 error = bus_release_resource(sc->sc_dev, SYS_RES_IRQ, irid, ires);
506 if (error)
507 device_printf(sc->sc_dev, "bus_release_resource() failed for %s"
508 " IRQ, error %d\n", iname, error);
509 }
510
511 static void
sec_primary_intr(void * arg)512 sec_primary_intr(void *arg)
513 {
514 struct sec_session *ses;
515 struct sec_softc *sc = arg;
516 struct sec_desc *desc;
517 struct cryptop *crp;
518 uint64_t isr;
519 uint8_t hash[HASH_MAX_LEN];
520 int i, wakeup = 0;
521
522 SEC_LOCK(sc, controller);
523
524 /* Check for errors */
525 isr = SEC_READ(sc, SEC_ISR);
526 if (isr & sc->sc_int_error_mask) {
527 /* Check each channel for error */
528 for (i = 0; i < SEC_CHANNELS; i++) {
529 if ((isr & SEC_INT_CH_ERR(i)) == 0)
530 continue;
531
532 device_printf(sc->sc_dev,
533 "I/O error on channel %i!\n", i);
534
535 /* Find and mark problematic descriptor */
536 desc = sec_find_desc(sc, SEC_READ(sc,
537 SEC_CHAN_CDPR(i)));
538
539 if (desc != NULL)
540 desc->sd_error = EIO;
541
542 /* Do partial channel reset */
543 sec_channel_reset(sc, i, 0);
544 }
545 }
546
547 /* ACK interrupt */
548 SEC_WRITE(sc, SEC_ICR, 0xFFFFFFFFFFFFFFFFULL);
549
550 SEC_UNLOCK(sc, controller);
551 SEC_LOCK(sc, descriptors);
552
553 /* Handle processed descriptors */
554 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
555
556 while (SEC_QUEUED_DESC_CNT(sc) > 0) {
557 desc = SEC_GET_QUEUED_DESC(sc);
558
559 if (desc->sd_desc->shd_done != 0xFF && desc->sd_error == 0) {
560 SEC_PUT_BACK_QUEUED_DESC(sc);
561 break;
562 }
563
564 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_PREREAD |
565 BUS_DMASYNC_PREWRITE);
566
567 crp = desc->sd_crp;
568 crp->crp_etype = desc->sd_error;
569 if (crp->crp_etype == 0) {
570 ses = crypto_get_driver_session(crp->crp_session);
571 if (ses->ss_mlen != 0) {
572 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
573 crypto_copydata(crp,
574 crp->crp_digest_start,
575 ses->ss_mlen, hash);
576 if (timingsafe_bcmp(
577 desc->sd_desc->shd_digest,
578 hash, ses->ss_mlen) != 0)
579 crp->crp_etype = EBADMSG;
580 } else
581 crypto_copyback(crp,
582 crp->crp_digest_start,
583 ses->ss_mlen,
584 desc->sd_desc->shd_digest);
585 }
586 }
587 crypto_done(desc->sd_crp);
588
589 SEC_DESC_FREE_POINTERS(desc);
590 SEC_DESC_FREE_LT(sc, desc);
591 SEC_DESC_QUEUED2FREE(sc);
592 }
593
594 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
595
596 if (!sc->sc_shutdown) {
597 wakeup = sc->sc_blocked;
598 sc->sc_blocked = 0;
599 }
600
601 SEC_UNLOCK(sc, descriptors);
602
603 /* Enqueue ready descriptors in hardware */
604 sec_enqueue(sc);
605
606 if (wakeup)
607 crypto_unblock(sc->sc_cid, wakeup);
608 }
609
610 static void
sec_secondary_intr(void * arg)611 sec_secondary_intr(void *arg)
612 {
613 struct sec_softc *sc = arg;
614
615 device_printf(sc->sc_dev, "spurious secondary interrupt!\n");
616 sec_primary_intr(arg);
617 }
618
619 static int
sec_controller_reset(struct sec_softc * sc)620 sec_controller_reset(struct sec_softc *sc)
621 {
622 int timeout = SEC_TIMEOUT;
623
624 /* Reset Controller */
625 SEC_WRITE(sc, SEC_MCR, SEC_MCR_SWR);
626
627 while (SEC_READ(sc, SEC_MCR) & SEC_MCR_SWR) {
628 DELAY(1000);
629 timeout -= 1000;
630
631 if (timeout < 0) {
632 device_printf(sc->sc_dev, "timeout while waiting for "
633 "device reset!\n");
634 return (ETIMEDOUT);
635 }
636 }
637
638 return (0);
639 }
640
641 static int
sec_channel_reset(struct sec_softc * sc,int channel,int full)642 sec_channel_reset(struct sec_softc *sc, int channel, int full)
643 {
644 int timeout = SEC_TIMEOUT;
645 uint64_t bit = (full) ? SEC_CHAN_CCR_R : SEC_CHAN_CCR_CON;
646 uint64_t reg;
647
648 /* Reset Channel */
649 reg = SEC_READ(sc, SEC_CHAN_CCR(channel));
650 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg | bit);
651
652 while (SEC_READ(sc, SEC_CHAN_CCR(channel)) & bit) {
653 DELAY(1000);
654 timeout -= 1000;
655
656 if (timeout < 0) {
657 device_printf(sc->sc_dev, "timeout while waiting for "
658 "channel reset!\n");
659 return (ETIMEDOUT);
660 }
661 }
662
663 if (full) {
664 reg = SEC_CHAN_CCR_CDIE | SEC_CHAN_CCR_NT | SEC_CHAN_CCR_BS;
665
666 switch(sc->sc_version) {
667 case 2:
668 reg |= SEC_CHAN_CCR_CDWE;
669 break;
670 case 3:
671 reg |= SEC_CHAN_CCR_AWSE | SEC_CHAN_CCR_WGN;
672 break;
673 }
674
675 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg);
676 }
677
678 return (0);
679 }
680
681 static int
sec_init(struct sec_softc * sc)682 sec_init(struct sec_softc *sc)
683 {
684 uint64_t reg;
685 int error, i;
686
687 /* Reset controller twice to clear all pending interrupts */
688 error = sec_controller_reset(sc);
689 if (error)
690 return (error);
691
692 error = sec_controller_reset(sc);
693 if (error)
694 return (error);
695
696 /* Reset channels */
697 for (i = 0; i < SEC_CHANNELS; i++) {
698 error = sec_channel_reset(sc, i, 1);
699 if (error)
700 return (error);
701 }
702
703 /* Enable Interrupts */
704 reg = SEC_INT_ITO;
705 for (i = 0; i < SEC_CHANNELS; i++)
706 reg |= SEC_INT_CH_DN(i) | SEC_INT_CH_ERR(i);
707
708 SEC_WRITE(sc, SEC_IER, reg);
709
710 return (error);
711 }
712
713 static void
sec_alloc_dma_mem_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)714 sec_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
715 {
716 struct sec_dma_mem *dma_mem = arg;
717
718 if (error)
719 return;
720
721 KASSERT(nseg == 1, ("Wrong number of segments, should be 1"));
722 dma_mem->dma_paddr = segs->ds_addr;
723 }
724
725 static void
sec_dma_map_desc_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)726 sec_dma_map_desc_cb(void *arg, bus_dma_segment_t *segs, int nseg,
727 int error)
728 {
729 struct sec_desc_map_info *sdmi = arg;
730 struct sec_softc *sc = sdmi->sdmi_sc;
731 struct sec_lt *lt = NULL;
732 bus_addr_t addr;
733 bus_size_t size;
734 int i;
735
736 SEC_LOCK_ASSERT(sc, descriptors);
737
738 if (error)
739 return;
740
741 for (i = 0; i < nseg; i++) {
742 addr = segs[i].ds_addr;
743 size = segs[i].ds_len;
744
745 /* Skip requested offset */
746 if (sdmi->sdmi_offset >= size) {
747 sdmi->sdmi_offset -= size;
748 continue;
749 }
750
751 addr += sdmi->sdmi_offset;
752 size -= sdmi->sdmi_offset;
753 sdmi->sdmi_offset = 0;
754
755 /* Do not link more than requested */
756 if (sdmi->sdmi_size < size)
757 size = sdmi->sdmi_size;
758
759 lt = SEC_ALLOC_LT_ENTRY(sc);
760 lt->sl_lt->shl_length = size;
761 lt->sl_lt->shl_r = 0;
762 lt->sl_lt->shl_n = 0;
763 lt->sl_lt->shl_ptr = addr;
764
765 if (sdmi->sdmi_lt_first == NULL)
766 sdmi->sdmi_lt_first = lt;
767
768 sdmi->sdmi_lt_used += 1;
769
770 if ((sdmi->sdmi_size -= size) == 0)
771 break;
772 }
773
774 sdmi->sdmi_lt_last = lt;
775 }
776
777 static int
sec_alloc_dma_mem(struct sec_softc * sc,struct sec_dma_mem * dma_mem,bus_size_t size)778 sec_alloc_dma_mem(struct sec_softc *sc, struct sec_dma_mem *dma_mem,
779 bus_size_t size)
780 {
781 int error;
782
783 if (dma_mem->dma_vaddr != NULL)
784 return (EBUSY);
785
786 error = bus_dma_tag_create(NULL, /* parent */
787 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */
788 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
789 BUS_SPACE_MAXADDR, /* highaddr */
790 NULL, NULL, /* filtfunc, filtfuncarg */
791 size, 1, /* maxsize, nsegments */
792 size, 0, /* maxsegsz, flags */
793 NULL, NULL, /* lockfunc, lockfuncarg */
794 &(dma_mem->dma_tag)); /* dmat */
795
796 if (error) {
797 device_printf(sc->sc_dev, "failed to allocate busdma tag, error"
798 " %i!\n", error);
799 goto err1;
800 }
801
802 error = bus_dmamem_alloc(dma_mem->dma_tag, &(dma_mem->dma_vaddr),
803 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &(dma_mem->dma_map));
804
805 if (error) {
806 device_printf(sc->sc_dev, "failed to allocate DMA safe"
807 " memory, error %i!\n", error);
808 goto err2;
809 }
810
811 error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map,
812 dma_mem->dma_vaddr, size, sec_alloc_dma_mem_cb, dma_mem,
813 BUS_DMA_NOWAIT);
814
815 if (error) {
816 device_printf(sc->sc_dev, "cannot get address of the DMA"
817 " memory, error %i\n", error);
818 goto err3;
819 }
820
821 dma_mem->dma_is_map = 0;
822 return (0);
823
824 err3:
825 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, dma_mem->dma_map);
826 err2:
827 bus_dma_tag_destroy(dma_mem->dma_tag);
828 err1:
829 dma_mem->dma_vaddr = NULL;
830 return(error);
831 }
832
833 static int
sec_desc_map_dma(struct sec_softc * sc,struct sec_dma_mem * dma_mem,struct cryptop * crp,bus_size_t size,struct sec_desc_map_info * sdmi)834 sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem,
835 struct cryptop *crp, bus_size_t size, struct sec_desc_map_info *sdmi)
836 {
837 int error;
838
839 if (dma_mem->dma_vaddr != NULL)
840 return (EBUSY);
841
842 switch (crp->crp_buf.cb_type) {
843 case CRYPTO_BUF_CONTIG:
844 break;
845 case CRYPTO_BUF_UIO:
846 size = SEC_FREE_LT_CNT(sc) * SEC_MAX_DMA_BLOCK_SIZE;
847 break;
848 case CRYPTO_BUF_MBUF:
849 size = m_length(crp->crp_buf.cb_mbuf, NULL);
850 break;
851 case CRYPTO_BUF_SINGLE_MBUF:
852 size = crp->crp_buf.cb_mbuf->m_len;
853 break;
854 case CRYPTO_BUF_VMPAGE:
855 size = PAGE_SIZE - crp->crp_buf.cb_vm_page_offset;
856 break;
857 default:
858 return (EINVAL);
859 }
860
861 error = bus_dma_tag_create(NULL, /* parent */
862 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */
863 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
864 BUS_SPACE_MAXADDR, /* highaddr */
865 NULL, NULL, /* filtfunc, filtfuncarg */
866 size, /* maxsize */
867 SEC_FREE_LT_CNT(sc), /* nsegments */
868 SEC_MAX_DMA_BLOCK_SIZE, 0, /* maxsegsz, flags */
869 NULL, NULL, /* lockfunc, lockfuncarg */
870 &(dma_mem->dma_tag)); /* dmat */
871
872 if (error) {
873 device_printf(sc->sc_dev, "failed to allocate busdma tag, error"
874 " %i!\n", error);
875 dma_mem->dma_vaddr = NULL;
876 return (error);
877 }
878
879 error = bus_dmamap_create(dma_mem->dma_tag, 0, &(dma_mem->dma_map));
880
881 if (error) {
882 device_printf(sc->sc_dev, "failed to create DMA map, error %i!"
883 "\n", error);
884 bus_dma_tag_destroy(dma_mem->dma_tag);
885 return (error);
886 }
887
888 error = bus_dmamap_load_crp(dma_mem->dma_tag, dma_mem->dma_map, crp,
889 sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT);
890
891 if (error) {
892 device_printf(sc->sc_dev, "cannot get address of the DMA"
893 " memory, error %i!\n", error);
894 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map);
895 bus_dma_tag_destroy(dma_mem->dma_tag);
896 return (error);
897 }
898
899 dma_mem->dma_is_map = 1;
900 dma_mem->dma_vaddr = crp;
901
902 return (0);
903 }
904
905 static void
sec_free_dma_mem(struct sec_dma_mem * dma_mem)906 sec_free_dma_mem(struct sec_dma_mem *dma_mem)
907 {
908
909 /* Check for double free */
910 if (dma_mem->dma_vaddr == NULL)
911 return;
912
913 bus_dmamap_unload(dma_mem->dma_tag, dma_mem->dma_map);
914
915 if (dma_mem->dma_is_map)
916 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map);
917 else
918 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr,
919 dma_mem->dma_map);
920
921 bus_dma_tag_destroy(dma_mem->dma_tag);
922 dma_mem->dma_vaddr = NULL;
923 }
924
925 static int
sec_eu_channel(struct sec_softc * sc,int eu)926 sec_eu_channel(struct sec_softc *sc, int eu)
927 {
928 uint64_t reg;
929 int channel = 0;
930
931 SEC_LOCK_ASSERT(sc, controller);
932
933 reg = SEC_READ(sc, SEC_EUASR);
934
935 switch (eu) {
936 case SEC_EU_AFEU:
937 channel = SEC_EUASR_AFEU(reg);
938 break;
939 case SEC_EU_DEU:
940 channel = SEC_EUASR_DEU(reg);
941 break;
942 case SEC_EU_MDEU_A:
943 case SEC_EU_MDEU_B:
944 channel = SEC_EUASR_MDEU(reg);
945 break;
946 case SEC_EU_RNGU:
947 channel = SEC_EUASR_RNGU(reg);
948 break;
949 case SEC_EU_PKEU:
950 channel = SEC_EUASR_PKEU(reg);
951 break;
952 case SEC_EU_AESU:
953 channel = SEC_EUASR_AESU(reg);
954 break;
955 case SEC_EU_KEU:
956 channel = SEC_EUASR_KEU(reg);
957 break;
958 case SEC_EU_CRCU:
959 channel = SEC_EUASR_CRCU(reg);
960 break;
961 }
962
963 return (channel - 1);
964 }
965
966 static int
sec_enqueue_desc(struct sec_softc * sc,struct sec_desc * desc,int channel)967 sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, int channel)
968 {
969 u_int fflvl = SEC_MAX_FIFO_LEVEL;
970 uint64_t reg;
971 int i;
972
973 SEC_LOCK_ASSERT(sc, controller);
974
975 /* Find free channel if have not got one */
976 if (channel < 0) {
977 for (i = 0; i < SEC_CHANNELS; i++) {
978 reg = SEC_READ(sc, SEC_CHAN_CSR(channel));
979
980 if ((reg & sc->sc_channel_idle_mask) == 0) {
981 channel = i;
982 break;
983 }
984 }
985 }
986
987 /* There is no free channel */
988 if (channel < 0)
989 return (-1);
990
991 /* Check FIFO level on selected channel */
992 reg = SEC_READ(sc, SEC_CHAN_CSR(channel));
993
994 switch(sc->sc_version) {
995 case 2:
996 fflvl = (reg >> SEC_CHAN_CSR2_FFLVL_S) & SEC_CHAN_CSR2_FFLVL_M;
997 break;
998 case 3:
999 fflvl = (reg >> SEC_CHAN_CSR3_FFLVL_S) & SEC_CHAN_CSR3_FFLVL_M;
1000 break;
1001 }
1002
1003 if (fflvl >= SEC_MAX_FIFO_LEVEL)
1004 return (-1);
1005
1006 /* Enqueue descriptor in channel */
1007 SEC_WRITE(sc, SEC_CHAN_FF(channel), desc->sd_desc_paddr);
1008
1009 return (channel);
1010 }
1011
1012 static void
sec_enqueue(struct sec_softc * sc)1013 sec_enqueue(struct sec_softc *sc)
1014 {
1015 struct sec_desc *desc;
1016 int ch0, ch1;
1017
1018 SEC_LOCK(sc, descriptors);
1019 SEC_LOCK(sc, controller);
1020
1021 while (SEC_READY_DESC_CNT(sc) > 0) {
1022 desc = SEC_GET_READY_DESC(sc);
1023
1024 ch0 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel0);
1025 ch1 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel1);
1026
1027 /*
1028 * Both EU are used by the same channel.
1029 * Enqueue descriptor in channel used by busy EUs.
1030 */
1031 if (ch0 >= 0 && ch0 == ch1) {
1032 if (sec_enqueue_desc(sc, desc, ch0) >= 0) {
1033 SEC_DESC_READY2QUEUED(sc);
1034 continue;
1035 }
1036 }
1037
1038 /*
1039 * Only one EU is free.
1040 * Enqueue descriptor in channel used by busy EU.
1041 */
1042 if ((ch0 >= 0 && ch1 < 0) || (ch1 >= 0 && ch0 < 0)) {
1043 if (sec_enqueue_desc(sc, desc, (ch0 >= 0) ? ch0 : ch1)
1044 >= 0) {
1045 SEC_DESC_READY2QUEUED(sc);
1046 continue;
1047 }
1048 }
1049
1050 /*
1051 * Both EU are free.
1052 * Enqueue descriptor in first free channel.
1053 */
1054 if (ch0 < 0 && ch1 < 0) {
1055 if (sec_enqueue_desc(sc, desc, -1) >= 0) {
1056 SEC_DESC_READY2QUEUED(sc);
1057 continue;
1058 }
1059 }
1060
1061 /* Current descriptor can not be queued at the moment */
1062 SEC_PUT_BACK_READY_DESC(sc);
1063 break;
1064 }
1065
1066 SEC_UNLOCK(sc, controller);
1067 SEC_UNLOCK(sc, descriptors);
1068 }
1069
1070 static struct sec_desc *
sec_find_desc(struct sec_softc * sc,bus_addr_t paddr)1071 sec_find_desc(struct sec_softc *sc, bus_addr_t paddr)
1072 {
1073 struct sec_desc *desc = NULL;
1074 int i;
1075
1076 SEC_LOCK_ASSERT(sc, descriptors);
1077
1078 for (i = 0; i < SEC_CHANNELS; i++) {
1079 if (sc->sc_desc[i].sd_desc_paddr == paddr) {
1080 desc = &(sc->sc_desc[i]);
1081 break;
1082 }
1083 }
1084
1085 return (desc);
1086 }
1087
1088 static int
sec_make_pointer_direct(struct sec_softc * sc,struct sec_desc * desc,u_int n,bus_addr_t data,bus_size_t dsize)1089 sec_make_pointer_direct(struct sec_softc *sc, struct sec_desc *desc, u_int n,
1090 bus_addr_t data, bus_size_t dsize)
1091 {
1092 struct sec_hw_desc_ptr *ptr;
1093
1094 SEC_LOCK_ASSERT(sc, descriptors);
1095
1096 ptr = &(desc->sd_desc->shd_pointer[n]);
1097 ptr->shdp_length = dsize;
1098 ptr->shdp_extent = 0;
1099 ptr->shdp_j = 0;
1100 ptr->shdp_ptr = data;
1101
1102 return (0);
1103 }
1104
1105 static int
sec_make_pointer(struct sec_softc * sc,struct sec_desc * desc,u_int n,struct cryptop * crp,bus_size_t doffset,bus_size_t dsize)1106 sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc,
1107 u_int n, struct cryptop *crp, bus_size_t doffset, bus_size_t dsize)
1108 {
1109 struct sec_desc_map_info sdmi = { sc, dsize, doffset, NULL, NULL, 0 };
1110 struct sec_hw_desc_ptr *ptr;
1111 int error;
1112
1113 SEC_LOCK_ASSERT(sc, descriptors);
1114
1115 error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), crp, dsize,
1116 &sdmi);
1117
1118 if (error)
1119 return (error);
1120
1121 sdmi.sdmi_lt_last->sl_lt->shl_r = 1;
1122 desc->sd_lt_used += sdmi.sdmi_lt_used;
1123
1124 ptr = &(desc->sd_desc->shd_pointer[n]);
1125 ptr->shdp_length = dsize;
1126 ptr->shdp_extent = 0;
1127 ptr->shdp_j = 1;
1128 ptr->shdp_ptr = sdmi.sdmi_lt_first->sl_lt_paddr;
1129
1130 return (0);
1131 }
1132
1133 static bool
sec_cipher_supported(const struct crypto_session_params * csp)1134 sec_cipher_supported(const struct crypto_session_params *csp)
1135 {
1136
1137 switch (csp->csp_cipher_alg) {
1138 case CRYPTO_AES_CBC:
1139 /* AESU */
1140 if (csp->csp_ivlen != AES_BLOCK_LEN)
1141 return (false);
1142 break;
1143 default:
1144 return (false);
1145 }
1146
1147 if (csp->csp_cipher_klen == 0 || csp->csp_cipher_klen > SEC_MAX_KEY_LEN)
1148 return (false);
1149
1150 return (true);
1151 }
1152
1153 static bool
sec_auth_supported(struct sec_softc * sc,const struct crypto_session_params * csp)1154 sec_auth_supported(struct sec_softc *sc,
1155 const struct crypto_session_params *csp)
1156 {
1157
1158 switch (csp->csp_auth_alg) {
1159 case CRYPTO_SHA2_384_HMAC:
1160 case CRYPTO_SHA2_512_HMAC:
1161 if (sc->sc_version < 3)
1162 return (false);
1163 /* FALLTHROUGH */
1164 case CRYPTO_SHA1_HMAC:
1165 case CRYPTO_SHA2_256_HMAC:
1166 if (csp->csp_auth_klen > SEC_MAX_KEY_LEN)
1167 return (false);
1168 break;
1169 case CRYPTO_SHA1:
1170 break;
1171 default:
1172 return (false);
1173 }
1174 return (true);
1175 }
1176
1177 static int
sec_probesession(device_t dev,const struct crypto_session_params * csp)1178 sec_probesession(device_t dev, const struct crypto_session_params *csp)
1179 {
1180 struct sec_softc *sc = device_get_softc(dev);
1181
1182 if (csp->csp_flags != 0)
1183 return (EINVAL);
1184 switch (csp->csp_mode) {
1185 case CSP_MODE_DIGEST:
1186 if (!sec_auth_supported(sc, csp))
1187 return (EINVAL);
1188 break;
1189 case CSP_MODE_CIPHER:
1190 if (!sec_cipher_supported(csp))
1191 return (EINVAL);
1192 break;
1193 case CSP_MODE_ETA:
1194 if (!sec_auth_supported(sc, csp) || !sec_cipher_supported(csp))
1195 return (EINVAL);
1196 break;
1197 default:
1198 return (EINVAL);
1199 }
1200 return (CRYPTODEV_PROBE_HARDWARE);
1201 }
1202
1203 static int
sec_newsession(device_t dev,crypto_session_t cses,const struct crypto_session_params * csp)1204 sec_newsession(device_t dev, crypto_session_t cses,
1205 const struct crypto_session_params *csp)
1206 {
1207 struct sec_eu_methods *eu = sec_eus;
1208 struct sec_session *ses;
1209
1210 ses = crypto_get_driver_session(cses);
1211
1212 /* Find EU for this session */
1213 while (eu->sem_make_desc != NULL) {
1214 if (eu->sem_newsession(csp))
1215 break;
1216 eu++;
1217 }
1218 KASSERT(eu->sem_make_desc != NULL, ("failed to find eu for session"));
1219
1220 /* Save cipher key */
1221 if (csp->csp_cipher_key != NULL)
1222 memcpy(ses->ss_key, csp->csp_cipher_key, csp->csp_cipher_klen);
1223
1224 /* Save digest key */
1225 if (csp->csp_auth_key != NULL)
1226 memcpy(ses->ss_mkey, csp->csp_auth_key, csp->csp_auth_klen);
1227
1228 if (csp->csp_auth_alg != 0) {
1229 if (csp->csp_auth_mlen == 0)
1230 ses->ss_mlen = crypto_auth_hash(csp)->hashsize;
1231 else
1232 ses->ss_mlen = csp->csp_auth_mlen;
1233 }
1234
1235 return (0);
1236 }
1237
1238 static int
sec_process(device_t dev,struct cryptop * crp,int hint)1239 sec_process(device_t dev, struct cryptop *crp, int hint)
1240 {
1241 struct sec_softc *sc = device_get_softc(dev);
1242 struct sec_desc *desc = NULL;
1243 const struct crypto_session_params *csp;
1244 struct sec_session *ses;
1245 int error = 0;
1246
1247 ses = crypto_get_driver_session(crp->crp_session);
1248 csp = crypto_get_params(crp->crp_session);
1249
1250 /* Check for input length */
1251 if (crypto_buffer_len(&crp->crp_buf) > SEC_MAX_DMA_BLOCK_SIZE) {
1252 crp->crp_etype = E2BIG;
1253 crypto_done(crp);
1254 return (0);
1255 }
1256
1257 SEC_LOCK(sc, descriptors);
1258 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1259
1260 /* Block driver if there is no free descriptors or we are going down */
1261 if (SEC_FREE_DESC_CNT(sc) == 0 || sc->sc_shutdown) {
1262 sc->sc_blocked |= CRYPTO_SYMQ;
1263 SEC_UNLOCK(sc, descriptors);
1264 return (ERESTART);
1265 }
1266
1267 /* Prepare descriptor */
1268 desc = SEC_GET_FREE_DESC(sc);
1269 desc->sd_lt_used = 0;
1270 desc->sd_error = 0;
1271 desc->sd_crp = crp;
1272
1273 if (csp->csp_cipher_alg != 0)
1274 crypto_read_iv(crp, desc->sd_desc->shd_iv);
1275
1276 if (crp->crp_cipher_key != NULL)
1277 memcpy(ses->ss_key, crp->crp_cipher_key, csp->csp_cipher_klen);
1278
1279 if (crp->crp_auth_key != NULL)
1280 memcpy(ses->ss_mkey, crp->crp_auth_key, csp->csp_auth_klen);
1281
1282 memcpy(desc->sd_desc->shd_key, ses->ss_key, csp->csp_cipher_klen);
1283 memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, csp->csp_auth_klen);
1284
1285 error = ses->ss_eu->sem_make_desc(sc, csp, desc, crp);
1286
1287 if (error) {
1288 SEC_DESC_FREE_POINTERS(desc);
1289 SEC_DESC_PUT_BACK_LT(sc, desc);
1290 SEC_PUT_BACK_FREE_DESC(sc);
1291 SEC_UNLOCK(sc, descriptors);
1292 crp->crp_etype = error;
1293 crypto_done(crp);
1294 return (0);
1295 }
1296
1297 /*
1298 * Skip DONE interrupt if this is not last request in burst, but only
1299 * if we are running on SEC 3.X. On SEC 2.X we have to enable DONE
1300 * signaling on each descriptor.
1301 */
1302 if ((hint & CRYPTO_HINT_MORE) && sc->sc_version == 3)
1303 desc->sd_desc->shd_dn = 0;
1304 else
1305 desc->sd_desc->shd_dn = 1;
1306
1307 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1308 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_POSTREAD |
1309 BUS_DMASYNC_POSTWRITE);
1310 SEC_DESC_FREE2READY(sc);
1311 SEC_UNLOCK(sc, descriptors);
1312
1313 /* Enqueue ready descriptors in hardware */
1314 sec_enqueue(sc);
1315
1316 return (0);
1317 }
1318
1319 static int
sec_build_common_ns_desc(struct sec_softc * sc,struct sec_desc * desc,const struct crypto_session_params * csp,struct cryptop * crp)1320 sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc,
1321 const struct crypto_session_params *csp, struct cryptop *crp)
1322 {
1323 struct sec_hw_desc *hd = desc->sd_desc;
1324 int error;
1325
1326 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP;
1327 hd->shd_eu_sel1 = SEC_EU_NONE;
1328 hd->shd_mode1 = 0;
1329
1330 /* Pointer 0: NULL */
1331 error = sec_make_pointer_direct(sc, desc, 0, 0, 0);
1332 if (error)
1333 return (error);
1334
1335 /* Pointer 1: IV IN */
1336 error = sec_make_pointer_direct(sc, desc, 1, desc->sd_desc_paddr +
1337 offsetof(struct sec_hw_desc, shd_iv), csp->csp_ivlen);
1338 if (error)
1339 return (error);
1340
1341 /* Pointer 2: Cipher Key */
1342 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr +
1343 offsetof(struct sec_hw_desc, shd_key), csp->csp_cipher_klen);
1344 if (error)
1345 return (error);
1346
1347 /* Pointer 3: Data IN */
1348 error = sec_make_pointer(sc, desc, 3, crp, crp->crp_payload_start,
1349 crp->crp_payload_length);
1350 if (error)
1351 return (error);
1352
1353 /* Pointer 4: Data OUT */
1354 error = sec_make_pointer(sc, desc, 4, crp, crp->crp_payload_start,
1355 crp->crp_payload_length);
1356 if (error)
1357 return (error);
1358
1359 /* Pointer 5: IV OUT (Not used: NULL) */
1360 error = sec_make_pointer_direct(sc, desc, 5, 0, 0);
1361 if (error)
1362 return (error);
1363
1364 /* Pointer 6: NULL */
1365 error = sec_make_pointer_direct(sc, desc, 6, 0, 0);
1366
1367 return (error);
1368 }
1369
1370 static int
sec_build_common_s_desc(struct sec_softc * sc,struct sec_desc * desc,const struct crypto_session_params * csp,struct cryptop * crp)1371 sec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc,
1372 const struct crypto_session_params *csp, struct cryptop *crp)
1373 {
1374 struct sec_hw_desc *hd = desc->sd_desc;
1375 u_int eu, mode, hashlen;
1376 int error;
1377
1378 error = sec_mdeu_config(csp, &eu, &mode, &hashlen);
1379 if (error)
1380 return (error);
1381
1382 hd->shd_desc_type = SEC_DT_HMAC_SNOOP;
1383 hd->shd_eu_sel1 = eu;
1384 hd->shd_mode1 = mode;
1385
1386 /* Pointer 0: HMAC Key */
1387 error = sec_make_pointer_direct(sc, desc, 0, desc->sd_desc_paddr +
1388 offsetof(struct sec_hw_desc, shd_mkey), csp->csp_auth_klen);
1389 if (error)
1390 return (error);
1391
1392 /* Pointer 1: HMAC-Only Data IN */
1393 error = sec_make_pointer(sc, desc, 1, crp, crp->crp_aad_start,
1394 crp->crp_aad_length);
1395 if (error)
1396 return (error);
1397
1398 /* Pointer 2: Cipher Key */
1399 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr +
1400 offsetof(struct sec_hw_desc, shd_key), csp->csp_cipher_klen);
1401 if (error)
1402 return (error);
1403
1404 /* Pointer 3: IV IN */
1405 error = sec_make_pointer_direct(sc, desc, 3, desc->sd_desc_paddr +
1406 offsetof(struct sec_hw_desc, shd_iv), csp->csp_ivlen);
1407 if (error)
1408 return (error);
1409
1410 /* Pointer 4: Data IN */
1411 error = sec_make_pointer(sc, desc, 4, crp, crp->crp_payload_start,
1412 crp->crp_payload_length);
1413 if (error)
1414 return (error);
1415
1416 /* Pointer 5: Data OUT */
1417 error = sec_make_pointer(sc, desc, 5, crp, crp->crp_payload_start,
1418 crp->crp_payload_length);
1419 if (error)
1420 return (error);
1421
1422 /* Pointer 6: HMAC OUT */
1423 error = sec_make_pointer_direct(sc, desc, 6, desc->sd_desc_paddr +
1424 offsetof(struct sec_hw_desc, shd_digest), hashlen);
1425
1426 return (error);
1427 }
1428
1429 /* AESU */
1430
1431 static bool
sec_aesu_newsession(const struct crypto_session_params * csp)1432 sec_aesu_newsession(const struct crypto_session_params *csp)
1433 {
1434
1435 return (csp->csp_cipher_alg == CRYPTO_AES_CBC);
1436 }
1437
1438 static int
sec_aesu_make_desc(struct sec_softc * sc,const struct crypto_session_params * csp,struct sec_desc * desc,struct cryptop * crp)1439 sec_aesu_make_desc(struct sec_softc *sc,
1440 const struct crypto_session_params *csp, struct sec_desc *desc,
1441 struct cryptop *crp)
1442 {
1443 struct sec_hw_desc *hd = desc->sd_desc;
1444 int error;
1445
1446 hd->shd_eu_sel0 = SEC_EU_AESU;
1447 hd->shd_mode0 = SEC_AESU_MODE_CBC;
1448
1449 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
1450 hd->shd_mode0 |= SEC_AESU_MODE_ED;
1451 hd->shd_dir = 0;
1452 } else
1453 hd->shd_dir = 1;
1454
1455 if (csp->csp_mode == CSP_MODE_ETA)
1456 error = sec_build_common_s_desc(sc, desc, csp, crp);
1457 else
1458 error = sec_build_common_ns_desc(sc, desc, csp, crp);
1459
1460 return (error);
1461 }
1462
1463 /* MDEU */
1464
1465 static bool
sec_mdeu_can_handle(u_int alg)1466 sec_mdeu_can_handle(u_int alg)
1467 {
1468 switch (alg) {
1469 case CRYPTO_SHA1:
1470 case CRYPTO_SHA1_HMAC:
1471 case CRYPTO_SHA2_256_HMAC:
1472 case CRYPTO_SHA2_384_HMAC:
1473 case CRYPTO_SHA2_512_HMAC:
1474 return (true);
1475 default:
1476 return (false);
1477 }
1478 }
1479
1480 static int
sec_mdeu_config(const struct crypto_session_params * csp,u_int * eu,u_int * mode,u_int * hashlen)1481 sec_mdeu_config(const struct crypto_session_params *csp, u_int *eu, u_int *mode,
1482 u_int *hashlen)
1483 {
1484
1485 *mode = SEC_MDEU_MODE_PD | SEC_MDEU_MODE_INIT;
1486 *eu = SEC_EU_NONE;
1487
1488 switch (csp->csp_auth_alg) {
1489 case CRYPTO_SHA1_HMAC:
1490 *mode |= SEC_MDEU_MODE_HMAC;
1491 /* FALLTHROUGH */
1492 case CRYPTO_SHA1:
1493 *eu = SEC_EU_MDEU_A;
1494 *mode |= SEC_MDEU_MODE_SHA1;
1495 *hashlen = SHA1_HASH_LEN;
1496 break;
1497 case CRYPTO_SHA2_256_HMAC:
1498 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA256;
1499 *eu = SEC_EU_MDEU_A;
1500 break;
1501 case CRYPTO_SHA2_384_HMAC:
1502 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA384;
1503 *eu = SEC_EU_MDEU_B;
1504 break;
1505 case CRYPTO_SHA2_512_HMAC:
1506 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA512;
1507 *eu = SEC_EU_MDEU_B;
1508 break;
1509 default:
1510 return (EINVAL);
1511 }
1512
1513 if (*mode & SEC_MDEU_MODE_HMAC)
1514 *hashlen = SEC_HMAC_HASH_LEN;
1515
1516 return (0);
1517 }
1518
1519 static bool
sec_mdeu_newsession(const struct crypto_session_params * csp)1520 sec_mdeu_newsession(const struct crypto_session_params *csp)
1521 {
1522
1523 return (sec_mdeu_can_handle(csp->csp_auth_alg));
1524 }
1525
1526 static int
sec_mdeu_make_desc(struct sec_softc * sc,const struct crypto_session_params * csp,struct sec_desc * desc,struct cryptop * crp)1527 sec_mdeu_make_desc(struct sec_softc *sc,
1528 const struct crypto_session_params *csp,
1529 struct sec_desc *desc, struct cryptop *crp)
1530 {
1531 struct sec_hw_desc *hd = desc->sd_desc;
1532 u_int eu, mode, hashlen;
1533 int error;
1534
1535 error = sec_mdeu_config(csp, &eu, &mode, &hashlen);
1536 if (error)
1537 return (error);
1538
1539 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP;
1540 hd->shd_eu_sel0 = eu;
1541 hd->shd_mode0 = mode;
1542 hd->shd_eu_sel1 = SEC_EU_NONE;
1543 hd->shd_mode1 = 0;
1544
1545 /* Pointer 0: NULL */
1546 error = sec_make_pointer_direct(sc, desc, 0, 0, 0);
1547 if (error)
1548 return (error);
1549
1550 /* Pointer 1: Context In (Not used: NULL) */
1551 error = sec_make_pointer_direct(sc, desc, 1, 0, 0);
1552 if (error)
1553 return (error);
1554
1555 /* Pointer 2: HMAC Key (or NULL, depending on digest type) */
1556 if (hd->shd_mode0 & SEC_MDEU_MODE_HMAC)
1557 error = sec_make_pointer_direct(sc, desc, 2,
1558 desc->sd_desc_paddr + offsetof(struct sec_hw_desc,
1559 shd_mkey), csp->csp_auth_klen);
1560 else
1561 error = sec_make_pointer_direct(sc, desc, 2, 0, 0);
1562
1563 if (error)
1564 return (error);
1565
1566 /* Pointer 3: Input Data */
1567 error = sec_make_pointer(sc, desc, 3, crp, crp->crp_payload_start,
1568 crp->crp_payload_length);
1569 if (error)
1570 return (error);
1571
1572 /* Pointer 4: NULL */
1573 error = sec_make_pointer_direct(sc, desc, 4, 0, 0);
1574 if (error)
1575 return (error);
1576
1577 /* Pointer 5: Hash out */
1578 error = sec_make_pointer_direct(sc, desc, 5, desc->sd_desc_paddr +
1579 offsetof(struct sec_hw_desc, shd_digest), hashlen);
1580 if (error)
1581 return (error);
1582
1583 /* Pointer 6: NULL */
1584 error = sec_make_pointer_direct(sc, desc, 6, 0, 0);
1585
1586 return (0);
1587 }
1588