1 /* $OpenBSD: safe.c,v 1.14 2004/05/07 14:42:26 millert Exp $ */
2
3 /*-
4 * Copyright (c) 2003 Sam Leffler, Errno Consulting
5 * Copyright (c) 2003 Global Technology Associates, Inc.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 * $FreeBSD: /repoman/r/ncvs/src/sys/dev/safe/safe.c,v 1.1 2003/07/21 21:46:07 sam Exp $
30 */
31
32 #include <sys/cdefs.h>
33
34 /*
35 * SafeNet SafeXcel-1141 hardware crypto accelerator
36 */
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/proc.h>
40 #include <sys/errno.h>
41 #include <sys/malloc.h>
42 #include <sys/kernel.h>
43 #include <sys/mbuf.h>
44 #include <sys/device.h>
45 #include <sys/timeout.h>
46
47 #include <machine/bus.h>
48
49 #include <syskern/md5.h>
50 #include <crypto/sha1.h>
51 #include <crypto/cryptodev.h>
52 #include <crypto/cryptosoft.h>
53 #include <dev/rndvar.h>
54
55 #include <dev/pci/pcivar.h>
56 #include <dev/pci/pcireg.h>
57 #include <dev/pci/pcidevs.h>
58
59 #include <dev/pci/safereg.h>
60 #include <dev/pci/safevar.h>
61
62 #ifndef bswap32
63 #define bswap32 NTOHL
64 #endif
65
66 #define KASSERT_X(x,y)
67
68 /*
69 * Prototypes and count for the pci_device structure
70 */
71 int safe_probe(struct device *, void *, void *);
72 void safe_attach(struct device *, struct device *, void *);
73
74 struct cfattach safe_ca = {
75 sizeof(struct safe_softc), safe_probe, safe_attach
76 };
77
78 struct cfdriver safe_cd = {
79 0, "safe", DV_DULL
80 };
81
82 int safe_intr(void *);
83 int safe_newsession(u_int32_t *, struct cryptoini *);
84 int safe_freesession(u_int64_t);
85 int safe_process(struct cryptop *);
86 int safe_kprocess(struct cryptkop *);
87 int safe_kstart(struct safe_softc *);
88 void safe_kload_reg(struct safe_softc *, u_int32_t, u_int32_t,
89 struct crparam *);
90 struct safe_softc *safe_kfind(struct cryptkop *);
91 void safe_kpoll(void *);
92 void safe_kfeed(struct safe_softc *);
93 int safe_ksigbits(struct crparam *cr);
94 void safe_callback(struct safe_softc *, struct safe_ringentry *);
95 void safe_feed(struct safe_softc *, struct safe_ringentry *);
96 void safe_mcopy(struct mbuf *, struct mbuf *, u_int);
97 void safe_rng_init(struct safe_softc *);
98 void safe_rng(void *);
99 int safe_dma_malloc(struct safe_softc *, bus_size_t,
100 struct safe_dma_alloc *, int);
101 #define safe_dma_sync(_sc, _dma, _flags) \
102 bus_dmamap_sync((_sc)->sc_dmat, (_dma)->dma_map, 0, \
103 (_dma)->dma_map->dm_mapsize, (_flags))
104 void safe_dma_free(struct safe_softc *, struct safe_dma_alloc *);
105 int safe_dmamap_aligned(const struct safe_operand *);
106 int safe_dmamap_uniform(const struct safe_operand *);
107
108 void safe_reset_board(struct safe_softc *);
109 void safe_init_board(struct safe_softc *);
110 void safe_init_pciregs(struct safe_softc *);
111 void safe_cleanchip(struct safe_softc *);
112 __inline u_int32_t safe_rng_read(struct safe_softc *);
113
114 int safe_free_entry(struct safe_softc *, struct safe_ringentry *);
115
116 #ifdef SAFE_DEBUG
117 int safe_debug;
118 #define DPRINTF(_x) if (safe_debug) printf _x
119
120 void safe_dump_dmastatus(struct safe_softc *, const char *);
121 void safe_dump_intrstate(struct safe_softc *, const char *);
122 void safe_dump_ringstate(struct safe_softc *, const char *);
123 void safe_dump_request(struct safe_softc *, const char *,
124 struct safe_ringentry *);
125 void safe_dump_ring(struct safe_softc *sc, const char *tag);
126 #else
127 #define DPRINTF(_x)
128 #endif
129
130 #define READ_REG(sc,r) \
131 bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
132
133 #define WRITE_REG(sc,reg,val) \
134 bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
135
136 struct safe_stats safestats;
137
138 int safe_rnginterval = 1; /* poll once a second */
139 int safe_rngbufsize = 16; /* 64 bytes each poll */
140 int safe_rngmaxalarm = 8; /* max alarms before reset */
141
142 int
safe_probe(struct device * parent,void * match,void * aux)143 safe_probe(struct device *parent, void *match, void *aux)
144 {
145 struct pci_attach_args *pa = aux;
146
147 if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_SAFENET &&
148 PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_SAFENET_SAFEXCEL)
149 return (1);
150 return (0);
151 }
152
153 void
safe_attach(struct device * parent,struct device * self,void * aux)154 safe_attach(struct device *parent, struct device *self, void *aux)
155 {
156 struct safe_softc *sc = (struct safe_softc *)self;
157 struct pci_attach_args *pa = aux;
158 pci_intr_handle_t ih;
159 const char *intrstr = NULL;
160 bus_size_t iosize;
161 bus_addr_t raddr;
162 u_int32_t cmd, devinfo;
163 int algs[CRYPTO_ALGORITHM_MAX + 1], i;
164
165 /* XXX handle power management */
166
167 SIMPLEQ_INIT(&sc->sc_pkq);
168 sc->sc_dmat = pa->pa_dmat;
169
170 cmd = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG);
171 cmd |= PCI_COMMAND_MEM_ENABLE | PCI_COMMAND_MASTER_ENABLE;
172 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, cmd);
173 cmd = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG);
174
175 if (!(cmd & PCI_COMMAND_MEM_ENABLE)) {
176 printf(": failed to enable memory mapping\n");
177 return;
178 }
179
180 if (!(cmd & PCI_COMMAND_MASTER_ENABLE)) {
181 printf(": failed to enable bus mastering\n");
182 return;
183 }
184
185 /*
186 * Setup memory-mapping of PCI registers.
187 */
188 if (pci_mapreg_map(pa, SAFE_BAR, PCI_MAPREG_TYPE_MEM, 0,
189 &sc->sc_st, &sc->sc_sh, NULL, &iosize, 0)) {
190 printf(": can't map register space\n");
191 goto bad;
192 }
193
194 if (pci_intr_map(pa, &ih)) {
195 printf(": couldn't map interrupt\n");
196 goto bad1;
197 }
198 intrstr = pci_intr_string(pa->pa_pc, ih);
199 sc->sc_ih = pci_intr_establish(pa->pa_pc, ih, IPL_NET, safe_intr, sc,
200 self->dv_xname);
201 if (sc->sc_ih == NULL) {
202 printf(": couldn't establish interrupt");
203 if (intrstr != NULL)
204 printf(" at %s", intrstr);
205 printf("\n");
206 goto bad2;
207 }
208
209 sc->sc_cid = crypto_get_driverid(0);
210 if (sc->sc_cid < 0) {
211 printf(": could not get crypto driver id\n");
212 goto bad3;
213 }
214
215 sc->sc_chiprev = READ_REG(sc, SAFE_DEVINFO) &
216 (SAFE_DEVINFO_REV_MAJ | SAFE_DEVINFO_REV_MIN);
217
218 /*
219 * Allocate packet engine descriptors.
220 */
221 if (safe_dma_malloc(sc,
222 SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
223 &sc->sc_ringalloc, 0)) {
224 printf(": cannot allocate PE descriptor ring\n");
225 goto bad4;
226 }
227 /*
228 * Hookup the static portion of all our data structures.
229 */
230 sc->sc_ring = (struct safe_ringentry *) sc->sc_ringalloc.dma_vaddr;
231 sc->sc_ringtop = sc->sc_ring + SAFE_MAX_NQUEUE;
232 sc->sc_front = sc->sc_ring;
233 sc->sc_back = sc->sc_ring;
234 raddr = sc->sc_ringalloc.dma_paddr;
235 bzero(sc->sc_ring, SAFE_MAX_NQUEUE * sizeof(struct safe_ringentry));
236 for (i = 0; i < SAFE_MAX_NQUEUE; i++) {
237 struct safe_ringentry *re = &sc->sc_ring[i];
238
239 re->re_desc.d_sa = raddr +
240 offsetof(struct safe_ringentry, re_sa);
241 re->re_sa.sa_staterec = raddr +
242 offsetof(struct safe_ringentry, re_sastate);
243
244 raddr += sizeof (struct safe_ringentry);
245 }
246
247 /*
248 * Allocate scatter and gather particle descriptors.
249 */
250 if (safe_dma_malloc(sc, SAFE_TOTAL_SPART * sizeof (struct safe_pdesc),
251 &sc->sc_spalloc, 0)) {
252 printf(": cannot allocate source particle descriptor ring\n");
253 safe_dma_free(sc, &sc->sc_ringalloc);
254 goto bad4;
255 }
256 sc->sc_spring = (struct safe_pdesc *) sc->sc_spalloc.dma_vaddr;
257 sc->sc_springtop = sc->sc_spring + SAFE_TOTAL_SPART;
258 sc->sc_spfree = sc->sc_spring;
259 bzero(sc->sc_spring, SAFE_TOTAL_SPART * sizeof(struct safe_pdesc));
260
261 if (safe_dma_malloc(sc, SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
262 &sc->sc_dpalloc, 0)) {
263 printf(": cannot allocate destination particle "
264 "descriptor ring\n");
265 safe_dma_free(sc, &sc->sc_spalloc);
266 safe_dma_free(sc, &sc->sc_ringalloc);
267 goto bad4;
268 }
269 sc->sc_dpring = (struct safe_pdesc *) sc->sc_dpalloc.dma_vaddr;
270 sc->sc_dpringtop = sc->sc_dpring + SAFE_TOTAL_DPART;
271 sc->sc_dpfree = sc->sc_dpring;
272 bzero(sc->sc_dpring, SAFE_TOTAL_DPART * sizeof(struct safe_pdesc));
273
274 printf(":");
275
276 devinfo = READ_REG(sc, SAFE_DEVINFO);
277 if (devinfo & SAFE_DEVINFO_RNG)
278 printf(" RNG");
279
280 bzero(algs, sizeof(algs));
281 if (devinfo & SAFE_DEVINFO_PKEY) {
282 printf(" PK");
283 algs[CRK_MOD_EXP] = CRYPTO_ALG_FLAG_SUPPORTED;
284 crypto_kregister(sc->sc_cid, algs, safe_kprocess);
285 timeout_set(&sc->sc_pkto, safe_kpoll, sc);
286 }
287
288 bzero(algs, sizeof(algs));
289 if (devinfo & SAFE_DEVINFO_DES) {
290 printf(" 3DES");
291 algs[CRYPTO_3DES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED;
292 algs[CRYPTO_DES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED;
293 }
294 if (devinfo & SAFE_DEVINFO_AES) {
295 printf(" AES");
296 algs[CRYPTO_AES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED;
297 }
298 if (devinfo & SAFE_DEVINFO_MD5) {
299 printf(" MD5");
300 algs[CRYPTO_MD5_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
301 }
302 if (devinfo & SAFE_DEVINFO_SHA1) {
303 printf(" SHA1");
304 algs[CRYPTO_SHA1_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
305 }
306 crypto_register(sc->sc_cid, algs, safe_newsession,
307 safe_freesession, safe_process);
308 /* XXX other supported algorithms? */
309
310 printf(", %s\n", intrstr);
311
312 safe_reset_board(sc); /* reset h/w */
313 safe_init_pciregs(sc); /* init pci settings */
314 safe_init_board(sc); /* init h/w */
315
316 if (devinfo & SAFE_DEVINFO_RNG) {
317 safe_rng_init(sc);
318
319 timeout_set(&sc->sc_rngto, safe_rng, sc);
320 timeout_add(&sc->sc_rngto, hz * safe_rnginterval);
321 }
322 return;
323
324 bad4:
325 /* XXX crypto_unregister_all(sc->sc_cid); */
326 bad3:
327 pci_intr_disestablish(pa->pa_pc, sc->sc_ih);
328 bad2:
329 /* pci_intr_unmap? */;
330 bad1:
331 bus_space_unmap(sc->sc_st, sc->sc_sh, iosize);
332 bad:
333 return;
334 }
335
336 int
safe_process(struct cryptop * crp)337 safe_process(struct cryptop *crp)
338 {
339 int err = 0, i, nicealign, uniform, s;
340 struct safe_softc *sc;
341 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
342 int bypass, oplen, ivsize, card;
343 int16_t coffset;
344 struct safe_session *ses;
345 struct safe_ringentry *re;
346 struct safe_sarec *sa;
347 struct safe_pdesc *pd;
348 u_int32_t cmd0, cmd1, staterec, iv[4];
349
350 s = splnet();
351 if (crp == NULL || crp->crp_callback == NULL) {
352 safestats.st_invalid++;
353 splx(s);
354 return (EINVAL);
355 }
356 card = SAFE_CARD(crp->crp_sid);
357 if (card >= safe_cd.cd_ndevs || safe_cd.cd_devs[card] == NULL) {
358 safestats.st_invalid++;
359 splx(s);
360 return (EINVAL);
361 }
362 sc = safe_cd.cd_devs[card];
363
364 if (SAFE_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
365 safestats.st_badsession++;
366 splx(s);
367 return (EINVAL);
368 }
369
370 if (sc->sc_front == sc->sc_back && sc->sc_nqchip != 0) {
371 safestats.st_ringfull++;
372 splx(s);
373 return (ERESTART);
374 }
375 re = sc->sc_front;
376
377 staterec = re->re_sa.sa_staterec; /* save */
378 /* NB: zero everything but the PE descriptor */
379 bzero(&re->re_sa, sizeof(struct safe_ringentry) - sizeof(re->re_desc));
380 re->re_sa.sa_staterec = staterec; /* restore */
381
382 re->re_crp = crp;
383 re->re_sesn = SAFE_SESSION(crp->crp_sid);
384
385 if (crp->crp_flags & CRYPTO_F_IMBUF) {
386 re->re_src_m = (struct mbuf *)crp->crp_buf;
387 re->re_dst_m = (struct mbuf *)crp->crp_buf;
388 } else if (crp->crp_flags & CRYPTO_F_IOV) {
389 re->re_src_io = (struct uio *)crp->crp_buf;
390 re->re_dst_io = (struct uio *)crp->crp_buf;
391 } else {
392 safestats.st_badflags++;
393 err = EINVAL;
394 goto errout; /* XXX we don't handle contiguous blocks! */
395 }
396
397 sa = &re->re_sa;
398 ses = &sc->sc_sessions[re->re_sesn];
399
400 crd1 = crp->crp_desc;
401 if (crd1 == NULL) {
402 safestats.st_nodesc++;
403 err = EINVAL;
404 goto errout;
405 }
406 crd2 = crd1->crd_next;
407
408 cmd0 = SAFE_SA_CMD0_BASIC; /* basic group operation */
409 cmd1 = 0;
410 if (crd2 == NULL) {
411 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
412 crd1->crd_alg == CRYPTO_SHA1_HMAC) {
413 maccrd = crd1;
414 enccrd = NULL;
415 cmd0 |= SAFE_SA_CMD0_OP_HASH;
416 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
417 crd1->crd_alg == CRYPTO_3DES_CBC ||
418 crd1->crd_alg == CRYPTO_AES_CBC) {
419 maccrd = NULL;
420 enccrd = crd1;
421 cmd0 |= SAFE_SA_CMD0_OP_CRYPT;
422 } else {
423 safestats.st_badalg++;
424 err = EINVAL;
425 goto errout;
426 }
427 } else {
428 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
429 crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
430 (crd2->crd_alg == CRYPTO_DES_CBC ||
431 crd2->crd_alg == CRYPTO_3DES_CBC ||
432 crd2->crd_alg == CRYPTO_AES_CBC) &&
433 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
434 maccrd = crd1;
435 enccrd = crd2;
436 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
437 crd1->crd_alg == CRYPTO_3DES_CBC ||
438 crd1->crd_alg == CRYPTO_AES_CBC) &&
439 (crd2->crd_alg == CRYPTO_MD5_HMAC ||
440 crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
441 (crd1->crd_flags & CRD_F_ENCRYPT)) {
442 enccrd = crd1;
443 maccrd = crd2;
444 } else {
445 safestats.st_badalg++;
446 err = EINVAL;
447 goto errout;
448 }
449 cmd0 |= SAFE_SA_CMD0_OP_BOTH;
450 }
451
452 if (enccrd) {
453 if (enccrd->crd_alg == CRYPTO_DES_CBC) {
454 cmd0 |= SAFE_SA_CMD0_DES;
455 cmd1 |= SAFE_SA_CMD1_CBC;
456 ivsize = 2*sizeof(u_int32_t);
457 } else if (enccrd->crd_alg == CRYPTO_3DES_CBC) {
458 cmd0 |= SAFE_SA_CMD0_3DES;
459 cmd1 |= SAFE_SA_CMD1_CBC;
460 ivsize = 2*sizeof(u_int32_t);
461 } else if (enccrd->crd_alg == CRYPTO_AES_CBC) {
462 cmd0 |= SAFE_SA_CMD0_AES;
463 cmd1 |= SAFE_SA_CMD1_CBC;
464 if (ses->ses_klen == 128)
465 cmd1 |= SAFE_SA_CMD1_AES128;
466 else if (ses->ses_klen == 192)
467 cmd1 |= SAFE_SA_CMD1_AES192;
468 else
469 cmd1 |= SAFE_SA_CMD1_AES256;
470 ivsize = 4*sizeof(u_int32_t);
471 } else {
472 cmd0 |= SAFE_SA_CMD0_CRYPT_NULL;
473 ivsize = 0;
474 }
475
476 /*
477 * Setup encrypt/decrypt state. When using basic ops
478 * we can't use an inline IV because hash/crypt offset
479 * must be from the end of the IV to the start of the
480 * crypt data and this leaves out the preceding header
481 * from the hash calculation. Instead we place the IV
482 * in the state record and set the hash/crypt offset to
483 * copy both the header+IV.
484 */
485 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
486 cmd0 |= SAFE_SA_CMD0_OUTBOUND;
487
488 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
489 bcopy(enccrd->crd_iv, iv, ivsize);
490 else
491 bcopy(ses->ses_iv, iv, ivsize);
492 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
493 if (crp->crp_flags & CRYPTO_F_IMBUF)
494 m_copyback(re->re_src_m,
495 enccrd->crd_inject, ivsize, iv);
496 else if (crp->crp_flags & CRYPTO_F_IOV)
497 cuio_copyback(re->re_src_io,
498 enccrd->crd_inject, ivsize, iv);
499 }
500 for (i = 0; i < ivsize / sizeof(iv[0]); i++)
501 re->re_sastate.sa_saved_iv[i] = htole32(iv[i]);
502 cmd0 |= SAFE_SA_CMD0_IVLD_STATE | SAFE_SA_CMD0_SAVEIV;
503 re->re_flags |= SAFE_QFLAGS_COPYOUTIV;
504 } else {
505 cmd0 |= SAFE_SA_CMD0_INBOUND;
506
507 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
508 bcopy(enccrd->crd_iv, iv, ivsize);
509 else if (crp->crp_flags & CRYPTO_F_IMBUF)
510 m_copydata(re->re_src_m, enccrd->crd_inject,
511 ivsize, (caddr_t)iv);
512 else if (crp->crp_flags & CRYPTO_F_IOV)
513 cuio_copydata(re->re_src_io, enccrd->crd_inject,
514 ivsize, (caddr_t)iv);
515 for (i = 0; i < ivsize / sizeof(iv[0]); i++)
516 re->re_sastate.sa_saved_iv[i] = htole32(iv[i]);
517 cmd0 |= SAFE_SA_CMD0_IVLD_STATE;
518 }
519 /*
520 * For basic encryption use the zero pad algorithm.
521 * This pads results to an 8-byte boundary and
522 * suppresses padding verification for inbound (i.e.
523 * decrypt) operations.
524 *
525 * NB: Not sure if the 8-byte pad boundary is a problem.
526 */
527 cmd0 |= SAFE_SA_CMD0_PAD_ZERO;
528
529 /* XXX assert key bufs have the same size */
530 for (i = 0; i < sizeof(sa->sa_key)/sizeof(sa->sa_key[0]); i++)
531 sa->sa_key[i] = ses->ses_key[i];
532 }
533
534 if (maccrd) {
535 if (maccrd->crd_alg == CRYPTO_MD5_HMAC) {
536 cmd0 |= SAFE_SA_CMD0_MD5;
537 cmd1 |= SAFE_SA_CMD1_HMAC; /* NB: enable HMAC */
538 } else if (maccrd->crd_alg == CRYPTO_SHA1_HMAC) {
539 cmd0 |= SAFE_SA_CMD0_SHA1;
540 cmd1 |= SAFE_SA_CMD1_HMAC; /* NB: enable HMAC */
541 } else {
542 cmd0 |= SAFE_SA_CMD0_HASH_NULL;
543 }
544 /*
545 * Digest data is loaded from the SA and the hash
546 * result is saved to the state block where we
547 * retrieve it for return to the caller.
548 */
549 /* XXX assert digest bufs have the same size */
550 for (i = 0;
551 i < sizeof(sa->sa_outdigest)/sizeof(sa->sa_outdigest[i]);
552 i++) {
553 sa->sa_indigest[i] = ses->ses_hminner[i];
554 sa->sa_outdigest[i] = ses->ses_hmouter[i];
555 }
556
557 cmd0 |= SAFE_SA_CMD0_HSLD_SA | SAFE_SA_CMD0_SAVEHASH;
558 re->re_flags |= SAFE_QFLAGS_COPYOUTICV;
559 }
560
561 if (enccrd && maccrd) {
562 /*
563 * The offset from hash data to the start of
564 * crypt data is the difference in the skips.
565 */
566 bypass = maccrd->crd_skip;
567 coffset = enccrd->crd_skip - maccrd->crd_skip;
568 if (coffset < 0) {
569 DPRINTF(("%s: hash does not precede crypt; "
570 "mac skip %u enc skip %u\n",
571 __func__, maccrd->crd_skip, enccrd->crd_skip));
572 safestats.st_skipmismatch++;
573 err = EINVAL;
574 goto errout;
575 }
576 oplen = enccrd->crd_skip + enccrd->crd_len;
577 if (maccrd->crd_skip + maccrd->crd_len != oplen) {
578 DPRINTF(("%s: hash amount %u != crypt amount %u\n",
579 __func__, maccrd->crd_skip + maccrd->crd_len,
580 oplen));
581 safestats.st_lenmismatch++;
582 err = EINVAL;
583 goto errout;
584 }
585 #ifdef SAFE_DEBUG
586 if (safe_debug) {
587 printf("mac: skip %d, len %d, inject %d\n",
588 maccrd->crd_skip, maccrd->crd_len,
589 maccrd->crd_inject);
590 printf("enc: skip %d, len %d, inject %d\n",
591 enccrd->crd_skip, enccrd->crd_len,
592 enccrd->crd_inject);
593 printf("bypass %d coffset %d oplen %d\n",
594 bypass, coffset, oplen);
595 }
596 #endif
597 if (coffset & 3) { /* offset must be 32-bit aligned */
598 DPRINTF(("%s: coffset %u misaligned\n",
599 __func__, coffset));
600 safestats.st_coffmisaligned++;
601 err = EINVAL;
602 goto errout;
603 }
604 coffset >>= 2;
605 if (coffset > 255) { /* offset must be <256 dwords */
606 DPRINTF(("%s: coffset %u too big\n",
607 __func__, coffset));
608 safestats.st_cofftoobig++;
609 err = EINVAL;
610 goto errout;
611 }
612 /*
613 * Tell the hardware to copy the header to the output.
614 * The header is defined as the data from the end of
615 * the bypass to the start of data to be encrypted.
616 * Typically this is the inline IV. Note that you need
617 * to do this even if src+dst are the same; it appears
618 * that w/o this bit the crypted data is written
619 * immediately after the bypass data.
620 */
621 cmd1 |= SAFE_SA_CMD1_HDRCOPY;
622 /*
623 * Disable IP header mutable bit handling. This is
624 * needed to get correct HMAC calculations.
625 */
626 cmd1 |= SAFE_SA_CMD1_MUTABLE;
627 } else {
628 if (enccrd) {
629 bypass = enccrd->crd_skip;
630 oplen = bypass + enccrd->crd_len;
631 } else {
632 bypass = maccrd->crd_skip;
633 oplen = bypass + maccrd->crd_len;
634 }
635 coffset = 0;
636 }
637 /* XXX verify multiple of 4 when using s/g */
638 if (bypass > 96) { /* bypass offset must be <= 96 bytes */
639 DPRINTF(("%s: bypass %u too big\n", __func__, bypass));
640 safestats.st_bypasstoobig++;
641 err = EINVAL;
642 goto errout;
643 }
644
645 if (bus_dmamap_create(sc->sc_dmat, SAFE_MAX_DMA, SAFE_MAX_PART,
646 SAFE_MAX_DSIZE, SAFE_MAX_DSIZE, BUS_DMA_ALLOCNOW | BUS_DMA_NOWAIT,
647 &re->re_src_map)) {
648 safestats.st_nomap++;
649 err = ENOMEM;
650 goto errout;
651 }
652 if (crp->crp_flags & CRYPTO_F_IMBUF) {
653 if (bus_dmamap_load_mbuf(sc->sc_dmat, re->re_src_map,
654 re->re_src_m, BUS_DMA_NOWAIT)) {
655 bus_dmamap_destroy(sc->sc_dmat, re->re_src_map);
656 re->re_src_map = NULL;
657 safestats.st_noload++;
658 err = ENOMEM;
659 goto errout;
660 }
661 } else if (crp->crp_flags & CRYPTO_F_IOV) {
662 if (bus_dmamap_load_uio(sc->sc_dmat, re->re_src_map,
663 re->re_src_io, BUS_DMA_NOWAIT) != 0) {
664 bus_dmamap_destroy(sc->sc_dmat, re->re_src_map);
665 re->re_src_map = NULL;
666 safestats.st_noload++;
667 err = ENOMEM;
668 goto errout;
669 }
670 }
671 nicealign = safe_dmamap_aligned(&re->re_src);
672 uniform = safe_dmamap_uniform(&re->re_src);
673
674 DPRINTF(("src nicealign %u uniform %u nsegs %u\n",
675 nicealign, uniform, re->re_src_nsegs));
676 if (re->re_src_nsegs > 1) {
677 re->re_desc.d_src = sc->sc_spalloc.dma_paddr +
678 ((caddr_t) sc->sc_spfree - (caddr_t) sc->sc_spring);
679 for (i = 0; i < re->re_src_nsegs; i++) {
680 /* NB: no need to check if there's space */
681 pd = sc->sc_spfree;
682 if (++(sc->sc_spfree) == sc->sc_springtop)
683 sc->sc_spfree = sc->sc_spring;
684
685 KASSERT_X((pd->pd_flags&3) == 0 ||
686 (pd->pd_flags&3) == SAFE_PD_DONE,
687 ("bogus source particle descriptor; flags %x",
688 pd->pd_flags));
689 pd->pd_addr = re->re_src_segs[i].ds_addr;
690 pd->pd_ctrl = SAFE_PD_READY |
691 ((re->re_src_segs[i].ds_len << SAFE_PD_LEN_S)
692 & SAFE_PD_LEN_M);
693 }
694 cmd0 |= SAFE_SA_CMD0_IGATHER;
695 } else {
696 /*
697 * No need for gather, reference the operand directly.
698 */
699 re->re_desc.d_src = re->re_src_segs[0].ds_addr;
700 }
701
702 if (enccrd == NULL && maccrd != NULL) {
703 /*
704 * Hash op; no destination needed.
705 */
706 } else {
707 if (crp->crp_flags & CRYPTO_F_IOV) {
708 if (!nicealign) {
709 safestats.st_iovmisaligned++;
710 err = EINVAL;
711 goto errout;
712 }
713 if (uniform != 1) {
714 /*
715 * Source is not suitable for direct use as
716 * the destination. Create a new scatter/gather
717 * list based on the destination requirements
718 * and check if that's ok.
719 */
720 if (bus_dmamap_create(sc->sc_dmat,
721 SAFE_MAX_DMA, SAFE_MAX_PART,
722 SAFE_MAX_DSIZE, SAFE_MAX_DSIZE,
723 BUS_DMA_ALLOCNOW | BUS_DMA_NOWAIT,
724 &re->re_dst_map)) {
725 safestats.st_nomap++;
726 err = ENOMEM;
727 goto errout;
728 }
729 if (bus_dmamap_load_uio(sc->sc_dmat,
730 re->re_dst_map, re->re_dst_io,
731 BUS_DMA_NOWAIT) != 0) {
732 bus_dmamap_destroy(sc->sc_dmat,
733 re->re_dst_map);
734 re->re_dst_map = NULL;
735 safestats.st_noload++;
736 err = ENOMEM;
737 goto errout;
738 }
739 uniform = safe_dmamap_uniform(&re->re_dst);
740 if (!uniform) {
741 /*
742 * There's no way to handle the DMA
743 * requirements with this uio. We
744 * could create a separate DMA area for
745 * the result and then copy it back,
746 * but for now we just bail and return
747 * an error. Note that uio requests
748 * > SAFE_MAX_DSIZE are handled because
749 * the DMA map and segment list for the
750 * destination wil result in a
751 * destination particle list that does
752 * the necessary scatter DMA.
753 */
754 safestats.st_iovnotuniform++;
755 err = EINVAL;
756 goto errout;
757 }
758 } else
759 re->re_dst = re->re_src;
760 } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
761 if (nicealign && uniform == 1) {
762 /*
763 * Source layout is suitable for direct
764 * sharing of the DMA map and segment list.
765 */
766 re->re_dst = re->re_src;
767 } else if (nicealign && uniform == 2) {
768 /*
769 * The source is properly aligned but requires a
770 * different particle list to handle DMA of the
771 * result. Create a new map and do the load to
772 * create the segment list. The particle
773 * descriptor setup code below will handle the
774 * rest.
775 */
776 if (bus_dmamap_create(sc->sc_dmat,
777 SAFE_MAX_DMA, SAFE_MAX_PART,
778 SAFE_MAX_DSIZE, SAFE_MAX_DSIZE,
779 BUS_DMA_ALLOCNOW | BUS_DMA_NOWAIT,
780 &re->re_dst_map)) {
781 safestats.st_nomap++;
782 err = ENOMEM;
783 goto errout;
784 }
785 if (bus_dmamap_load_mbuf(sc->sc_dmat,
786 re->re_dst_map, re->re_dst_m,
787 BUS_DMA_NOWAIT) != 0) {
788 bus_dmamap_destroy(sc->sc_dmat,
789 re->re_dst_map);
790 re->re_dst_map = NULL;
791 safestats.st_noload++;
792 err = ENOMEM;
793 goto errout;
794 }
795 } else { /* !(aligned and/or uniform) */
796 int totlen, len;
797 struct mbuf *m, *top, **mp;
798
799 /*
800 * DMA constraints require that we allocate a
801 * new mbuf chain for the destination. We
802 * allocate an entire new set of mbufs of
803 * optimal/required size and then tell the
804 * hardware to copy any bits that are not
805 * created as a byproduct of the operation.
806 */
807 if (!nicealign)
808 safestats.st_unaligned++;
809 if (!uniform)
810 safestats.st_notuniform++;
811 totlen = re->re_src_mapsize;
812 if (re->re_src_m->m_flags & M_PKTHDR) {
813 len = MHLEN;
814 MGETHDR(m, M_DONTWAIT, MT_DATA);
815 } else {
816 len = MLEN;
817 MGET(m, M_DONTWAIT, MT_DATA);
818 }
819 if (m == NULL) {
820 safestats.st_nombuf++;
821 err = sc->sc_nqchip ? ERESTART : ENOMEM;
822 goto errout;
823 }
824 if (len == MHLEN)
825 M_DUP_PKTHDR(m, re->re_src_m);
826 if (totlen >= MINCLSIZE) {
827 MCLGET(m, M_DONTWAIT);
828 if ((m->m_flags & M_EXT) == 0) {
829 m_free(m);
830 safestats.st_nomcl++;
831 err = sc->sc_nqchip ?
832 ERESTART : ENOMEM;
833 goto errout;
834 }
835 len = MCLBYTES;
836 }
837 m->m_len = len;
838 top = NULL;
839 mp = ⊤
840
841 while (totlen > 0) {
842 if (top) {
843 MGET(m, M_DONTWAIT, MT_DATA);
844 if (m == NULL) {
845 m_freem(top);
846 safestats.st_nombuf++;
847 err = sc->sc_nqchip ?
848 ERESTART : ENOMEM;
849 goto errout;
850 }
851 len = MLEN;
852 }
853 if (top && totlen >= MINCLSIZE) {
854 MCLGET(m, M_DONTWAIT);
855 if ((m->m_flags & M_EXT) == 0) {
856 *mp = m;
857 m_freem(top);
858 safestats.st_nomcl++;
859 err = sc->sc_nqchip ?
860 ERESTART : ENOMEM;
861 goto errout;
862 }
863 len = MCLBYTES;
864 }
865 m->m_len = len = min(totlen, len);
866 totlen -= len;
867 *mp = m;
868 mp = &m->m_next;
869 }
870 re->re_dst_m = top;
871 if (bus_dmamap_create(sc->sc_dmat,
872 SAFE_MAX_DMA, SAFE_MAX_PART,
873 SAFE_MAX_DSIZE, SAFE_MAX_DSIZE,
874 BUS_DMA_ALLOCNOW | BUS_DMA_NOWAIT,
875 &re->re_dst_map) != 0) {
876 safestats.st_nomap++;
877 err = ENOMEM;
878 goto errout;
879 }
880 if (bus_dmamap_load_mbuf(sc->sc_dmat,
881 re->re_dst_map, re->re_dst_m,
882 BUS_DMA_NOWAIT) != 0) {
883 bus_dmamap_destroy(sc->sc_dmat,
884 re->re_dst_map);
885 re->re_dst_map = NULL;
886 safestats.st_noload++;
887 err = ENOMEM;
888 goto errout;
889 }
890 if (re->re_src_mapsize > oplen) {
891 /*
892 * There's data following what the
893 * hardware will copy for us. If this
894 * isn't just the ICV (that's going to
895 * be written on completion), copy it
896 * to the new mbufs
897 */
898 if (!(maccrd &&
899 (re->re_src_mapsize-oplen) == 12 &&
900 maccrd->crd_inject == oplen))
901 safe_mcopy(re->re_src_m,
902 re->re_dst_m,
903 oplen);
904 else
905 safestats.st_noicvcopy++;
906 }
907 }
908 } else {
909 safestats.st_badflags++;
910 err = EINVAL;
911 goto errout;
912 }
913
914 if (re->re_dst_nsegs > 1) {
915 re->re_desc.d_dst = sc->sc_dpalloc.dma_paddr +
916 ((caddr_t) sc->sc_dpfree - (caddr_t) sc->sc_dpring);
917 for (i = 0; i < re->re_dst_nsegs; i++) {
918 pd = sc->sc_dpfree;
919 KASSERT_X((pd->pd_flags&3) == 0 ||
920 (pd->pd_flags&3) == SAFE_PD_DONE,
921 ("bogus dest particle descriptor; flags %x",
922 pd->pd_flags));
923 if (++(sc->sc_dpfree) == sc->sc_dpringtop)
924 sc->sc_dpfree = sc->sc_dpring;
925 pd->pd_addr = re->re_dst_segs[i].ds_addr;
926 pd->pd_ctrl = SAFE_PD_READY;
927 }
928 cmd0 |= SAFE_SA_CMD0_OSCATTER;
929 } else {
930 /*
931 * No need for scatter, reference the operand directly.
932 */
933 re->re_desc.d_dst = re->re_dst_segs[0].ds_addr;
934 }
935 }
936
937 /*
938 * All done with setup; fillin the SA command words
939 * and the packet engine descriptor. The operation
940 * is now ready for submission to the hardware.
941 */
942 sa->sa_cmd0 = cmd0 | SAFE_SA_CMD0_IPCI | SAFE_SA_CMD0_OPCI;
943 sa->sa_cmd1 = cmd1
944 | (coffset << SAFE_SA_CMD1_OFFSET_S)
945 | SAFE_SA_CMD1_SAREV1 /* Rev 1 SA data structure */
946 | SAFE_SA_CMD1_SRPCI;
947
948 /*
949 * NB: the order of writes is important here. In case the
950 * chip is scanning the ring because of an outstanding request
951 * it might nab this one too. In that case we need to make
952 * sure the setup is complete before we write the length
953 * field of the descriptor as it signals the descriptor is
954 * ready for processing.
955 */
956 re->re_desc.d_csr = SAFE_PE_CSR_READY | SAFE_PE_CSR_SAPCI;
957 if (maccrd)
958 re->re_desc.d_csr |= SAFE_PE_CSR_LOADSA | SAFE_PE_CSR_HASHFINAL;
959 re->re_desc.d_len = oplen
960 | SAFE_PE_LEN_READY
961 | (bypass << SAFE_PE_LEN_BYPASS_S)
962 ;
963
964 safestats.st_ipackets++;
965 safestats.st_ibytes += oplen;
966
967 if (++(sc->sc_front) == sc->sc_ringtop)
968 sc->sc_front = sc->sc_ring;
969
970 /* XXX honor batching */
971 safe_feed(sc, re);
972 splx(s);
973 return (0);
974
975 errout:
976 if ((re->re_dst_m != NULL) && (re->re_src_m != re->re_dst_m))
977 m_freem(re->re_dst_m);
978
979 if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {
980 bus_dmamap_unload(sc->sc_dmat, re->re_dst_map);
981 bus_dmamap_destroy(sc->sc_dmat, re->re_dst_map);
982 }
983 if (re->re_src_map != NULL) {
984 bus_dmamap_unload(sc->sc_dmat, re->re_src_map);
985 bus_dmamap_destroy(sc->sc_dmat, re->re_src_map);
986 }
987 crp->crp_etype = err;
988 crypto_done(crp);
989 splx(s);
990 return (err);
991 }
992
993 /*
994 * Resets the board. Values in the regesters are left as is
995 * from the reset (i.e. initial values are assigned elsewhere).
996 */
997 void
safe_reset_board(struct safe_softc * sc)998 safe_reset_board(struct safe_softc *sc)
999 {
1000 u_int32_t v;
1001
1002 /*
1003 * Reset the device. The manual says no delay
1004 * is needed between marking and clearing reset.
1005 */
1006 v = READ_REG(sc, SAFE_PE_DMACFG) &
1007 ~(SAFE_PE_DMACFG_PERESET | SAFE_PE_DMACFG_PDRRESET |
1008 SAFE_PE_DMACFG_SGRESET);
1009 WRITE_REG(sc, SAFE_PE_DMACFG, v
1010 | SAFE_PE_DMACFG_PERESET
1011 | SAFE_PE_DMACFG_PDRRESET
1012 | SAFE_PE_DMACFG_SGRESET);
1013 WRITE_REG(sc, SAFE_PE_DMACFG, v);
1014 }
1015
1016 /*
1017 * Initialize registers we need to touch only once.
1018 */
1019 void
safe_init_board(struct safe_softc * sc)1020 safe_init_board(struct safe_softc *sc)
1021 {
1022 u_int32_t v, dwords;
1023
1024 v = READ_REG(sc, SAFE_PE_DMACFG);
1025 v &= ~(SAFE_PE_DMACFG_PEMODE | SAFE_PE_DMACFG_ESPACKET);
1026 v |= SAFE_PE_DMACFG_FSENA /* failsafe enable */
1027 | SAFE_PE_DMACFG_GPRPCI /* gather ring on PCI */
1028 | SAFE_PE_DMACFG_SPRPCI /* scatter ring on PCI */
1029 | SAFE_PE_DMACFG_ESDESC /* endian-swap descriptors */
1030 | SAFE_PE_DMACFG_ESPDESC /* endian-swap part. desc's */
1031 | SAFE_PE_DMACFG_ESSA /* endian-swap SA data */
1032 ;
1033 WRITE_REG(sc, SAFE_PE_DMACFG, v);
1034
1035 WRITE_REG(sc, SAFE_CRYPTO_CTRL, SAFE_CRYPTO_CTRL_PKEY |
1036 SAFE_CRYPTO_CTRL_3DES | SAFE_CRYPTO_CTRL_RNG);
1037
1038 #if BYTE_ORDER == LITTLE_ENDIAN
1039 WRITE_REG(sc, SAFE_ENDIAN, SAFE_ENDIAN_TGT_PASS|SAFE_ENDIAN_DMA_PASS);
1040 #elif BYTE_ORDER == BIG_ENDIAN
1041 WRITE_REG(sc, SAFE_ENDIAN, SAFE_ENDIAN_TGT_PASS|SAFE_ENDIAN_DMA_SWAB);
1042 #endif
1043
1044 if (sc->sc_chiprev == SAFE_REV(1,0)) {
1045 /*
1046 * Avoid large PCI DMA transfers. Rev 1.0 has a bug where
1047 * "target mode transfers" done while the chip is DMA'ing
1048 * >1020 bytes cause the hardware to lockup. To avoid this
1049 * we reduce the max PCI transfer size and use small source
1050 * particle descriptors (<= 256 bytes).
1051 */
1052 WRITE_REG(sc, SAFE_DMA_CFG, 256);
1053 printf("%s: Reduce max DMA size to %u words for rev %u.%u WAR\n",
1054 sc->sc_dev.dv_xname,
1055 (READ_REG(sc, SAFE_DMA_CFG)>>2) & 0xff,
1056 SAFE_REV_MAJ(sc->sc_chiprev),
1057 SAFE_REV_MIN(sc->sc_chiprev));
1058 }
1059
1060 /* NB: operands+results are overlaid */
1061 WRITE_REG(sc, SAFE_PE_PDRBASE, sc->sc_ringalloc.dma_paddr);
1062 WRITE_REG(sc, SAFE_PE_RDRBASE, sc->sc_ringalloc.dma_paddr);
1063 /*
1064 * Configure ring entry size and number of items in the ring.
1065 */
1066 KASSERT_X((sizeof(struct safe_ringentry) % sizeof(u_int32_t)) == 0,
1067 ("PE ring entry not 32-bit aligned!"));
1068 dwords = sizeof(struct safe_ringentry) / sizeof(u_int32_t);
1069 WRITE_REG(sc, SAFE_PE_RINGCFG,
1070 (dwords << SAFE_PE_RINGCFG_OFFSET_S) | SAFE_MAX_NQUEUE);
1071 WRITE_REG(sc, SAFE_PE_RINGPOLL, 0); /* disable polling */
1072
1073 WRITE_REG(sc, SAFE_PE_GRNGBASE, sc->sc_spalloc.dma_paddr);
1074 WRITE_REG(sc, SAFE_PE_SRNGBASE, sc->sc_dpalloc.dma_paddr);
1075 WRITE_REG(sc, SAFE_PE_PARTSIZE,
1076 (SAFE_TOTAL_DPART<<16) | SAFE_TOTAL_SPART);
1077 /*
1078 * NB: destination particles are fixed size. We use
1079 * an mbuf cluster and require all results go to
1080 * clusters or smaller.
1081 */
1082 WRITE_REG(sc, SAFE_PE_PARTCFG, SAFE_MAX_DSIZE);
1083
1084 WRITE_REG(sc, SAFE_HI_CLR, SAFE_INT_PE_CDONE | SAFE_INT_PE_DDONE |
1085 SAFE_INT_PE_ERROR | SAFE_INT_PE_ODONE);
1086
1087 /* it's now safe to enable PE mode, do it */
1088 WRITE_REG(sc, SAFE_PE_DMACFG, v | SAFE_PE_DMACFG_PEMODE);
1089
1090 /*
1091 * Configure hardware to use level-triggered interrupts and
1092 * to interrupt after each descriptor is processed.
1093 */
1094 DELAY(1000);
1095 WRITE_REG(sc, SAFE_HI_CFG, SAFE_HI_CFG_LEVEL);
1096 DELAY(1000);
1097 WRITE_REG(sc, SAFE_HI_MASK, SAFE_INT_PE_DDONE | SAFE_INT_PE_ERROR);
1098 DELAY(1000);
1099 WRITE_REG(sc, SAFE_HI_DESC_CNT, 1);
1100 DELAY(1000);
1101 }
1102
1103 /*
1104 * Init PCI registers
1105 */
1106 void
safe_init_pciregs(struct safe_softc * sc)1107 safe_init_pciregs(struct safe_softc *sc)
1108 {
1109 }
1110
1111 int
safe_dma_malloc(struct safe_softc * sc,bus_size_t size,struct safe_dma_alloc * dma,int mapflags)1112 safe_dma_malloc(struct safe_softc *sc, bus_size_t size,
1113 struct safe_dma_alloc *dma, int mapflags)
1114 {
1115 int r;
1116
1117 if ((r = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0,
1118 &dma->dma_seg, 1, &dma->dma_nseg, BUS_DMA_NOWAIT)) != 0)
1119 goto fail_0;
1120
1121 if ((r = bus_dmamem_map(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg,
1122 size, &dma->dma_vaddr, mapflags | BUS_DMA_NOWAIT)) != 0)
1123 goto fail_1;
1124
1125 if ((r = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
1126 BUS_DMA_NOWAIT, &dma->dma_map)) != 0)
1127 goto fail_2;
1128
1129 if ((r = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr,
1130 size, NULL, BUS_DMA_NOWAIT)) != 0)
1131 goto fail_3;
1132
1133 dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr;
1134 dma->dma_size = size;
1135 return (0);
1136
1137 fail_3:
1138 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
1139 fail_2:
1140 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, size);
1141 fail_1:
1142 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg);
1143 fail_0:
1144 dma->dma_map = NULL;
1145 return (r);
1146 }
1147
1148 void
safe_dma_free(struct safe_softc * sc,struct safe_dma_alloc * dma)1149 safe_dma_free(struct safe_softc *sc, struct safe_dma_alloc *dma)
1150 {
1151 bus_dmamap_unload(sc->sc_dmat, dma->dma_map);
1152 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, dma->dma_size);
1153 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg);
1154 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
1155 }
1156
1157
1158 #define SAFE_RNG_MAXWAIT 1000
1159
1160 void
safe_rng_init(struct safe_softc * sc)1161 safe_rng_init(struct safe_softc *sc)
1162 {
1163 u_int32_t w, v;
1164 int i;
1165
1166 WRITE_REG(sc, SAFE_RNG_CTRL, 0);
1167 /* use default value according to the manual */
1168 WRITE_REG(sc, SAFE_RNG_CNFG, 0x834); /* magic from SafeNet */
1169 WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1170
1171 /*
1172 * There is a bug in rev 1.0 of the 1140 that when the RNG
1173 * is brought out of reset the ready status flag does not
1174 * work until the RNG has finished its internal initialization.
1175 *
1176 * So in order to determine the device is through its
1177 * initialization we must read the data register, using the
1178 * status reg in the read in case it is initialized. Then read
1179 * the data register until it changes from the first read.
1180 * Once it changes read the data register until it changes
1181 * again. At this time the RNG is considered initialized.
1182 * This could take between 750ms - 1000ms in time.
1183 */
1184 i = 0;
1185 w = READ_REG(sc, SAFE_RNG_OUT);
1186 do {
1187 v = READ_REG(sc, SAFE_RNG_OUT);
1188 if (v != w) {
1189 w = v;
1190 break;
1191 }
1192 DELAY(10);
1193 } while (++i < SAFE_RNG_MAXWAIT);
1194
1195 /* Wait Until data changes again */
1196 i = 0;
1197 do {
1198 v = READ_REG(sc, SAFE_RNG_OUT);
1199 if (v != w)
1200 break;
1201 DELAY(10);
1202 } while (++i < SAFE_RNG_MAXWAIT);
1203 }
1204
1205 __inline u_int32_t
safe_rng_read(struct safe_softc * sc)1206 safe_rng_read(struct safe_softc *sc)
1207 {
1208 int i;
1209
1210 i = 0;
1211 while (READ_REG(sc, SAFE_RNG_STAT) != 0 && ++i < SAFE_RNG_MAXWAIT)
1212 ;
1213 return (READ_REG(sc, SAFE_RNG_OUT));
1214 }
1215
1216 void
safe_rng(void * arg)1217 safe_rng(void *arg)
1218 {
1219 struct safe_softc *sc = arg;
1220 u_int32_t buf[SAFE_RNG_MAXBUFSIZ]; /* NB: maybe move to softc */
1221 u_int maxwords;
1222 int i;
1223
1224 safestats.st_rng++;
1225 /*
1226 * Fetch the next block of data.
1227 */
1228 maxwords = safe_rngbufsize;
1229 if (maxwords > SAFE_RNG_MAXBUFSIZ)
1230 maxwords = SAFE_RNG_MAXBUFSIZ;
1231 retry:
1232 for (i = 0; i < maxwords; i++)
1233 buf[i] = safe_rng_read(sc);
1234 /*
1235 * Check the comparator alarm count and reset the h/w if
1236 * it exceeds our threshold. This guards against the
1237 * hardware oscillators resonating with external signals.
1238 */
1239 if (READ_REG(sc, SAFE_RNG_ALM_CNT) > safe_rngmaxalarm) {
1240 u_int32_t freq_inc, w;
1241
1242 DPRINTF(("%s: alarm count %u exceeds threshold %u\n", __func__,
1243 READ_REG(sc, SAFE_RNG_ALM_CNT), safe_rngmaxalarm));
1244 safestats.st_rngalarm++;
1245 WRITE_REG(sc, SAFE_RNG_CTRL,
1246 READ_REG(sc, SAFE_RNG_CTRL) | SAFE_RNG_CTRL_SHORTEN);
1247 freq_inc = 18;
1248 for (i = 0; i < 64; i++) {
1249 w = READ_REG(sc, SAFE_RNG_CNFG);
1250 freq_inc = ((w + freq_inc) & 0x3fL);
1251 w = ((w & ~0x3fL) | freq_inc);
1252 WRITE_REG(sc, SAFE_RNG_CNFG, w);
1253
1254 WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1255
1256 (void) safe_rng_read(sc);
1257 DELAY(25);
1258
1259 if (READ_REG(sc, SAFE_RNG_ALM_CNT) == 0) {
1260 WRITE_REG(sc, SAFE_RNG_CTRL,
1261 READ_REG(sc, SAFE_RNG_CTRL) &
1262 ~SAFE_RNG_CTRL_SHORTEN);
1263 goto retry;
1264 }
1265 freq_inc = 1;
1266 }
1267 WRITE_REG(sc, SAFE_RNG_CTRL,
1268 READ_REG(sc, SAFE_RNG_CTRL) & ~SAFE_RNG_CTRL_SHORTEN);
1269 } else
1270 WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1271
1272 for (i = 0; i < maxwords; i++)
1273 add_true_randomness(buf[i]);
1274
1275 timeout_add(&sc->sc_rngto, hz * safe_rnginterval);
1276 }
1277
1278 /*
1279 * Allocate a new 'session' and return an encoded session id. 'sidp'
1280 * contains our registration id, and should contain an encoded session
1281 * id on successful allocation.
1282 */
1283 int
safe_newsession(u_int32_t * sidp,struct cryptoini * cri)1284 safe_newsession(u_int32_t *sidp, struct cryptoini *cri)
1285 {
1286 struct cryptoini *c, *encini = NULL, *macini = NULL;
1287 struct safe_softc *sc = NULL;
1288 struct safe_session *ses = NULL;
1289 MD5_CTX md5ctx;
1290 SHA1_CTX sha1ctx;
1291 int i, sesn;
1292
1293 if (sidp == NULL || cri == NULL)
1294 return (EINVAL);
1295 for (i = 0; i < safe_cd.cd_ndevs; i++) {
1296 sc = safe_cd.cd_devs[i];
1297 if (sc == NULL || sc->sc_cid == (*sidp))
1298 break;
1299 }
1300 if (sc == NULL)
1301 return (EINVAL);
1302
1303 for (c = cri; c != NULL; c = c->cri_next) {
1304 if (c->cri_alg == CRYPTO_MD5_HMAC ||
1305 c->cri_alg == CRYPTO_SHA1_HMAC) {
1306 if (macini)
1307 return (EINVAL);
1308 macini = c;
1309 } else if (c->cri_alg == CRYPTO_DES_CBC ||
1310 c->cri_alg == CRYPTO_3DES_CBC ||
1311 c->cri_alg == CRYPTO_AES_CBC) {
1312 if (encini)
1313 return (EINVAL);
1314 encini = c;
1315 } else
1316 return (EINVAL);
1317 }
1318 if (encini == NULL && macini == NULL)
1319 return (EINVAL);
1320 if (encini) { /* validate key length */
1321 switch (encini->cri_alg) {
1322 case CRYPTO_DES_CBC:
1323 if (encini->cri_klen != 64)
1324 return (EINVAL);
1325 break;
1326 case CRYPTO_3DES_CBC:
1327 if (encini->cri_klen != 192)
1328 return (EINVAL);
1329 break;
1330 case CRYPTO_AES_CBC:
1331 if (encini->cri_klen != 128 &&
1332 encini->cri_klen != 192 &&
1333 encini->cri_klen != 256)
1334 return (EINVAL);
1335 break;
1336 }
1337 }
1338
1339 if (sc->sc_sessions == NULL) {
1340 ses = sc->sc_sessions = (struct safe_session *)malloc(
1341 sizeof(struct safe_session), M_DEVBUF, M_NOWAIT);
1342 if (ses == NULL)
1343 return (ENOMEM);
1344 sesn = 0;
1345 sc->sc_nsessions = 1;
1346 } else {
1347 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
1348 if (sc->sc_sessions[sesn].ses_used == 0) {
1349 ses = &sc->sc_sessions[sesn];
1350 break;
1351 }
1352 }
1353
1354 if (ses == NULL) {
1355 sesn = sc->sc_nsessions;
1356 ses = (struct safe_session *)malloc((sesn + 1) *
1357 sizeof(struct safe_session), M_DEVBUF, M_NOWAIT);
1358 if (ses == NULL)
1359 return (ENOMEM);
1360 bcopy(sc->sc_sessions, ses, sesn *
1361 sizeof(struct safe_session));
1362 bzero(sc->sc_sessions, sesn *
1363 sizeof(struct safe_session));
1364 free(sc->sc_sessions, M_DEVBUF);
1365 sc->sc_sessions = ses;
1366 ses = &sc->sc_sessions[sesn];
1367 sc->sc_nsessions++;
1368 }
1369 }
1370
1371 bzero(ses, sizeof(struct safe_session));
1372 ses->ses_used = 1;
1373
1374 if (encini) {
1375 /* get an IV */
1376 get_random_bytes(ses->ses_iv, sizeof(ses->ses_iv));
1377
1378 ses->ses_klen = encini->cri_klen;
1379 bcopy(encini->cri_key, ses->ses_key, ses->ses_klen / 8);
1380
1381 for (i = 0;
1382 i < sizeof(ses->ses_key)/sizeof(ses->ses_key[0]); i++)
1383 ses->ses_key[i] = htole32(ses->ses_key[i]);
1384 }
1385
1386 if (macini) {
1387 for (i = 0; i < macini->cri_klen / 8; i++)
1388 macini->cri_key[i] ^= HMAC_IPAD_VAL;
1389
1390 if (macini->cri_alg == CRYPTO_MD5_HMAC) {
1391 MD5Init(&md5ctx);
1392 MD5Update(&md5ctx, macini->cri_key,
1393 macini->cri_klen / 8);
1394 MD5Update(&md5ctx, hmac_ipad_buffer,
1395 HMAC_BLOCK_LEN - (macini->cri_klen / 8));
1396 bcopy(md5ctx.state, ses->ses_hminner,
1397 sizeof(md5ctx.state));
1398 } else {
1399 SHA1Init(&sha1ctx);
1400 SHA1Update(&sha1ctx, macini->cri_key,
1401 macini->cri_klen / 8);
1402 SHA1Update(&sha1ctx, hmac_ipad_buffer,
1403 HMAC_BLOCK_LEN - (macini->cri_klen / 8));
1404 bcopy(sha1ctx.state, ses->ses_hminner,
1405 sizeof(sha1ctx.state));
1406 }
1407
1408 for (i = 0; i < macini->cri_klen / 8; i++)
1409 macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
1410
1411 if (macini->cri_alg == CRYPTO_MD5_HMAC) {
1412 MD5Init(&md5ctx);
1413 MD5Update(&md5ctx, macini->cri_key,
1414 macini->cri_klen / 8);
1415 MD5Update(&md5ctx, hmac_opad_buffer,
1416 HMAC_BLOCK_LEN - (macini->cri_klen / 8));
1417 bcopy(md5ctx.state, ses->ses_hmouter,
1418 sizeof(md5ctx.state));
1419 } else {
1420 SHA1Init(&sha1ctx);
1421 SHA1Update(&sha1ctx, macini->cri_key,
1422 macini->cri_klen / 8);
1423 SHA1Update(&sha1ctx, hmac_opad_buffer,
1424 HMAC_BLOCK_LEN - (macini->cri_klen / 8));
1425 bcopy(sha1ctx.state, ses->ses_hmouter,
1426 sizeof(sha1ctx.state));
1427 }
1428
1429 for (i = 0; i < macini->cri_klen / 8; i++)
1430 macini->cri_key[i] ^= HMAC_OPAD_VAL;
1431
1432 /* PE is little-endian, insure proper byte order */
1433 for (i = 0;
1434 i < sizeof(ses->ses_hminner)/sizeof(ses->ses_hminner[0]);
1435 i++) {
1436 ses->ses_hminner[i] = htole32(ses->ses_hminner[i]);
1437 ses->ses_hmouter[i] = htole32(ses->ses_hmouter[i]);
1438 }
1439 }
1440
1441 *sidp = SAFE_SID(sc->sc_dev.dv_unit, sesn);
1442 return (0);
1443 }
1444
1445 /*
1446 * Deallocate a session.
1447 */
1448 int
safe_freesession(u_int64_t tid)1449 safe_freesession(u_int64_t tid)
1450 {
1451 struct safe_softc *sc;
1452 int session, ret, card;
1453 u_int32_t sid = ((u_int32_t) tid) & 0xffffffff;
1454
1455 card = SAFE_CARD(sid);
1456 if (card >= safe_cd.cd_ndevs || safe_cd.cd_devs[card] == NULL)
1457 return (EINVAL);
1458 sc = safe_cd.cd_devs[card];
1459
1460 if (sc == NULL)
1461 return (EINVAL);
1462
1463 session = SAFE_SESSION(sid);
1464 if (session < sc->sc_nsessions) {
1465 bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session]));
1466 ret = 0;
1467 } else
1468 ret = EINVAL;
1469 return (ret);
1470 }
1471
1472 /*
1473 * Is the operand suitable aligned for direct DMA. Each
1474 * segment must be aligned on a 32-bit boundary and all
1475 * but the last segment must be a multiple of 4 bytes.
1476 */
1477 int
safe_dmamap_aligned(const struct safe_operand * op)1478 safe_dmamap_aligned(const struct safe_operand *op)
1479 {
1480 int i;
1481
1482 for (i = 0; i < op->map->dm_nsegs; i++) {
1483 if (op->map->dm_segs[i].ds_addr & 3)
1484 return (0);
1485 if (i != (op->map->dm_nsegs - 1) &&
1486 (op->map->dm_segs[i].ds_len & 3))
1487 return (0);
1488 }
1489 return (1);
1490 }
1491
1492 /*
1493 * Clean up after a chip crash.
1494 * It is assumed that the caller in splimp()
1495 */
1496 void
safe_cleanchip(struct safe_softc * sc)1497 safe_cleanchip(struct safe_softc *sc)
1498 {
1499
1500 if (sc->sc_nqchip != 0) {
1501 struct safe_ringentry *re = sc->sc_back;
1502
1503 while (re != sc->sc_front) {
1504 if (re->re_desc.d_csr != 0)
1505 safe_free_entry(sc, re);
1506 if (++re == sc->sc_ringtop)
1507 re = sc->sc_ring;
1508 }
1509 sc->sc_back = re;
1510 sc->sc_nqchip = 0;
1511 }
1512 }
1513
1514 /*
1515 * free a safe_q
1516 * It is assumed that the caller is within splimp().
1517 */
1518 int
safe_free_entry(struct safe_softc * sc,struct safe_ringentry * re)1519 safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re)
1520 {
1521 struct cryptop *crp;
1522
1523 /*
1524 * Free header MCR
1525 */
1526 if ((re->re_dst_m != NULL) && (re->re_src_m != re->re_dst_m))
1527 m_freem(re->re_dst_m);
1528
1529 crp = (struct cryptop *)re->re_crp;
1530
1531 re->re_desc.d_csr = 0;
1532
1533 crp->crp_etype = EFAULT;
1534 crypto_done(crp);
1535 return (0);
1536 }
1537
1538 /*
1539 * safe_feed() - post a request to chip
1540 */
1541 void
safe_feed(struct safe_softc * sc,struct safe_ringentry * re)1542 safe_feed(struct safe_softc *sc, struct safe_ringentry *re)
1543 {
1544 bus_dmamap_sync(sc->sc_dmat, re->re_src_map,
1545 0, re->re_src_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
1546 if (re->re_dst_map != NULL)
1547 bus_dmamap_sync(sc->sc_dmat, re->re_dst_map, 0,
1548 re->re_dst_map->dm_mapsize, BUS_DMASYNC_PREREAD);
1549 /* XXX have no smaller granularity */
1550 safe_dma_sync(sc, &sc->sc_ringalloc,
1551 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1552 safe_dma_sync(sc, &sc->sc_spalloc, BUS_DMASYNC_PREWRITE);
1553 safe_dma_sync(sc, &sc->sc_dpalloc, BUS_DMASYNC_PREWRITE);
1554
1555 #ifdef SAFE_DEBUG
1556 if (safe_debug) {
1557 safe_dump_ringstate(sc, __func__);
1558 safe_dump_request(sc, __func__, re);
1559 }
1560 #endif
1561 sc->sc_nqchip++;
1562 if (sc->sc_nqchip > safestats.st_maxqchip)
1563 safestats.st_maxqchip = sc->sc_nqchip;
1564 /* poke h/w to check descriptor ring, any value can be written */
1565 WRITE_REG(sc, SAFE_HI_RD_DESCR, 0);
1566 }
1567
1568 /*
1569 * Is the operand suitable for direct DMA as the destination
1570 * of an operation. The hardware requires that each ``particle''
1571 * but the last in an operation result have the same size. We
1572 * fix that size at SAFE_MAX_DSIZE bytes. This routine returns
1573 * 0 if some segment is not a multiple of of this size, 1 if all
1574 * segments are exactly this size, or 2 if segments are at worst
1575 * a multple of this size.
1576 */
1577 int
safe_dmamap_uniform(const struct safe_operand * op)1578 safe_dmamap_uniform(const struct safe_operand *op)
1579 {
1580 int result = 1, i;
1581
1582 if (op->map->dm_nsegs <= 0)
1583 return (result);
1584
1585 for (i = 0; i < op->map->dm_nsegs-1; i++) {
1586 if (op->map->dm_segs[i].ds_len % SAFE_MAX_DSIZE)
1587 return (0);
1588 if (op->map->dm_segs[i].ds_len != SAFE_MAX_DSIZE)
1589 result = 2;
1590 }
1591 return (result);
1592 }
1593
1594 /*
1595 * Copy all data past offset from srcm to dstm.
1596 */
1597 void
safe_mcopy(struct mbuf * srcm,struct mbuf * dstm,u_int offset)1598 safe_mcopy(struct mbuf *srcm, struct mbuf *dstm, u_int offset)
1599 {
1600 u_int j, dlen, slen;
1601 caddr_t dptr, sptr;
1602
1603 /*
1604 * Advance src and dst to offset.
1605 */
1606 j = offset;
1607 while (j >= 0) {
1608 if (srcm->m_len > j)
1609 break;
1610 j -= srcm->m_len;
1611 srcm = srcm->m_next;
1612 if (srcm == NULL)
1613 return;
1614 }
1615 sptr = mtod(srcm, caddr_t) + j;
1616 slen = srcm->m_len - j;
1617
1618 j = offset;
1619 while (j >= 0) {
1620 if (dstm->m_len > j)
1621 break;
1622 j -= dstm->m_len;
1623 dstm = dstm->m_next;
1624 if (dstm == NULL)
1625 return;
1626 }
1627 dptr = mtod(dstm, caddr_t) + j;
1628 dlen = dstm->m_len - j;
1629
1630 /*
1631 * Copy everything that remains.
1632 */
1633 for (;;) {
1634 j = min(slen, dlen);
1635 bcopy(sptr, dptr, j);
1636 if (slen == j) {
1637 srcm = srcm->m_next;
1638 if (srcm == NULL)
1639 return;
1640 sptr = srcm->m_data;
1641 slen = srcm->m_len;
1642 } else
1643 sptr += j, slen -= j;
1644 if (dlen == j) {
1645 dstm = dstm->m_next;
1646 if (dstm == NULL)
1647 return;
1648 dptr = dstm->m_data;
1649 dlen = dstm->m_len;
1650 } else
1651 dptr += j, dlen -= j;
1652 }
1653 }
1654
1655 void
safe_callback(struct safe_softc * sc,struct safe_ringentry * re)1656 safe_callback(struct safe_softc *sc, struct safe_ringentry *re)
1657 {
1658 struct cryptop *crp = (struct cryptop *)re->re_crp;
1659 struct cryptodesc *crd;
1660
1661 safestats.st_opackets++;
1662 safestats.st_obytes += (re->re_dst_map == NULL) ?
1663 re->re_src_mapsize : re->re_dst_mapsize;
1664
1665 safe_dma_sync(sc, &sc->sc_ringalloc,
1666 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
1667 if (re->re_desc.d_csr & SAFE_PE_CSR_STATUS) {
1668 printf("%s: csr 0x%x cmd0 0x%x cmd1 0x%x\n",
1669 sc->sc_dev.dv_xname, re->re_desc.d_csr,
1670 re->re_sa.sa_cmd0, re->re_sa.sa_cmd1);
1671 safestats.st_peoperr++;
1672 crp->crp_etype = EIO; /* something more meaningful? */
1673 }
1674 if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {
1675 bus_dmamap_sync(sc->sc_dmat, re->re_dst_map, 0,
1676 re->re_dst_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
1677 bus_dmamap_unload(sc->sc_dmat, re->re_dst_map);
1678 bus_dmamap_destroy(sc->sc_dmat, re->re_dst_map);
1679 }
1680 bus_dmamap_sync(sc->sc_dmat, re->re_src_map, 0,
1681 re->re_src_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
1682 bus_dmamap_unload(sc->sc_dmat, re->re_src_map);
1683 bus_dmamap_destroy(sc->sc_dmat, re->re_src_map);
1684
1685 /*
1686 * If result was written to a different mbuf chain, swap
1687 * it in as the return value and reclaim the original.
1688 */
1689 if ((crp->crp_flags & CRYPTO_F_IMBUF) && re->re_src_m != re->re_dst_m) {
1690 m_freem(re->re_src_m);
1691 crp->crp_buf = (caddr_t)re->re_dst_m;
1692 }
1693
1694 if (re->re_flags & SAFE_QFLAGS_COPYOUTIV) {
1695 /* copy out IV for future use */
1696 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1697 int ivsize;
1698
1699 if (crd->crd_alg == CRYPTO_DES_CBC ||
1700 crd->crd_alg == CRYPTO_3DES_CBC) {
1701 ivsize = 2*sizeof(u_int32_t);
1702 } else if (crd->crd_alg == CRYPTO_AES_CBC) {
1703 ivsize = 4*sizeof(u_int32_t);
1704 } else
1705 continue;
1706 if (crp->crp_flags & CRYPTO_F_IMBUF) {
1707 m_copydata((struct mbuf *)crp->crp_buf,
1708 crd->crd_skip + crd->crd_len - ivsize,
1709 ivsize,
1710 (caddr_t) sc->sc_sessions[re->re_sesn].ses_iv);
1711 } else if (crp->crp_flags & CRYPTO_F_IOV) {
1712 cuio_copydata((struct uio *)crp->crp_buf,
1713 crd->crd_skip + crd->crd_len - ivsize,
1714 ivsize,
1715 (caddr_t)sc->sc_sessions[re->re_sesn].ses_iv);
1716 }
1717 break;
1718 }
1719 }
1720
1721 if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) {
1722 /* copy out ICV result */
1723 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1724 if (!(crd->crd_alg == CRYPTO_MD5_HMAC ||
1725 crd->crd_alg == CRYPTO_SHA1_HMAC))
1726 continue;
1727 if (crd->crd_alg == CRYPTO_SHA1_HMAC) {
1728 /*
1729 * SHA-1 ICV's are byte-swapped; fix 'em up
1730 * before copy them to their destination.
1731 */
1732 bswap32(re->re_sastate.sa_saved_indigest[0]);
1733 bswap32(re->re_sastate.sa_saved_indigest[1]);
1734 bswap32(re->re_sastate.sa_saved_indigest[2]);
1735 }
1736 if (crp->crp_flags & CRYPTO_F_IMBUF) {
1737 m_copyback((struct mbuf *)crp->crp_buf,
1738 crd->crd_inject, 12,
1739 (caddr_t)re->re_sastate.sa_saved_indigest);
1740 } else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac) {
1741 bcopy((caddr_t)re->re_sastate.sa_saved_indigest,
1742 crp->crp_mac, 12);
1743 }
1744 break;
1745 }
1746 }
1747
1748 crypto_done(crp);
1749 }
1750
1751 /*
1752 * SafeXcel Interrupt routine
1753 */
1754 int
safe_intr(void * arg)1755 safe_intr(void *arg)
1756 {
1757 struct safe_softc *sc = arg;
1758 volatile u_int32_t stat;
1759
1760 stat = READ_REG(sc, SAFE_HM_STAT);
1761 if (stat == 0) /* shared irq, not for us */
1762 return (0);
1763
1764 WRITE_REG(sc, SAFE_HI_CLR, stat); /* IACK */
1765
1766 if ((stat & SAFE_INT_PE_DDONE)) {
1767 /*
1768 * Descriptor(s) done; scan the ring and
1769 * process completed operations.
1770 */
1771 while (sc->sc_back != sc->sc_front) {
1772 struct safe_ringentry *re = sc->sc_back;
1773 #ifdef SAFE_DEBUG
1774 if (safe_debug) {
1775 safe_dump_ringstate(sc, __func__);
1776 safe_dump_request(sc, __func__, re);
1777 }
1778 #endif
1779 /*
1780 * safe_process marks ring entries that were allocated
1781 * but not used with a csr of zero. This insures the
1782 * ring front pointer never needs to be set backwards
1783 * in the event that an entry is allocated but not used
1784 * because of a setup error.
1785 */
1786 if (re->re_desc.d_csr != 0) {
1787 if (!SAFE_PE_CSR_IS_DONE(re->re_desc.d_csr))
1788 break;
1789 if (!SAFE_PE_LEN_IS_DONE(re->re_desc.d_len))
1790 break;
1791 sc->sc_nqchip--;
1792 safe_callback(sc, re);
1793 }
1794 if (++(sc->sc_back) == sc->sc_ringtop)
1795 sc->sc_back = sc->sc_ring;
1796 }
1797 }
1798
1799 return (1);
1800 }
1801
1802 struct safe_softc *
safe_kfind(struct cryptkop * krp)1803 safe_kfind(struct cryptkop *krp)
1804 {
1805 struct safe_softc *sc;
1806 int i;
1807
1808 for (i = 0; i < safe_cd.cd_ndevs; i++) {
1809 sc = safe_cd.cd_devs[i];
1810 if (sc == NULL)
1811 continue;
1812 if (sc->sc_cid == krp->krp_hid)
1813 return (sc);
1814 }
1815 return (NULL);
1816 }
1817
1818 int
safe_kprocess(struct cryptkop * krp)1819 safe_kprocess(struct cryptkop *krp)
1820 {
1821 struct safe_softc *sc;
1822 struct safe_pkq *q;
1823 int s;
1824
1825 if ((sc = safe_kfind(krp)) == NULL) {
1826 krp->krp_status = EINVAL;
1827 goto err;
1828 }
1829
1830 if (krp->krp_op != CRK_MOD_EXP) {
1831 krp->krp_status = EOPNOTSUPP;
1832 goto err;
1833 }
1834
1835 q = (struct safe_pkq *)malloc(sizeof(*q), M_DEVBUF, M_NOWAIT);
1836 if (q == NULL) {
1837 krp->krp_status = ENOMEM;
1838 goto err;
1839 }
1840 q->pkq_krp = krp;
1841
1842 s = splnet();
1843 SIMPLEQ_INSERT_TAIL(&sc->sc_pkq, q, pkq_next);
1844 safe_kfeed(sc);
1845 splx(s);
1846 return (0);
1847
1848 err:
1849 crypto_kdone(krp);
1850 return (0);
1851 }
1852
1853 #define SAFE_CRK_PARAM_BASE 0
1854 #define SAFE_CRK_PARAM_EXP 1
1855 #define SAFE_CRK_PARAM_MOD 2
1856
1857 int
safe_kstart(struct safe_softc * sc)1858 safe_kstart(struct safe_softc *sc)
1859 {
1860 struct cryptkop *krp = sc->sc_pkq_cur->pkq_krp;
1861 int exp_bits, mod_bits, base_bits;
1862 u_int32_t op, a_off, b_off, c_off, d_off;
1863
1864 if (krp->krp_iparams < 3 || krp->krp_oparams != 1) {
1865 krp->krp_status = EINVAL;
1866 return (1);
1867 }
1868
1869 base_bits = safe_ksigbits(&krp->krp_param[SAFE_CRK_PARAM_BASE]);
1870 if (base_bits > 2048)
1871 goto too_big;
1872 if (base_bits <= 0) /* 5. base not zero */
1873 goto too_small;
1874
1875 exp_bits = safe_ksigbits(&krp->krp_param[SAFE_CRK_PARAM_EXP]);
1876 if (exp_bits > 2048)
1877 goto too_big;
1878 if (exp_bits <= 0) /* 1. exponent word length > 0 */
1879 goto too_small; /* 4. exponent not zero */
1880
1881 mod_bits = safe_ksigbits(&krp->krp_param[SAFE_CRK_PARAM_MOD]);
1882 if (mod_bits > 2048)
1883 goto too_big;
1884 if (mod_bits <= 32) /* 2. modulus word length > 1 */
1885 goto too_small; /* 8. MSW of modulus != zero */
1886 if (mod_bits < exp_bits) /* 3 modulus len >= exponent len */
1887 goto too_small;
1888 if ((krp->krp_param[SAFE_CRK_PARAM_MOD].crp_p[0] & 1) == 0)
1889 goto bad_domain; /* 6. modulus is odd */
1890 if (mod_bits > krp->krp_param[krp->krp_iparams].crp_nbits)
1891 goto too_small; /* make sure result will fit */
1892
1893 /* 7. modulus > base */
1894 if (mod_bits < base_bits)
1895 goto too_small;
1896 if (mod_bits == base_bits) {
1897 u_int8_t *basep, *modp;
1898 int i;
1899
1900 basep = krp->krp_param[SAFE_CRK_PARAM_BASE].crp_p +
1901 ((base_bits + 7) / 8) - 1;
1902 modp = krp->krp_param[SAFE_CRK_PARAM_MOD].crp_p +
1903 ((mod_bits + 7) / 8) - 1;
1904
1905 for (i = 0; i < (mod_bits + 7) / 8; i++, basep--, modp--) {
1906 if (*modp < *basep)
1907 goto too_small;
1908 if (*modp > *basep)
1909 break;
1910 }
1911 }
1912
1913 /* And on the 9th step, he rested. */
1914
1915 WRITE_REG(sc, SAFE_PK_A_LEN, (exp_bits + 31) / 32);
1916 WRITE_REG(sc, SAFE_PK_B_LEN, (mod_bits + 31) / 32);
1917 if (mod_bits > 1024) {
1918 op = SAFE_PK_FUNC_EXP4;
1919 a_off = 0x000;
1920 b_off = 0x100;
1921 c_off = 0x200;
1922 d_off = 0x300;
1923 } else {
1924 op = SAFE_PK_FUNC_EXP16;
1925 a_off = 0x000;
1926 b_off = 0x080;
1927 c_off = 0x100;
1928 d_off = 0x180;
1929 }
1930 sc->sc_pk_reslen = b_off - a_off;
1931 sc->sc_pk_resoff = d_off;
1932
1933 /* A is exponent, B is modulus, C is base, D is result */
1934 safe_kload_reg(sc, a_off, b_off - a_off,
1935 &krp->krp_param[SAFE_CRK_PARAM_EXP]);
1936 WRITE_REG(sc, SAFE_PK_A_ADDR, a_off >> 2);
1937 safe_kload_reg(sc, b_off, b_off - a_off,
1938 &krp->krp_param[SAFE_CRK_PARAM_MOD]);
1939 WRITE_REG(sc, SAFE_PK_B_ADDR, b_off >> 2);
1940 safe_kload_reg(sc, c_off, b_off - a_off,
1941 &krp->krp_param[SAFE_CRK_PARAM_BASE]);
1942 WRITE_REG(sc, SAFE_PK_C_ADDR, c_off >> 2);
1943 WRITE_REG(sc, SAFE_PK_D_ADDR, d_off >> 2);
1944
1945 WRITE_REG(sc, SAFE_PK_FUNC, op | SAFE_PK_FUNC_RUN);
1946
1947 return (0);
1948
1949 too_big:
1950 krp->krp_status = E2BIG;
1951 return (1);
1952 too_small:
1953 krp->krp_status = ERANGE;
1954 return (1);
1955 bad_domain:
1956 krp->krp_status = EDOM;
1957 return (1);
1958 }
1959
1960 int
safe_ksigbits(struct crparam * cr)1961 safe_ksigbits(struct crparam *cr)
1962 {
1963 u_int plen = (cr->crp_nbits + 7) / 8;
1964 int i, sig = plen * 8;
1965 u_int8_t c, *p = cr->crp_p;
1966
1967 for (i = plen - 1; i >= 0; i--) {
1968 c = p[i];
1969 if (c != 0) {
1970 while ((c & 0x80) == 0) {
1971 sig--;
1972 c <<= 1;
1973 }
1974 break;
1975 }
1976 sig -= 8;
1977 }
1978 return (sig);
1979 }
1980
1981 void
safe_kfeed(struct safe_softc * sc)1982 safe_kfeed(struct safe_softc *sc)
1983 {
1984 if (SIMPLEQ_EMPTY(&sc->sc_pkq) && sc->sc_pkq_cur == NULL)
1985 return;
1986 if (sc->sc_pkq_cur != NULL)
1987 return;
1988 while (!SIMPLEQ_EMPTY(&sc->sc_pkq)) {
1989 struct safe_pkq *q = SIMPLEQ_FIRST(&sc->sc_pkq);
1990
1991 sc->sc_pkq_cur = q;
1992 SIMPLEQ_REMOVE_HEAD(&sc->sc_pkq, pkq_next);
1993 if (safe_kstart(sc) != 0) {
1994 crypto_kdone(q->pkq_krp);
1995 free(q, M_DEVBUF);
1996 sc->sc_pkq_cur = NULL;
1997 } else {
1998 /* op started, start polling */
1999 timeout_add(&sc->sc_pkto, 1);
2000 break;
2001 }
2002 }
2003 }
2004
2005 void
safe_kpoll(void * vsc)2006 safe_kpoll(void *vsc)
2007 {
2008 struct safe_softc *sc = vsc;
2009 struct safe_pkq *q;
2010 struct crparam *res;
2011 int s, i;
2012 u_int32_t buf[64];
2013
2014 s = splnet();
2015 if (sc->sc_pkq_cur == NULL)
2016 goto out;
2017 if (READ_REG(sc, SAFE_PK_FUNC) & SAFE_PK_FUNC_RUN) {
2018 /* still running, check back later */
2019 timeout_add(&sc->sc_pkto, 1);
2020 goto out;
2021 }
2022
2023 q = sc->sc_pkq_cur;
2024 res = &q->pkq_krp->krp_param[q->pkq_krp->krp_iparams];
2025 bzero(buf, sizeof(buf));
2026 bzero(res->crp_p, (res->crp_nbits + 7) / 8);
2027 for (i = 0; i < sc->sc_pk_reslen >> 2; i++)
2028 buf[i] = letoh32(READ_REG(sc, SAFE_PK_RAM_START +
2029 sc->sc_pk_resoff + (i << 2)));
2030 bcopy(buf, res->crp_p, (res->crp_nbits + 7) / 8);
2031 res->crp_nbits = sc->sc_pk_reslen * 8;
2032 res->crp_nbits = safe_ksigbits(res);
2033
2034 for (i = SAFE_PK_RAM_START; i < SAFE_PK_RAM_END; i += 4)
2035 WRITE_REG(sc, i, 0);
2036
2037 crypto_kdone(q->pkq_krp);
2038 free(q, M_DEVBUF);
2039 sc->sc_pkq_cur = NULL;
2040
2041 safe_kfeed(sc);
2042 out:
2043 splx(s);
2044 }
2045
2046 void
safe_kload_reg(struct safe_softc * sc,u_int32_t off,u_int32_t len,struct crparam * n)2047 safe_kload_reg(struct safe_softc *sc, u_int32_t off, u_int32_t len,
2048 struct crparam *n)
2049 {
2050 u_int32_t buf[64], i;
2051
2052 bzero(buf, sizeof(buf));
2053 bcopy(n->crp_p, buf, (n->crp_nbits + 7) / 8);
2054
2055 for (i = 0; i < len >> 2; i++)
2056 WRITE_REG(sc, SAFE_PK_RAM_START + off + (i << 2),
2057 htole32(buf[i]));
2058 }
2059
2060 #ifdef SAFE_DEBUG
2061
2062 void
safe_dump_dmastatus(struct safe_softc * sc,const char * tag)2063 safe_dump_dmastatus(struct safe_softc *sc, const char *tag)
2064 {
2065 printf("%s: ENDIAN 0x%x SRC 0x%x DST 0x%x STAT 0x%x\n", tag,
2066 READ_REG(sc, SAFE_DMA_ENDIAN), READ_REG(sc, SAFE_DMA_SRCADDR),
2067 READ_REG(sc, SAFE_DMA_DSTADDR), READ_REG(sc, SAFE_DMA_STAT));
2068 }
2069
2070 void
safe_dump_intrstate(struct safe_softc * sc,const char * tag)2071 safe_dump_intrstate(struct safe_softc *sc, const char *tag)
2072 {
2073 printf("%s: HI_CFG 0x%x HI_MASK 0x%x HI_DESC_CNT 0x%x HU_STAT 0x%x HM_STAT 0x%x\n",
2074 tag, READ_REG(sc, SAFE_HI_CFG), READ_REG(sc, SAFE_HI_MASK),
2075 READ_REG(sc, SAFE_HI_DESC_CNT), READ_REG(sc, SAFE_HU_STAT),
2076 READ_REG(sc, SAFE_HM_STAT));
2077 }
2078
2079 void
safe_dump_ringstate(struct safe_softc * sc,const char * tag)2080 safe_dump_ringstate(struct safe_softc *sc, const char *tag)
2081 {
2082 u_int32_t estat = READ_REG(sc, SAFE_PE_ERNGSTAT);
2083
2084 /* NB: assume caller has lock on ring */
2085 printf("%s: ERNGSTAT %x (next %u) back %u front %u\n",
2086 tag, estat, (estat >> SAFE_PE_ERNGSTAT_NEXT_S),
2087 sc->sc_back - sc->sc_ring, sc->sc_front - sc->sc_ring);
2088 }
2089
2090 void
safe_dump_request(struct safe_softc * sc,const char * tag,struct safe_ringentry * re)2091 safe_dump_request(struct safe_softc *sc, const char* tag, struct safe_ringentry *re)
2092 {
2093 int ix, nsegs;
2094
2095 ix = re - sc->sc_ring;
2096 printf("%s: %p (%u): csr %x src %x dst %x sa %x len %x\n", tag,
2097 re, ix, re->re_desc.d_csr, re->re_desc.d_src, re->re_desc.d_dst,
2098 re->re_desc.d_sa, re->re_desc.d_len);
2099 if (re->re_src_nsegs > 1) {
2100 ix = (re->re_desc.d_src - sc->sc_spalloc.dma_paddr) /
2101 sizeof(struct safe_pdesc);
2102 for (nsegs = re->re_src_nsegs; nsegs; nsegs--) {
2103 printf(" spd[%u] %p: %p size %u flags %x", ix,
2104 &sc->sc_spring[ix],
2105 (caddr_t)sc->sc_spring[ix].pd_addr,
2106 sc->sc_spring[ix].pd_size,
2107 sc->sc_spring[ix].pd_flags);
2108 if (sc->sc_spring[ix].pd_size == 0)
2109 printf(" (zero!)");
2110 printf("\n");
2111 if (++ix == SAFE_TOTAL_SPART)
2112 ix = 0;
2113 }
2114 }
2115 if (re->re_dst_nsegs > 1) {
2116 ix = (re->re_desc.d_dst - sc->sc_dpalloc.dma_paddr) /
2117 sizeof(struct safe_pdesc);
2118 for (nsegs = re->re_dst_nsegs; nsegs; nsegs--) {
2119 printf(" dpd[%u] %p: %p flags %x\n", ix,
2120 &sc->sc_dpring[ix],
2121 (caddr_t) sc->sc_dpring[ix].pd_addr,
2122 sc->sc_dpring[ix].pd_flags);
2123 if (++ix == SAFE_TOTAL_DPART)
2124 ix = 0;
2125 }
2126 }
2127 printf("sa: cmd0 %08x cmd1 %08x staterec %x\n",
2128 re->re_sa.sa_cmd0, re->re_sa.sa_cmd1, re->re_sa.sa_staterec);
2129 printf("sa: key %x %x %x %x %x %x %x %x\n", re->re_sa.sa_key[0],
2130 re->re_sa.sa_key[1], re->re_sa.sa_key[2], re->re_sa.sa_key[3],
2131 re->re_sa.sa_key[4], re->re_sa.sa_key[5], re->re_sa.sa_key[6],
2132 re->re_sa.sa_key[7]);
2133 printf("sa: indigest %x %x %x %x %x\n", re->re_sa.sa_indigest[0],
2134 re->re_sa.sa_indigest[1], re->re_sa.sa_indigest[2],
2135 re->re_sa.sa_indigest[3], re->re_sa.sa_indigest[4]);
2136 printf("sa: outdigest %x %x %x %x %x\n", re->re_sa.sa_outdigest[0],
2137 re->re_sa.sa_outdigest[1], re->re_sa.sa_outdigest[2],
2138 re->re_sa.sa_outdigest[3], re->re_sa.sa_outdigest[4]);
2139 printf("sr: iv %x %x %x %x\n",
2140 re->re_sastate.sa_saved_iv[0], re->re_sastate.sa_saved_iv[1],
2141 re->re_sastate.sa_saved_iv[2], re->re_sastate.sa_saved_iv[3]);
2142 printf("sr: hashbc %u indigest %x %x %x %x %x\n",
2143 re->re_sastate.sa_saved_hashbc,
2144 re->re_sastate.sa_saved_indigest[0],
2145 re->re_sastate.sa_saved_indigest[1],
2146 re->re_sastate.sa_saved_indigest[2],
2147 re->re_sastate.sa_saved_indigest[3],
2148 re->re_sastate.sa_saved_indigest[4]);
2149 }
2150
2151 void
safe_dump_ring(struct safe_softc * sc,const char * tag)2152 safe_dump_ring(struct safe_softc *sc, const char *tag)
2153 {
2154 printf("\nSafeNet Ring State:\n");
2155 safe_dump_intrstate(sc, tag);
2156 safe_dump_dmastatus(sc, tag);
2157 safe_dump_ringstate(sc, tag);
2158 if (sc->sc_nqchip) {
2159 struct safe_ringentry *re = sc->sc_back;
2160 do {
2161 safe_dump_request(sc, tag, re);
2162 if (++re == sc->sc_ringtop)
2163 re = sc->sc_ring;
2164 } while (re != sc->sc_front);
2165 }
2166 }
2167
2168 #endif /* SAFE_DEBUG */
2169