1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2003-2012 Broadcom Corporation
5 * All Rights Reserved
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
25 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
26 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
27 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
28 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD: stable/12/sys/mips/nlm/dev/sec/nlmsec.c 336439 2018-07-18 00:56:25Z cem $");
33
34 #include <sys/cdefs.h>
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/proc.h>
38 #include <sys/errno.h>
39 #include <sys/malloc.h>
40 #include <sys/kernel.h>
41 #include <sys/module.h>
42 #include <sys/mbuf.h>
43 #include <sys/lock.h>
44 #include <sys/mutex.h>
45 #include <sys/sysctl.h>
46 #include <sys/bus.h>
47 #include <sys/random.h>
48 #include <sys/rman.h>
49 #include <sys/uio.h>
50 #include <sys/kobj.h>
51
52 #include <dev/pci/pcivar.h>
53
54 #include <opencrypto/cryptodev.h>
55
56 #include "cryptodev_if.h"
57
58 #include <vm/vm.h>
59 #include <vm/pmap.h>
60
61 #include <mips/nlm/hal/haldefs.h>
62 #include <mips/nlm/hal/iomap.h>
63 #include <mips/nlm/xlp.h>
64 #include <mips/nlm/hal/sys.h>
65 #include <mips/nlm/hal/fmn.h>
66 #include <mips/nlm/hal/nlmsaelib.h>
67 #include <mips/nlm/dev/sec/nlmseclib.h>
68 #include <mips/nlm/hal/cop2.h>
69 #include <mips/nlm/hal/mips-extns.h>
70 #include <mips/nlm/msgring.h>
71
72 unsigned int creditleft;
73
74 void xlp_sec_print_data(struct cryptop *crp);
75
76 static int xlp_sec_init(struct xlp_sec_softc *sc);
77 static int xlp_sec_newsession(device_t , crypto_session_t, struct cryptoini *);
78 static int xlp_sec_process(device_t , struct cryptop *, int);
79 static int xlp_copyiv(struct xlp_sec_softc *, struct xlp_sec_command *,
80 struct cryptodesc *enccrd);
81 static int xlp_get_nsegs(struct cryptop *, unsigned int *);
82 static int xlp_alloc_cmd_params(struct xlp_sec_command *, unsigned int);
83 static void xlp_free_cmd_params(struct xlp_sec_command *);
84
85 static int xlp_sec_probe(device_t);
86 static int xlp_sec_attach(device_t);
87 static int xlp_sec_detach(device_t);
88
89 static device_method_t xlp_sec_methods[] = {
90 /* device interface */
91 DEVMETHOD(device_probe, xlp_sec_probe),
92 DEVMETHOD(device_attach, xlp_sec_attach),
93 DEVMETHOD(device_detach, xlp_sec_detach),
94
95 /* bus interface */
96 DEVMETHOD(bus_print_child, bus_generic_print_child),
97 DEVMETHOD(bus_driver_added, bus_generic_driver_added),
98
99 /* crypto device methods */
100 DEVMETHOD(cryptodev_newsession, xlp_sec_newsession),
101 DEVMETHOD(cryptodev_process, xlp_sec_process),
102
103 DEVMETHOD_END
104 };
105
106 static driver_t xlp_sec_driver = {
107 "nlmsec",
108 xlp_sec_methods,
109 sizeof(struct xlp_sec_softc)
110 };
111 static devclass_t xlp_sec_devclass;
112
113 DRIVER_MODULE(nlmsec, pci, xlp_sec_driver, xlp_sec_devclass, 0, 0);
114 MODULE_DEPEND(nlmsec, crypto, 1, 1, 1);
115
116 void
117 nlm_xlpsec_msgring_handler(int vc, int size, int code, int src_id,
118 struct nlm_fmn_msg *msg, void *data);
119
120 #ifdef NLM_SEC_DEBUG
121
122 #define extract_bits(x, bitshift, bitcnt) \
123 (((unsigned long long)x >> bitshift) & ((1ULL << bitcnt) - 1))
124
125 void
print_crypto_params(struct xlp_sec_command * cmd,struct nlm_fmn_msg m)126 print_crypto_params(struct xlp_sec_command *cmd, struct nlm_fmn_msg m)
127 {
128 unsigned long long msg0,msg1,msg2,msg3,msg4,msg5,msg6,msg7,msg8;
129
130 msg0 = cmd->ctrlp->desc0;
131 msg1 = cmd->paramp->desc0;
132 msg2 = cmd->paramp->desc1;
133 msg3 = cmd->paramp->desc2;
134 msg4 = cmd->paramp->desc3;
135 msg5 = cmd->paramp->segment[0][0];
136 msg6 = cmd->paramp->segment[0][1];
137 msg7 = m.msg[0];
138 msg8 = m.msg[1];
139
140 printf("msg0 %llx msg1 %llx msg2 %llx msg3 %llx msg4 %llx msg5 %llx"
141 "msg6 %llx msg7 %llx msg8 %llx\n", msg0, msg1, msg2, msg3, msg4,
142 msg5, msg6, msg7, msg8);
143
144 printf("c0: hmac %d htype %d hmode %d ctype %d cmode %d arc4 %x\n",
145 (unsigned int)extract_bits(msg0, 61, 1),
146 (unsigned int)extract_bits(msg0, 52, 8),
147 (unsigned int)extract_bits(msg0, 43, 8),
148 (unsigned int)extract_bits(msg0, 34, 8),
149 (unsigned int)extract_bits(msg0, 25, 8),
150 (unsigned int)extract_bits(msg0, 0, 23));
151
152 printf("p0: tls %d hsrc %d hl3 %d enc %d ivl %d hd %llx\n",
153 (unsigned int)extract_bits(msg1, 63, 1),
154 (unsigned int)extract_bits(msg1,62,1),
155 (unsigned int)extract_bits(msg1,60,1),
156 (unsigned int)extract_bits(msg1,59,1),
157 (unsigned int)extract_bits(msg1,41,16), extract_bits(msg1,0,40));
158
159 printf("p1: clen %u hl %u\n", (unsigned int)extract_bits(msg2, 32, 32),
160 (unsigned int)extract_bits(msg2,0,32));
161
162 printf("p2: ivoff %d cbit %d coff %d hbit %d hclb %d hoff %d\n",
163 (unsigned int)extract_bits(msg3, 45, 17),
164 (unsigned int)extract_bits(msg3, 42,3),
165 (unsigned int)extract_bits(msg3, 22,16),
166 (unsigned int)extract_bits(msg3, 19,3),
167 (unsigned int)extract_bits(msg3, 18,1),
168 (unsigned int)extract_bits(msg3, 0, 16));
169
170 printf("p3: desfbid %d tlen %d arc4 %x hmacpad %d\n",
171 (unsigned int)extract_bits(msg4, 48,16),
172 (unsigned int)extract_bits(msg4,11,16),
173 (unsigned int)extract_bits(msg4,6,3),
174 (unsigned int)extract_bits(msg4,5,1));
175
176 printf("p4: sflen %d sddr %llx \n",
177 (unsigned int)extract_bits(msg5, 48, 16),extract_bits(msg5, 0, 40));
178
179 printf("p5: dflen %d cl3 %d cclob %d cdest %llx \n",
180 (unsigned int)extract_bits(msg6, 48, 16),
181 (unsigned int)extract_bits(msg6, 46, 1),
182 (unsigned int)extract_bits(msg6, 41, 1), extract_bits(msg6, 0, 40));
183
184 printf("fmn0: fbid %d dfrlen %d dfrv %d cklen %d cdescaddr %llx\n",
185 (unsigned int)extract_bits(msg7, 48, 16),
186 (unsigned int)extract_bits(msg7,46,2),
187 (unsigned int)extract_bits(msg7,45,1),
188 (unsigned int)extract_bits(msg7,40,5),
189 (extract_bits(msg7,0,34)<< 6));
190
191 printf("fmn1: arc4 %d hklen %d pdesclen %d pktdescad %llx\n",
192 (unsigned int)extract_bits(msg8, 63, 1),
193 (unsigned int)extract_bits(msg8,56,5),
194 (unsigned int)extract_bits(msg8,43,12),
195 (extract_bits(msg8,0,34) << 6));
196
197 return;
198 }
199
200 void
xlp_sec_print_data(struct cryptop * crp)201 xlp_sec_print_data(struct cryptop *crp)
202 {
203 int i, key_len;
204 struct cryptodesc *crp_desc;
205
206 printf("session = %p, crp_ilen = %d, crp_olen=%d \n", crp->crp_session,
207 crp->crp_ilen, crp->crp_olen);
208
209 printf("crp_flags = 0x%x\n", crp->crp_flags);
210
211 printf("crp buf:\n");
212 for (i = 0; i < crp->crp_ilen; i++) {
213 printf("%c ", crp->crp_buf[i]);
214 if (i % 10 == 0)
215 printf("\n");
216 }
217
218 printf("\n");
219 printf("****************** desc ****************\n");
220 crp_desc = crp->crp_desc;
221 printf("crd_skip=%d, crd_len=%d, crd_flags=0x%x, crd_alg=%d\n",
222 crp_desc->crd_skip, crp_desc->crd_len, crp_desc->crd_flags,
223 crp_desc->crd_alg);
224
225 key_len = crp_desc->crd_klen / 8;
226 printf("key(%d) :\n", key_len);
227 for (i = 0; i < key_len; i++)
228 printf("%d", crp_desc->crd_key[i]);
229 printf("\n");
230
231 printf(" IV : \n");
232 for (i = 0; i < EALG_MAX_BLOCK_LEN; i++)
233 printf("%d", crp_desc->crd_iv[i]);
234 printf("\n");
235
236 printf("crd_next=%p\n", crp_desc->crd_next);
237 return;
238 }
239
240 void
print_cmd(struct xlp_sec_command * cmd)241 print_cmd(struct xlp_sec_command *cmd)
242 {
243 printf("session_num :%d\n",cmd->session_num);
244 printf("crp :0x%x\n",(uint32_t)cmd->crp);
245 printf("enccrd :0x%x\n",(uint32_t)cmd->enccrd);
246 printf("maccrd :0x%x\n",(uint32_t)cmd->maccrd);
247 printf("ses :%d\n",(uint32_t)cmd->ses);
248 printf("ctrlp :0x%x\n",(uint32_t)cmd->ctrlp);
249 printf("paramp :0x%x\n",(uint32_t)cmd->paramp);
250 printf("hashdest :0x%x\n",(uint32_t)cmd->hashdest);
251 printf("hashsrc :%d\n",cmd->hashsrc);
252 printf("hmacpad :%d\n",cmd->hmacpad);
253 printf("hashoff :%d\n",cmd->hashoff);
254 printf("hashlen :%d\n",cmd->hashlen);
255 printf("cipheroff :%d\n",cmd->cipheroff);
256 printf("cipherlen :%d\n",cmd->cipherlen);
257 printf("ivoff :%d\n",cmd->ivoff);
258 printf("ivlen :%d\n",cmd->ivlen);
259 printf("hashalg :%d\n",cmd->hashalg);
260 printf("hashmode :%d\n",cmd->hashmode);
261 printf("cipheralg :%d\n",cmd->cipheralg);
262 printf("ciphermode :%d\n",cmd->ciphermode);
263 printf("nsegs :%d\n",cmd->nsegs);
264 printf("hash_dst_len :%d\n",cmd->hash_dst_len);
265 }
266 #endif /* NLM_SEC_DEBUG */
267
268 static int
xlp_sec_init(struct xlp_sec_softc * sc)269 xlp_sec_init(struct xlp_sec_softc *sc)
270 {
271
272 /* Register interrupt handler for the SEC CMS messages */
273 if (register_msgring_handler(sc->sec_vc_start,
274 sc->sec_vc_end, nlm_xlpsec_msgring_handler, sc) != 0) {
275 printf("Couldn't register sec msgring handler\n");
276 return (-1);
277 }
278
279 /* Do the CMS credit initialization */
280 /* Currently it is configured by default to 50 when kernel comes up */
281
282 return (0);
283 }
284
285 /* This function is called from an interrupt handler */
286 void
nlm_xlpsec_msgring_handler(int vc,int size,int code,int src_id,struct nlm_fmn_msg * msg,void * data)287 nlm_xlpsec_msgring_handler(int vc, int size, int code, int src_id,
288 struct nlm_fmn_msg *msg, void *data)
289 {
290 struct xlp_sec_command *cmd = NULL;
291 struct xlp_sec_softc *sc = NULL;
292 struct cryptodesc *crd = NULL;
293 unsigned int ivlen = 0;
294
295 KASSERT(code == FMN_SWCODE_CRYPTO,
296 ("%s: bad code = %d, expected code = %d\n", __FUNCTION__,
297 code, FMN_SWCODE_CRYPTO));
298
299 sc = (struct xlp_sec_softc *)data;
300 KASSERT(src_id >= sc->sec_vc_start && src_id <= sc->sec_vc_end,
301 ("%s: bad src_id = %d, expect %d - %d\n", __FUNCTION__,
302 src_id, sc->sec_vc_start, sc->sec_vc_end));
303
304 cmd = (struct xlp_sec_command *)(uintptr_t)msg->msg[0];
305 KASSERT(cmd != NULL && cmd->crp != NULL,
306 ("%s :cmd not received properly\n",__FUNCTION__));
307
308 KASSERT(CRYPTO_ERROR(msg->msg[1]) == 0,
309 ("%s: Message rcv msg0 %llx msg1 %llx err %x \n", __FUNCTION__,
310 (unsigned long long)msg->msg[0], (unsigned long long)msg->msg[1],
311 (int)CRYPTO_ERROR(msg->msg[1])));
312
313 crd = cmd->enccrd;
314 /* Copy the last 8 or 16 bytes to the session iv, so that in few
315 * cases this will be used as IV for the next request
316 */
317 if (crd != NULL) {
318 if ((crd->crd_alg == CRYPTO_DES_CBC ||
319 crd->crd_alg == CRYPTO_3DES_CBC ||
320 crd->crd_alg == CRYPTO_AES_CBC) &&
321 (crd->crd_flags & CRD_F_ENCRYPT)) {
322 ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ?
323 XLP_SEC_AES_IV_LENGTH : XLP_SEC_DES_IV_LENGTH);
324 crypto_copydata(cmd->crp->crp_flags, cmd->crp->crp_buf,
325 crd->crd_skip + crd->crd_len - ivlen, ivlen,
326 cmd->ses->ses_iv);
327 }
328 }
329
330 /* If there are not enough credits to send, then send request
331 * will fail with ERESTART and the driver will be blocked until it is
332 * unblocked here after knowing that there are sufficient credits to
333 * send the request again.
334 */
335 if (sc->sc_needwakeup) {
336 atomic_add_int(&creditleft, sc->sec_msgsz);
337 if (creditleft >= (NLM_CRYPTO_LEFT_REQS)) {
338 crypto_unblock(sc->sc_cid, sc->sc_needwakeup);
339 sc->sc_needwakeup &= (~(CRYPTO_SYMQ | CRYPTO_ASYMQ));
340 }
341 }
342 if(cmd->maccrd) {
343 crypto_copyback(cmd->crp->crp_flags,
344 cmd->crp->crp_buf, cmd->maccrd->crd_inject,
345 cmd->hash_dst_len, cmd->hashdest);
346 }
347
348 /* This indicates completion of the crypto operation */
349 crypto_done(cmd->crp);
350
351 xlp_free_cmd_params(cmd);
352
353 return;
354 }
355
356 static int
xlp_sec_probe(device_t dev)357 xlp_sec_probe(device_t dev)
358 {
359 struct xlp_sec_softc *sc;
360
361 if (pci_get_vendor(dev) == PCI_VENDOR_NETLOGIC &&
362 pci_get_device(dev) == PCI_DEVICE_ID_NLM_SAE) {
363 sc = device_get_softc(dev);
364 return (BUS_PROBE_DEFAULT);
365 }
366 return (ENXIO);
367 }
368
369 /*
370 * Attach an interface that successfully probed.
371 */
372 static int
xlp_sec_attach(device_t dev)373 xlp_sec_attach(device_t dev)
374 {
375 struct xlp_sec_softc *sc = device_get_softc(dev);
376 uint64_t base;
377 int qstart, qnum;
378 int freq, node;
379
380 sc->sc_dev = dev;
381
382 node = nlm_get_device_node(pci_get_slot(dev));
383 freq = nlm_set_device_frequency(node, DFS_DEVICE_SAE, 250);
384 if (bootverbose)
385 device_printf(dev, "SAE Freq: %dMHz\n", freq);
386 if(pci_get_device(dev) == PCI_DEVICE_ID_NLM_SAE) {
387 device_set_desc(dev, "XLP Security Accelerator");
388 sc->sc_cid = crypto_get_driverid(dev,
389 sizeof(struct xlp_sec_session), CRYPTOCAP_F_HARDWARE);
390 if (sc->sc_cid < 0) {
391 printf("xlp_sec - error : could not get the driver"
392 " id\n");
393 goto error_exit;
394 }
395 if (crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0) != 0)
396 printf("register failed for CRYPTO_DES_CBC\n");
397
398 if (crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0) != 0)
399 printf("register failed for CRYPTO_3DES_CBC\n");
400
401 if (crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0) != 0)
402 printf("register failed for CRYPTO_AES_CBC\n");
403
404 if (crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0) != 0)
405 printf("register failed for CRYPTO_ARC4\n");
406
407 if (crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0) != 0)
408 printf("register failed for CRYPTO_MD5\n");
409
410 if (crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0) != 0)
411 printf("register failed for CRYPTO_SHA1\n");
412
413 if (crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0) != 0)
414 printf("register failed for CRYPTO_MD5_HMAC\n");
415
416 if (crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0) != 0)
417 printf("register failed for CRYPTO_SHA1_HMAC\n");
418
419 base = nlm_get_sec_pcibase(node);
420 qstart = nlm_qidstart(base);
421 qnum = nlm_qnum(base);
422 sc->sec_vc_start = qstart;
423 sc->sec_vc_end = qstart + qnum - 1;
424 }
425
426 if (xlp_sec_init(sc) != 0)
427 goto error_exit;
428 if (bootverbose)
429 device_printf(dev, "SEC Initialization complete!\n");
430 return (0);
431
432 error_exit:
433 return (ENXIO);
434
435 }
436
437 /*
438 * Detach an interface that successfully probed.
439 */
440 static int
xlp_sec_detach(device_t dev)441 xlp_sec_detach(device_t dev)
442 {
443 return (0);
444 }
445
446 static int
xlp_sec_newsession(device_t dev,crypto_session_t cses,struct cryptoini * cri)447 xlp_sec_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
448 {
449 struct cryptoini *c;
450 struct xlp_sec_softc *sc = device_get_softc(dev);
451 int mac = 0, cry = 0;
452 struct xlp_sec_session *ses;
453 struct xlp_sec_command *cmd = NULL;
454
455 if (cri == NULL || sc == NULL)
456 return (EINVAL);
457
458 ses = crypto_get_driver_session(cses);
459 cmd = &ses->cmd;
460
461 for (c = cri; c != NULL; c = c->cri_next) {
462 switch (c->cri_alg) {
463 case CRYPTO_MD5:
464 case CRYPTO_SHA1:
465 case CRYPTO_MD5_HMAC:
466 case CRYPTO_SHA1_HMAC:
467 if (mac)
468 return (EINVAL);
469 mac = 1;
470 ses->hs_mlen = c->cri_mlen;
471 if (ses->hs_mlen == 0) {
472 switch (c->cri_alg) {
473 case CRYPTO_MD5:
474 case CRYPTO_MD5_HMAC:
475 ses->hs_mlen = 16;
476 break;
477 case CRYPTO_SHA1:
478 case CRYPTO_SHA1_HMAC:
479 ses->hs_mlen = 20;
480 break;
481 }
482 }
483 break;
484 case CRYPTO_DES_CBC:
485 case CRYPTO_3DES_CBC:
486 case CRYPTO_AES_CBC:
487 /* XXX this may read fewer, does it matter? */
488 read_random(ses->ses_iv, c->cri_alg ==
489 CRYPTO_AES_CBC ? XLP_SEC_AES_IV_LENGTH :
490 XLP_SEC_DES_IV_LENGTH);
491 /* FALLTHROUGH */
492 case CRYPTO_ARC4:
493 if (cry)
494 return (EINVAL);
495 cry = 1;
496 break;
497 default:
498 return (EINVAL);
499 }
500 }
501 if (mac == 0 && cry == 0)
502 return (EINVAL);
503
504 cmd->hash_dst_len = ses->hs_mlen;
505 return (0);
506 }
507
508 /*
509 * XXX freesession routine should run a zero'd mac/encrypt key into context
510 * ram. to blow away any keys already stored there.
511 */
512
513 static int
xlp_copyiv(struct xlp_sec_softc * sc,struct xlp_sec_command * cmd,struct cryptodesc * enccrd)514 xlp_copyiv(struct xlp_sec_softc *sc, struct xlp_sec_command *cmd,
515 struct cryptodesc *enccrd)
516 {
517 unsigned int ivlen = 0;
518 struct cryptop *crp = NULL;
519
520 crp = cmd->crp;
521
522 if (enccrd->crd_alg != CRYPTO_ARC4) {
523 ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ?
524 XLP_SEC_AES_IV_LENGTH : XLP_SEC_DES_IV_LENGTH);
525 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
526 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
527 bcopy(enccrd->crd_iv, cmd->iv, ivlen);
528 } else {
529 bcopy(cmd->ses->ses_iv, cmd->iv, ivlen);
530 }
531 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
532 crypto_copyback(crp->crp_flags,
533 crp->crp_buf, enccrd->crd_inject,
534 ivlen, cmd->iv);
535 }
536 } else {
537 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
538 bcopy(enccrd->crd_iv, cmd->iv, ivlen);
539 } else {
540 crypto_copydata(crp->crp_flags, crp->crp_buf,
541 enccrd->crd_inject, ivlen, cmd->iv);
542 }
543 }
544 }
545 return (0);
546 }
547
548 static int
xlp_get_nsegs(struct cryptop * crp,unsigned int * nsegs)549 xlp_get_nsegs(struct cryptop *crp, unsigned int *nsegs)
550 {
551
552 if (crp->crp_flags & CRYPTO_F_IMBUF) {
553 struct mbuf *m = NULL;
554
555 m = (struct mbuf *)crp->crp_buf;
556 while (m != NULL) {
557 *nsegs += NLM_CRYPTO_NUM_SEGS_REQD(m->m_len);
558 m = m->m_next;
559 }
560 } else if (crp->crp_flags & CRYPTO_F_IOV) {
561 struct uio *uio = NULL;
562 struct iovec *iov = NULL;
563 int iol = 0;
564
565 uio = (struct uio *)crp->crp_buf;
566 iov = (struct iovec *)uio->uio_iov;
567 iol = uio->uio_iovcnt;
568 while (iol > 0) {
569 *nsegs += NLM_CRYPTO_NUM_SEGS_REQD(iov->iov_len);
570 iol--;
571 iov++;
572 }
573 } else {
574 *nsegs = NLM_CRYPTO_NUM_SEGS_REQD(crp->crp_ilen);
575 }
576 return (0);
577 }
578
579 static int
xlp_alloc_cmd_params(struct xlp_sec_command * cmd,unsigned int nsegs)580 xlp_alloc_cmd_params(struct xlp_sec_command *cmd, unsigned int nsegs)
581 {
582 int err = 0;
583
584 if(cmd == NULL) {
585 err = EINVAL;
586 goto error;
587 }
588 if ((cmd->ctrlp = malloc(sizeof(struct nlm_crypto_pkt_ctrl), M_DEVBUF,
589 M_NOWAIT | M_ZERO)) == NULL) {
590 err = ENOMEM;
591 goto error;
592 }
593 if (((uintptr_t)cmd->ctrlp & (XLP_L2L3_CACHELINE_SIZE - 1))) {
594 err = EINVAL;
595 goto error;
596 }
597 /* (nsegs - 1) because one seg is part of the structure already */
598 if ((cmd->paramp = malloc(sizeof(struct nlm_crypto_pkt_param) +
599 (16 * (nsegs - 1)), M_DEVBUF, M_NOWAIT | M_ZERO)) == NULL) {
600 err = ENOMEM;
601 goto error;
602 }
603 if (((uintptr_t)cmd->paramp & (XLP_L2L3_CACHELINE_SIZE - 1))) {
604 err = EINVAL;
605 goto error;
606 }
607 if ((cmd->iv = malloc(EALG_MAX_BLOCK_LEN, M_DEVBUF,
608 M_NOWAIT | M_ZERO)) == NULL) {
609 err = ENOMEM;
610 goto error;
611 }
612 if ((cmd->hashdest = malloc(HASH_MAX_LEN, M_DEVBUF,
613 M_NOWAIT | M_ZERO)) == NULL) {
614 err = ENOMEM;
615 goto error;
616 }
617 error:
618 return (err);
619 }
620
621 static void
xlp_free_cmd_params(struct xlp_sec_command * cmd)622 xlp_free_cmd_params(struct xlp_sec_command *cmd)
623 {
624 if (cmd->ctrlp != NULL)
625 free(cmd->ctrlp, M_DEVBUF);
626 if (cmd->paramp != NULL)
627 free(cmd->paramp, M_DEVBUF);
628 if (cmd->iv != NULL)
629 free(cmd->iv, M_DEVBUF);
630 if (cmd->hashdest != NULL)
631 free(cmd->hashdest, M_DEVBUF);
632 if (cmd != NULL)
633 free(cmd, M_DEVBUF);
634 return;
635 }
636
637 static int
xlp_sec_process(device_t dev,struct cryptop * crp,int hint)638 xlp_sec_process(device_t dev, struct cryptop *crp, int hint)
639 {
640 struct xlp_sec_softc *sc = device_get_softc(dev);
641 struct xlp_sec_command *cmd = NULL;
642 int err = -1, ret = 0;
643 struct cryptodesc *crd1, *crd2;
644 struct xlp_sec_session *ses;
645 unsigned int nsegs = 0;
646
647 if (crp == NULL || crp->crp_callback == NULL) {
648 return (EINVAL);
649 }
650 if (sc == NULL) {
651 err = EINVAL;
652 goto errout;
653 }
654 ses = crypto_get_driver_session(crp->crp_session);
655
656 if ((cmd = malloc(sizeof(struct xlp_sec_command), M_DEVBUF,
657 M_NOWAIT | M_ZERO)) == NULL) {
658 err = ENOMEM;
659 goto errout;
660 }
661
662 cmd->crp = crp;
663 cmd->ses = ses;
664 cmd->hash_dst_len = ses->hs_mlen;
665
666 if ((crd1 = crp->crp_desc) == NULL) {
667 err = EINVAL;
668 goto errout;
669 }
670 crd2 = crd1->crd_next;
671
672 if ((ret = xlp_get_nsegs(crp, &nsegs)) != 0) {
673 err = EINVAL;
674 goto errout;
675 }
676 if (((crd1 != NULL) && (crd1->crd_flags & CRD_F_IV_EXPLICIT)) ||
677 ((crd2 != NULL) && (crd2->crd_flags & CRD_F_IV_EXPLICIT))) {
678 /* Since IV is given as separate segment to avoid copy */
679 nsegs += 1;
680 }
681 cmd->nsegs = nsegs;
682
683 if ((err = xlp_alloc_cmd_params(cmd, nsegs)) != 0)
684 goto errout;
685
686 if ((crd1 != NULL) && (crd2 == NULL)) {
687 if (crd1->crd_alg == CRYPTO_DES_CBC ||
688 crd1->crd_alg == CRYPTO_3DES_CBC ||
689 crd1->crd_alg == CRYPTO_AES_CBC ||
690 crd1->crd_alg == CRYPTO_ARC4) {
691 cmd->enccrd = crd1;
692 cmd->maccrd = NULL;
693 if ((ret = nlm_get_cipher_param(cmd)) != 0) {
694 err = EINVAL;
695 goto errout;
696 }
697 if (crd1->crd_flags & CRD_F_IV_EXPLICIT)
698 cmd->cipheroff = cmd->ivlen;
699 else
700 cmd->cipheroff = cmd->enccrd->crd_skip;
701 cmd->cipherlen = cmd->enccrd->crd_len;
702 if (crd1->crd_flags & CRD_F_IV_PRESENT)
703 cmd->ivoff = 0;
704 else
705 cmd->ivoff = cmd->enccrd->crd_inject;
706 if ((err = xlp_copyiv(sc, cmd, cmd->enccrd)) != 0)
707 goto errout;
708 if ((err = nlm_crypto_do_cipher(sc, cmd)) != 0)
709 goto errout;
710 } else if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
711 crd1->crd_alg == CRYPTO_SHA1_HMAC ||
712 crd1->crd_alg == CRYPTO_SHA1 ||
713 crd1->crd_alg == CRYPTO_MD5) {
714 cmd->enccrd = NULL;
715 cmd->maccrd = crd1;
716 if ((ret = nlm_get_digest_param(cmd)) != 0) {
717 err = EINVAL;
718 goto errout;
719 }
720 cmd->hashoff = cmd->maccrd->crd_skip;
721 cmd->hashlen = cmd->maccrd->crd_len;
722 cmd->hmacpad = 0;
723 cmd->hashsrc = 0;
724 if ((err = nlm_crypto_do_digest(sc, cmd)) != 0)
725 goto errout;
726 } else {
727 err = EINVAL;
728 goto errout;
729 }
730 } else if( (crd1 != NULL) && (crd2 != NULL) ) {
731 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
732 crd1->crd_alg == CRYPTO_SHA1_HMAC ||
733 crd1->crd_alg == CRYPTO_MD5 ||
734 crd1->crd_alg == CRYPTO_SHA1) &&
735 (crd2->crd_alg == CRYPTO_DES_CBC ||
736 crd2->crd_alg == CRYPTO_3DES_CBC ||
737 crd2->crd_alg == CRYPTO_AES_CBC ||
738 crd2->crd_alg == CRYPTO_ARC4)) {
739 cmd->maccrd = crd1;
740 cmd->enccrd = crd2;
741 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
742 crd1->crd_alg == CRYPTO_ARC4 ||
743 crd1->crd_alg == CRYPTO_3DES_CBC ||
744 crd1->crd_alg == CRYPTO_AES_CBC) &&
745 (crd2->crd_alg == CRYPTO_MD5_HMAC ||
746 crd2->crd_alg == CRYPTO_SHA1_HMAC ||
747 crd2->crd_alg == CRYPTO_MD5 ||
748 crd2->crd_alg == CRYPTO_SHA1)) {
749 cmd->enccrd = crd1;
750 cmd->maccrd = crd2;
751 } else {
752 err = EINVAL;
753 goto errout;
754 }
755 if ((ret = nlm_get_cipher_param(cmd)) != 0) {
756 err = EINVAL;
757 goto errout;
758 }
759 if ((ret = nlm_get_digest_param(cmd)) != 0) {
760 err = EINVAL;
761 goto errout;
762 }
763 cmd->ivoff = cmd->enccrd->crd_inject;
764 cmd->hashoff = cmd->maccrd->crd_skip;
765 cmd->hashlen = cmd->maccrd->crd_len;
766 cmd->hmacpad = 0;
767 if (cmd->enccrd->crd_flags & CRD_F_ENCRYPT)
768 cmd->hashsrc = 1;
769 else
770 cmd->hashsrc = 0;
771 cmd->cipheroff = cmd->enccrd->crd_skip;
772 cmd->cipherlen = cmd->enccrd->crd_len;
773 if ((err = xlp_copyiv(sc, cmd, cmd->enccrd)) != 0)
774 goto errout;
775 if ((err = nlm_crypto_do_cipher_digest(sc, cmd)) != 0)
776 goto errout;
777 } else {
778 err = EINVAL;
779 goto errout;
780 }
781 return (0);
782 errout:
783 xlp_free_cmd_params(cmd);
784 if (err == ERESTART) {
785 sc->sc_needwakeup |= CRYPTO_SYMQ;
786 creditleft = 0;
787 return (err);
788 }
789 crp->crp_etype = err;
790 crypto_done(crp);
791 return (err);
792 }
793