1 /*-
2 * Copyright (c) 2003-2012 Broadcom Corporation
3 * All Rights Reserved
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in
13 * the documentation and/or other materials provided with the
14 * distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
23 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
25 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
26 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31
32 #include <sys/cdefs.h>
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/proc.h>
36 #include <sys/errno.h>
37 #include <sys/malloc.h>
38 #include <sys/kernel.h>
39 #include <sys/module.h>
40 #include <sys/mbuf.h>
41 #include <sys/lock.h>
42 #include <sys/mutex.h>
43 #include <sys/sysctl.h>
44 #include <sys/bus.h>
45 #include <sys/random.h>
46 #include <sys/rman.h>
47 #include <sys/uio.h>
48 #include <sys/kobj.h>
49
50 #include <dev/pci/pcivar.h>
51
52 #include <opencrypto/cryptodev.h>
53
54 #include "cryptodev_if.h"
55
56 #include <vm/vm.h>
57 #include <vm/pmap.h>
58
59 #include <mips/nlm/hal/haldefs.h>
60 #include <mips/nlm/hal/iomap.h>
61 #include <mips/nlm/xlp.h>
62 #include <mips/nlm/hal/sys.h>
63 #include <mips/nlm/hal/fmn.h>
64 #include <mips/nlm/hal/nlmsaelib.h>
65 #include <mips/nlm/dev/sec/nlmseclib.h>
66 #include <mips/nlm/hal/cop2.h>
67 #include <mips/nlm/hal/mips-extns.h>
68 #include <mips/nlm/msgring.h>
69
70 unsigned int creditleft;
71
72 void xlp_sec_print_data(struct cryptop *crp);
73
74 static int xlp_sec_init(struct xlp_sec_softc *sc);
75 static int xlp_sec_newsession(device_t , uint32_t *, struct cryptoini *);
76 static int xlp_sec_freesession(device_t , uint64_t);
77 static int xlp_sec_process(device_t , struct cryptop *, int);
78 static int xlp_copyiv(struct xlp_sec_softc *, struct xlp_sec_command *,
79 struct cryptodesc *enccrd);
80 static int xlp_get_nsegs(struct cryptop *, unsigned int *);
81 static int xlp_alloc_cmd_params(struct xlp_sec_command *, unsigned int);
82 static void xlp_free_cmd_params(struct xlp_sec_command *);
83
84 static int xlp_sec_probe(device_t);
85 static int xlp_sec_attach(device_t);
86 static int xlp_sec_detach(device_t);
87
88 static device_method_t xlp_sec_methods[] = {
89 /* device interface */
90 DEVMETHOD(device_probe, xlp_sec_probe),
91 DEVMETHOD(device_attach, xlp_sec_attach),
92 DEVMETHOD(device_detach, xlp_sec_detach),
93
94 /* bus interface */
95 DEVMETHOD(bus_print_child, bus_generic_print_child),
96 DEVMETHOD(bus_driver_added, bus_generic_driver_added),
97
98 /* crypto device methods */
99 DEVMETHOD(cryptodev_newsession, xlp_sec_newsession),
100 DEVMETHOD(cryptodev_freesession,xlp_sec_freesession),
101 DEVMETHOD(cryptodev_process, xlp_sec_process),
102
103 DEVMETHOD_END
104 };
105
106 static driver_t xlp_sec_driver = {
107 "nlmsec",
108 xlp_sec_methods,
109 sizeof(struct xlp_sec_softc)
110 };
111 static devclass_t xlp_sec_devclass;
112
113 DRIVER_MODULE(nlmsec, pci, xlp_sec_driver, xlp_sec_devclass, 0, 0);
114 MODULE_DEPEND(nlmsec, crypto, 1, 1, 1);
115
116 void
117 nlm_xlpsec_msgring_handler(int vc, int size, int code, int src_id,
118 struct nlm_fmn_msg *msg, void *data);
119
120 #ifdef NLM_SEC_DEBUG
121
122 #define extract_bits(x, bitshift, bitcnt) \
123 (((unsigned long long)x >> bitshift) & ((1ULL << bitcnt) - 1))
124
125 void
print_crypto_params(struct xlp_sec_command * cmd,struct nlm_fmn_msg m)126 print_crypto_params(struct xlp_sec_command *cmd, struct nlm_fmn_msg m)
127 {
128 unsigned long long msg0,msg1,msg2,msg3,msg4,msg5,msg6,msg7,msg8;
129
130 msg0 = cmd->ctrlp->desc0;
131 msg1 = cmd->paramp->desc0;
132 msg2 = cmd->paramp->desc1;
133 msg3 = cmd->paramp->desc2;
134 msg4 = cmd->paramp->desc3;
135 msg5 = cmd->paramp->segment[0][0];
136 msg6 = cmd->paramp->segment[0][1];
137 msg7 = m.msg[0];
138 msg8 = m.msg[1];
139
140 printf("msg0 %llx msg1 %llx msg2 %llx msg3 %llx msg4 %llx msg5 %llx"
141 "msg6 %llx msg7 %llx msg8 %llx\n", msg0, msg1, msg2, msg3, msg4,
142 msg5, msg6, msg7, msg8);
143
144 printf("c0: hmac %d htype %d hmode %d ctype %d cmode %d arc4 %x\n",
145 (unsigned int)extract_bits(msg0, 61, 1),
146 (unsigned int)extract_bits(msg0, 52, 8),
147 (unsigned int)extract_bits(msg0, 43, 8),
148 (unsigned int)extract_bits(msg0, 34, 8),
149 (unsigned int)extract_bits(msg0, 25, 8),
150 (unsigned int)extract_bits(msg0, 0, 23));
151
152 printf("p0: tls %d hsrc %d hl3 %d enc %d ivl %d hd %llx\n",
153 (unsigned int)extract_bits(msg1, 63, 1),
154 (unsigned int)extract_bits(msg1,62,1),
155 (unsigned int)extract_bits(msg1,60,1),
156 (unsigned int)extract_bits(msg1,59,1),
157 (unsigned int)extract_bits(msg1,41,16), extract_bits(msg1,0,40));
158
159 printf("p1: clen %u hl %u\n", (unsigned int)extract_bits(msg2, 32, 32),
160 (unsigned int)extract_bits(msg2,0,32));
161
162 printf("p2: ivoff %d cbit %d coff %d hbit %d hclb %d hoff %d\n",
163 (unsigned int)extract_bits(msg3, 45, 17),
164 (unsigned int)extract_bits(msg3, 42,3),
165 (unsigned int)extract_bits(msg3, 22,16),
166 (unsigned int)extract_bits(msg3, 19,3),
167 (unsigned int)extract_bits(msg3, 18,1),
168 (unsigned int)extract_bits(msg3, 0, 16));
169
170 printf("p3: desfbid %d tlen %d arc4 %x hmacpad %d\n",
171 (unsigned int)extract_bits(msg4, 48,16),
172 (unsigned int)extract_bits(msg4,11,16),
173 (unsigned int)extract_bits(msg4,6,3),
174 (unsigned int)extract_bits(msg4,5,1));
175
176 printf("p4: sflen %d sddr %llx \n",
177 (unsigned int)extract_bits(msg5, 48, 16),extract_bits(msg5, 0, 40));
178
179 printf("p5: dflen %d cl3 %d cclob %d cdest %llx \n",
180 (unsigned int)extract_bits(msg6, 48, 16),
181 (unsigned int)extract_bits(msg6, 46, 1),
182 (unsigned int)extract_bits(msg6, 41, 1), extract_bits(msg6, 0, 40));
183
184 printf("fmn0: fbid %d dfrlen %d dfrv %d cklen %d cdescaddr %llx\n",
185 (unsigned int)extract_bits(msg7, 48, 16),
186 (unsigned int)extract_bits(msg7,46,2),
187 (unsigned int)extract_bits(msg7,45,1),
188 (unsigned int)extract_bits(msg7,40,5),
189 (extract_bits(msg7,0,34)<< 6));
190
191 printf("fmn1: arc4 %d hklen %d pdesclen %d pktdescad %llx\n",
192 (unsigned int)extract_bits(msg8, 63, 1),
193 (unsigned int)extract_bits(msg8,56,5),
194 (unsigned int)extract_bits(msg8,43,12),
195 (extract_bits(msg8,0,34) << 6));
196
197 return;
198 }
199
200 void
xlp_sec_print_data(struct cryptop * crp)201 xlp_sec_print_data(struct cryptop *crp)
202 {
203 int i, key_len;
204 struct cryptodesc *crp_desc;
205
206 printf("session id = 0x%llx, crp_ilen = %d, crp_olen=%d \n",
207 crp->crp_sid, crp->crp_ilen, crp->crp_olen);
208
209 printf("crp_flags = 0x%x\n", crp->crp_flags);
210
211 printf("crp buf:\n");
212 for (i = 0; i < crp->crp_ilen; i++) {
213 printf("%c ", crp->crp_buf[i]);
214 if (i % 10 == 0)
215 printf("\n");
216 }
217
218 printf("\n");
219 printf("****************** desc ****************\n");
220 crp_desc = crp->crp_desc;
221 printf("crd_skip=%d, crd_len=%d, crd_flags=0x%x, crd_alg=%d\n",
222 crp_desc->crd_skip, crp_desc->crd_len, crp_desc->crd_flags,
223 crp_desc->crd_alg);
224
225 key_len = crp_desc->crd_klen / 8;
226 printf("key(%d) :\n", key_len);
227 for (i = 0; i < key_len; i++)
228 printf("%d", crp_desc->crd_key[i]);
229 printf("\n");
230
231 printf(" IV : \n");
232 for (i = 0; i < EALG_MAX_BLOCK_LEN; i++)
233 printf("%d", crp_desc->crd_iv[i]);
234 printf("\n");
235
236 printf("crd_next=%p\n", crp_desc->crd_next);
237 return;
238 }
239
240 void
print_cmd(struct xlp_sec_command * cmd)241 print_cmd(struct xlp_sec_command *cmd)
242 {
243 printf("session_num :%d\n",cmd->session_num);
244 printf("crp :0x%x\n",(uint32_t)cmd->crp);
245 printf("enccrd :0x%x\n",(uint32_t)cmd->enccrd);
246 printf("maccrd :0x%x\n",(uint32_t)cmd->maccrd);
247 printf("ses :%d\n",(uint32_t)cmd->ses);
248 printf("ctrlp :0x%x\n",(uint32_t)cmd->ctrlp);
249 printf("paramp :0x%x\n",(uint32_t)cmd->paramp);
250 printf("hashdest :0x%x\n",(uint32_t)cmd->hashdest);
251 printf("hashsrc :%d\n",cmd->hashsrc);
252 printf("hmacpad :%d\n",cmd->hmacpad);
253 printf("hashoff :%d\n",cmd->hashoff);
254 printf("hashlen :%d\n",cmd->hashlen);
255 printf("cipheroff :%d\n",cmd->cipheroff);
256 printf("cipherlen :%d\n",cmd->cipherlen);
257 printf("ivoff :%d\n",cmd->ivoff);
258 printf("ivlen :%d\n",cmd->ivlen);
259 printf("hashalg :%d\n",cmd->hashalg);
260 printf("hashmode :%d\n",cmd->hashmode);
261 printf("cipheralg :%d\n",cmd->cipheralg);
262 printf("ciphermode :%d\n",cmd->ciphermode);
263 printf("nsegs :%d\n",cmd->nsegs);
264 printf("hash_dst_len :%d\n",cmd->hash_dst_len);
265 }
266 #endif /* NLM_SEC_DEBUG */
267
268 static int
xlp_sec_init(struct xlp_sec_softc * sc)269 xlp_sec_init(struct xlp_sec_softc *sc)
270 {
271
272 /* Register interrupt handler for the SEC CMS messages */
273 if (register_msgring_handler(sc->sec_vc_start,
274 sc->sec_vc_end, nlm_xlpsec_msgring_handler, sc) != 0) {
275 printf("Couldn't register sec msgring handler\n");
276 return (-1);
277 }
278
279 /* Do the CMS credit initialization */
280 /* Currently it is configured by default to 50 when kernel comes up */
281
282 return (0);
283 }
284
285 /* This function is called from an interrupt handler */
286 void
nlm_xlpsec_msgring_handler(int vc,int size,int code,int src_id,struct nlm_fmn_msg * msg,void * data)287 nlm_xlpsec_msgring_handler(int vc, int size, int code, int src_id,
288 struct nlm_fmn_msg *msg, void *data)
289 {
290 struct xlp_sec_command *cmd = NULL;
291 struct xlp_sec_softc *sc = NULL;
292 struct cryptodesc *crd = NULL;
293 unsigned int ivlen = 0;
294
295 KASSERT(code == FMN_SWCODE_CRYPTO,
296 ("%s: bad code = %d, expected code = %d\n", __FUNCTION__,
297 code, FMN_SWCODE_CRYPTO));
298
299 sc = (struct xlp_sec_softc *)data;
300 KASSERT(src_id >= sc->sec_vc_start && src_id <= sc->sec_vc_end,
301 ("%s: bad src_id = %d, expect %d - %d\n", __FUNCTION__,
302 src_id, sc->sec_vc_start, sc->sec_vc_end));
303
304 cmd = (struct xlp_sec_command *)(uintptr_t)msg->msg[0];
305 KASSERT(cmd != NULL && cmd->crp != NULL,
306 ("%s :cmd not received properly\n",__FUNCTION__));
307
308 KASSERT(CRYPTO_ERROR(msg->msg[1]) == 0,
309 ("%s: Message rcv msg0 %llx msg1 %llx err %x \n", __FUNCTION__,
310 (unsigned long long)msg->msg[0], (unsigned long long)msg->msg[1],
311 (int)CRYPTO_ERROR(msg->msg[1])));
312
313 crd = cmd->enccrd;
314 /* Copy the last 8 or 16 bytes to the session iv, so that in few
315 * cases this will be used as IV for the next request
316 */
317 if (crd != NULL) {
318 if ((crd->crd_alg == CRYPTO_DES_CBC ||
319 crd->crd_alg == CRYPTO_3DES_CBC ||
320 crd->crd_alg == CRYPTO_AES_CBC) &&
321 (crd->crd_flags & CRD_F_ENCRYPT)) {
322 ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ?
323 XLP_SEC_AES_IV_LENGTH : XLP_SEC_DES_IV_LENGTH);
324 crypto_copydata(cmd->crp->crp_flags, cmd->crp->crp_buf,
325 crd->crd_skip + crd->crd_len - ivlen, ivlen,
326 sc->sc_sessions[cmd->session_num].ses_iv);
327 }
328 }
329
330 /* If there are not enough credits to send, then send request
331 * will fail with ERESTART and the driver will be blocked until it is
332 * unblocked here after knowing that there are sufficient credits to
333 * send the request again.
334 */
335 if (sc->sc_needwakeup) {
336 atomic_add_int(&creditleft, sc->sec_msgsz);
337 if (creditleft >= (NLM_CRYPTO_LEFT_REQS)) {
338 crypto_unblock(sc->sc_cid, sc->sc_needwakeup);
339 sc->sc_needwakeup &= (~(CRYPTO_SYMQ | CRYPTO_ASYMQ));
340 }
341 }
342 if(cmd->maccrd) {
343 crypto_copyback(cmd->crp->crp_flags,
344 cmd->crp->crp_buf, cmd->maccrd->crd_inject,
345 cmd->hash_dst_len, cmd->hashdest);
346 }
347
348 /* This indicates completion of the crypto operation */
349 crypto_done(cmd->crp);
350
351 xlp_free_cmd_params(cmd);
352
353 return;
354 }
355
356 static int
xlp_sec_probe(device_t dev)357 xlp_sec_probe(device_t dev)
358 {
359 struct xlp_sec_softc *sc;
360
361 if (pci_get_vendor(dev) == PCI_VENDOR_NETLOGIC &&
362 pci_get_device(dev) == PCI_DEVICE_ID_NLM_SAE) {
363 sc = device_get_softc(dev);
364 return (BUS_PROBE_DEFAULT);
365 }
366 return (ENXIO);
367 }
368
369 /*
370 * Attach an interface that successfully probed.
371 */
372 static int
xlp_sec_attach(device_t dev)373 xlp_sec_attach(device_t dev)
374 {
375 struct xlp_sec_softc *sc = device_get_softc(dev);
376 uint64_t base;
377 int qstart, qnum;
378 int freq, node;
379
380 sc->sc_dev = dev;
381
382 node = nlm_get_device_node(pci_get_slot(dev));
383 freq = nlm_set_device_frequency(node, DFS_DEVICE_SAE, 250);
384 if (bootverbose)
385 device_printf(dev, "SAE Freq: %dMHz\n", freq);
386 if(pci_get_device(dev) == PCI_DEVICE_ID_NLM_SAE) {
387 device_set_desc(dev, "XLP Security Accelerator");
388 sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
389 if (sc->sc_cid < 0) {
390 printf("xlp_sec - error : could not get the driver"
391 " id\n");
392 goto error_exit;
393 }
394 if (crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0) != 0)
395 printf("register failed for CRYPTO_DES_CBC\n");
396
397 if (crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0) != 0)
398 printf("register failed for CRYPTO_3DES_CBC\n");
399
400 if (crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0) != 0)
401 printf("register failed for CRYPTO_AES_CBC\n");
402
403 if (crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0) != 0)
404 printf("register failed for CRYPTO_ARC4\n");
405
406 if (crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0) != 0)
407 printf("register failed for CRYPTO_MD5\n");
408
409 if (crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0) != 0)
410 printf("register failed for CRYPTO_SHA1\n");
411
412 if (crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0) != 0)
413 printf("register failed for CRYPTO_MD5_HMAC\n");
414
415 if (crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0) != 0)
416 printf("register failed for CRYPTO_SHA1_HMAC\n");
417
418 base = nlm_get_sec_pcibase(node);
419 qstart = nlm_qidstart(base);
420 qnum = nlm_qnum(base);
421 sc->sec_vc_start = qstart;
422 sc->sec_vc_end = qstart + qnum - 1;
423 }
424
425 if (xlp_sec_init(sc) != 0)
426 goto error_exit;
427 if (bootverbose)
428 device_printf(dev, "SEC Initialization complete!\n");
429 return (0);
430
431 error_exit:
432 return (ENXIO);
433
434 }
435
436 /*
437 * Detach an interface that successfully probed.
438 */
439 static int
xlp_sec_detach(device_t dev)440 xlp_sec_detach(device_t dev)
441 {
442 return (0);
443 }
444
445 /*
446 * Allocate a new 'session' and return an encoded session id. 'sidp'
447 * contains our registration id, and should contain an encoded session
448 * id on successful allocation.
449 */
450 static int
xlp_sec_newsession(device_t dev,u_int32_t * sidp,struct cryptoini * cri)451 xlp_sec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
452 {
453 struct cryptoini *c;
454 struct xlp_sec_softc *sc = device_get_softc(dev);
455 int mac = 0, cry = 0, sesn;
456 struct xlp_sec_session *ses = NULL;
457 struct xlp_sec_command *cmd = NULL;
458
459 if (sidp == NULL || cri == NULL || sc == NULL)
460 return (EINVAL);
461
462 if (sc->sc_sessions == NULL) {
463 ses = sc->sc_sessions = malloc(sizeof(struct xlp_sec_session),
464 M_DEVBUF, M_NOWAIT);
465 if (ses == NULL)
466 return (ENOMEM);
467 sesn = 0;
468 sc->sc_nsessions = 1;
469 } else {
470 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
471 if (!sc->sc_sessions[sesn].hs_used) {
472 ses = &sc->sc_sessions[sesn];
473 break;
474 }
475 }
476
477 if (ses == NULL) {
478 sesn = sc->sc_nsessions;
479 ses = malloc((sesn + 1)*sizeof(struct xlp_sec_session),
480 M_DEVBUF, M_NOWAIT);
481 if (ses == NULL)
482 return (ENOMEM);
483 bcopy(sc->sc_sessions, ses, sesn * sizeof(*ses));
484 bzero(sc->sc_sessions, sesn * sizeof(*ses));
485 free(sc->sc_sessions, M_DEVBUF);
486 sc->sc_sessions = ses;
487 ses = &sc->sc_sessions[sesn];
488 sc->sc_nsessions++;
489 }
490 }
491 bzero(ses, sizeof(*ses));
492 ses->sessionid = sesn;
493 cmd = &ses->cmd;
494 ses->hs_used = 1;
495
496 for (c = cri; c != NULL; c = c->cri_next) {
497 switch (c->cri_alg) {
498 case CRYPTO_MD5:
499 case CRYPTO_SHA1:
500 case CRYPTO_MD5_HMAC:
501 case CRYPTO_SHA1_HMAC:
502 if (mac)
503 return (EINVAL);
504 mac = 1;
505 ses->hs_mlen = c->cri_mlen;
506 if (ses->hs_mlen == 0) {
507 switch (c->cri_alg) {
508 case CRYPTO_MD5:
509 case CRYPTO_MD5_HMAC:
510 ses->hs_mlen = 16;
511 break;
512 case CRYPTO_SHA1:
513 case CRYPTO_SHA1_HMAC:
514 ses->hs_mlen = 20;
515 break;
516 }
517 }
518 break;
519 case CRYPTO_DES_CBC:
520 case CRYPTO_3DES_CBC:
521 case CRYPTO_AES_CBC:
522 /* XXX this may read fewer, does it matter? */
523 read_random(ses->ses_iv, c->cri_alg ==
524 CRYPTO_AES_CBC ? XLP_SEC_AES_IV_LENGTH :
525 XLP_SEC_DES_IV_LENGTH);
526 /* FALLTHROUGH */
527 case CRYPTO_ARC4:
528 if (cry)
529 return (EINVAL);
530 cry = 1;
531 break;
532 default:
533 return (EINVAL);
534 }
535 }
536 if (mac == 0 && cry == 0)
537 return (EINVAL);
538
539 cmd->hash_dst_len = ses->hs_mlen;
540 *sidp = XLP_SEC_SID(device_get_unit(sc->sc_dev), sesn);
541 return (0);
542 }
543
544 /*
545 * Deallocate a session.
546 * XXX this routine should run a zero'd mac/encrypt key into context ram.
547 * XXX to blow away any keys already stored there.
548 */
549 static int
xlp_sec_freesession(device_t dev,u_int64_t tid)550 xlp_sec_freesession(device_t dev, u_int64_t tid)
551 {
552 struct xlp_sec_softc *sc = device_get_softc(dev);
553 int session;
554 u_int32_t sid = CRYPTO_SESID2LID(tid);
555
556 if (sc == NULL)
557 return (EINVAL);
558
559 session = XLP_SEC_SESSION(sid);
560 if (session >= sc->sc_nsessions)
561 return (EINVAL);
562
563 sc->sc_sessions[session].hs_used = 0;
564 return (0);
565 }
566
567 static int
xlp_copyiv(struct xlp_sec_softc * sc,struct xlp_sec_command * cmd,struct cryptodesc * enccrd)568 xlp_copyiv(struct xlp_sec_softc *sc, struct xlp_sec_command *cmd,
569 struct cryptodesc *enccrd)
570 {
571 unsigned int ivlen = 0;
572 int session;
573 struct cryptop *crp = NULL;
574
575 crp = cmd->crp;
576 session = cmd->session_num;
577
578 if (enccrd->crd_alg != CRYPTO_ARC4) {
579 ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ?
580 XLP_SEC_AES_IV_LENGTH : XLP_SEC_DES_IV_LENGTH);
581 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
582 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
583 bcopy(enccrd->crd_iv, cmd->iv, ivlen);
584 } else {
585 bcopy(sc->sc_sessions[session].ses_iv, cmd->iv,
586 ivlen);
587 }
588 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
589 crypto_copyback(crp->crp_flags,
590 crp->crp_buf, enccrd->crd_inject,
591 ivlen, cmd->iv);
592 }
593 } else {
594 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
595 bcopy(enccrd->crd_iv, cmd->iv, ivlen);
596 } else {
597 crypto_copydata(crp->crp_flags, crp->crp_buf,
598 enccrd->crd_inject, ivlen, cmd->iv);
599 }
600 }
601 }
602 return (0);
603 }
604
605 static int
xlp_get_nsegs(struct cryptop * crp,unsigned int * nsegs)606 xlp_get_nsegs(struct cryptop *crp, unsigned int *nsegs)
607 {
608
609 if (crp->crp_flags & CRYPTO_F_IMBUF) {
610 struct mbuf *m = NULL;
611
612 m = (struct mbuf *)crp->crp_buf;
613 while (m != NULL) {
614 *nsegs += NLM_CRYPTO_NUM_SEGS_REQD(m->m_len);
615 m = m->m_next;
616 }
617 } else if (crp->crp_flags & CRYPTO_F_IOV) {
618 struct uio *uio = NULL;
619 struct iovec *iov = NULL;
620 int iol = 0;
621
622 uio = (struct uio *)crp->crp_buf;
623 iov = (struct iovec *)uio->uio_iov;
624 iol = uio->uio_iovcnt;
625 while (iol > 0) {
626 *nsegs += NLM_CRYPTO_NUM_SEGS_REQD(iov->iov_len);
627 iol--;
628 iov++;
629 }
630 } else {
631 *nsegs = NLM_CRYPTO_NUM_SEGS_REQD(crp->crp_ilen);
632 }
633 return (0);
634 }
635
636 static int
xlp_alloc_cmd_params(struct xlp_sec_command * cmd,unsigned int nsegs)637 xlp_alloc_cmd_params(struct xlp_sec_command *cmd, unsigned int nsegs)
638 {
639 int err = 0;
640
641 if(cmd == NULL) {
642 err = EINVAL;
643 goto error;
644 }
645 if ((cmd->ctrlp = malloc(sizeof(struct nlm_crypto_pkt_ctrl), M_DEVBUF,
646 M_NOWAIT | M_ZERO)) == NULL) {
647 err = ENOMEM;
648 goto error;
649 }
650 if (((uintptr_t)cmd->ctrlp & (XLP_L2L3_CACHELINE_SIZE - 1))) {
651 err = EINVAL;
652 goto error;
653 }
654 /* (nsegs - 1) because one seg is part of the structure already */
655 if ((cmd->paramp = malloc(sizeof(struct nlm_crypto_pkt_param) +
656 (16 * (nsegs - 1)), M_DEVBUF, M_NOWAIT | M_ZERO)) == NULL) {
657 err = ENOMEM;
658 goto error;
659 }
660 if (((uintptr_t)cmd->paramp & (XLP_L2L3_CACHELINE_SIZE - 1))) {
661 err = EINVAL;
662 goto error;
663 }
664 if ((cmd->iv = malloc(EALG_MAX_BLOCK_LEN, M_DEVBUF,
665 M_NOWAIT | M_ZERO)) == NULL) {
666 err = ENOMEM;
667 goto error;
668 }
669 if ((cmd->hashdest = malloc(HASH_MAX_LEN, M_DEVBUF,
670 M_NOWAIT | M_ZERO)) == NULL) {
671 err = ENOMEM;
672 goto error;
673 }
674 error:
675 return (err);
676 }
677
678 static void
xlp_free_cmd_params(struct xlp_sec_command * cmd)679 xlp_free_cmd_params(struct xlp_sec_command *cmd)
680 {
681 if (cmd->ctrlp != NULL)
682 free(cmd->ctrlp, M_DEVBUF);
683 if (cmd->paramp != NULL)
684 free(cmd->paramp, M_DEVBUF);
685 if (cmd->iv != NULL)
686 free(cmd->iv, M_DEVBUF);
687 if (cmd->hashdest != NULL)
688 free(cmd->hashdest, M_DEVBUF);
689 if (cmd != NULL)
690 free(cmd, M_DEVBUF);
691 return;
692 }
693
694 static int
xlp_sec_process(device_t dev,struct cryptop * crp,int hint)695 xlp_sec_process(device_t dev, struct cryptop *crp, int hint)
696 {
697 struct xlp_sec_softc *sc = device_get_softc(dev);
698 struct xlp_sec_command *cmd = NULL;
699 int session, err = -1, ret = 0;
700 struct cryptodesc *crd1, *crd2;
701 struct xlp_sec_session *ses;
702 unsigned int nsegs = 0;
703
704 if (crp == NULL || crp->crp_callback == NULL) {
705 return (EINVAL);
706 }
707 session = XLP_SEC_SESSION(crp->crp_sid);
708 if (sc == NULL || session >= sc->sc_nsessions) {
709 err = EINVAL;
710 goto errout;
711 }
712 ses = &sc->sc_sessions[session];
713
714 if ((cmd = malloc(sizeof(struct xlp_sec_command), M_DEVBUF,
715 M_NOWAIT | M_ZERO)) == NULL) {
716 err = ENOMEM;
717 goto errout;
718 }
719
720 cmd->crp = crp;
721 cmd->session_num = session;
722 cmd->hash_dst_len = ses->hs_mlen;
723
724 if ((crd1 = crp->crp_desc) == NULL) {
725 err = EINVAL;
726 goto errout;
727 }
728 crd2 = crd1->crd_next;
729
730 if ((ret = xlp_get_nsegs(crp, &nsegs)) != 0) {
731 err = EINVAL;
732 goto errout;
733 }
734 if (((crd1 != NULL) && (crd1->crd_flags & CRD_F_IV_EXPLICIT)) ||
735 ((crd2 != NULL) && (crd2->crd_flags & CRD_F_IV_EXPLICIT))) {
736 /* Since IV is given as separate segment to avoid copy */
737 nsegs += 1;
738 }
739 cmd->nsegs = nsegs;
740
741 if ((err = xlp_alloc_cmd_params(cmd, nsegs)) != 0)
742 goto errout;
743
744 if ((crd1 != NULL) && (crd2 == NULL)) {
745 if (crd1->crd_alg == CRYPTO_DES_CBC ||
746 crd1->crd_alg == CRYPTO_3DES_CBC ||
747 crd1->crd_alg == CRYPTO_AES_CBC ||
748 crd1->crd_alg == CRYPTO_ARC4) {
749 cmd->enccrd = crd1;
750 cmd->maccrd = NULL;
751 if ((ret = nlm_get_cipher_param(cmd)) != 0) {
752 err = EINVAL;
753 goto errout;
754 }
755 if (crd1->crd_flags & CRD_F_IV_EXPLICIT)
756 cmd->cipheroff = cmd->ivlen;
757 else
758 cmd->cipheroff = cmd->enccrd->crd_skip;
759 cmd->cipherlen = cmd->enccrd->crd_len;
760 if (crd1->crd_flags & CRD_F_IV_PRESENT)
761 cmd->ivoff = 0;
762 else
763 cmd->ivoff = cmd->enccrd->crd_inject;
764 if ((err = xlp_copyiv(sc, cmd, cmd->enccrd)) != 0)
765 goto errout;
766 if ((err = nlm_crypto_do_cipher(sc, cmd)) != 0)
767 goto errout;
768 } else if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
769 crd1->crd_alg == CRYPTO_SHA1_HMAC ||
770 crd1->crd_alg == CRYPTO_SHA1 ||
771 crd1->crd_alg == CRYPTO_MD5) {
772 cmd->enccrd = NULL;
773 cmd->maccrd = crd1;
774 if ((ret = nlm_get_digest_param(cmd)) != 0) {
775 err = EINVAL;
776 goto errout;
777 }
778 cmd->hashoff = cmd->maccrd->crd_skip;
779 cmd->hashlen = cmd->maccrd->crd_len;
780 cmd->hmacpad = 0;
781 cmd->hashsrc = 0;
782 if ((err = nlm_crypto_do_digest(sc, cmd)) != 0)
783 goto errout;
784 } else {
785 err = EINVAL;
786 goto errout;
787 }
788 } else if( (crd1 != NULL) && (crd2 != NULL) ) {
789 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
790 crd1->crd_alg == CRYPTO_SHA1_HMAC ||
791 crd1->crd_alg == CRYPTO_MD5 ||
792 crd1->crd_alg == CRYPTO_SHA1) &&
793 (crd2->crd_alg == CRYPTO_DES_CBC ||
794 crd2->crd_alg == CRYPTO_3DES_CBC ||
795 crd2->crd_alg == CRYPTO_AES_CBC ||
796 crd2->crd_alg == CRYPTO_ARC4)) {
797 cmd->maccrd = crd1;
798 cmd->enccrd = crd2;
799 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
800 crd1->crd_alg == CRYPTO_ARC4 ||
801 crd1->crd_alg == CRYPTO_3DES_CBC ||
802 crd1->crd_alg == CRYPTO_AES_CBC) &&
803 (crd2->crd_alg == CRYPTO_MD5_HMAC ||
804 crd2->crd_alg == CRYPTO_SHA1_HMAC ||
805 crd2->crd_alg == CRYPTO_MD5 ||
806 crd2->crd_alg == CRYPTO_SHA1)) {
807 cmd->enccrd = crd1;
808 cmd->maccrd = crd2;
809 } else {
810 err = EINVAL;
811 goto errout;
812 }
813 if ((ret = nlm_get_cipher_param(cmd)) != 0) {
814 err = EINVAL;
815 goto errout;
816 }
817 if ((ret = nlm_get_digest_param(cmd)) != 0) {
818 err = EINVAL;
819 goto errout;
820 }
821 cmd->ivoff = cmd->enccrd->crd_inject;
822 cmd->hashoff = cmd->maccrd->crd_skip;
823 cmd->hashlen = cmd->maccrd->crd_len;
824 cmd->hmacpad = 0;
825 if (cmd->enccrd->crd_flags & CRD_F_ENCRYPT)
826 cmd->hashsrc = 1;
827 else
828 cmd->hashsrc = 0;
829 cmd->cipheroff = cmd->enccrd->crd_skip;
830 cmd->cipherlen = cmd->enccrd->crd_len;
831 if ((err = xlp_copyiv(sc, cmd, cmd->enccrd)) != 0)
832 goto errout;
833 if ((err = nlm_crypto_do_cipher_digest(sc, cmd)) != 0)
834 goto errout;
835 } else {
836 err = EINVAL;
837 goto errout;
838 }
839 return (0);
840 errout:
841 xlp_free_cmd_params(cmd);
842 if (err == ERESTART) {
843 sc->sc_needwakeup |= CRYPTO_SYMQ;
844 creditleft = 0;
845 return (err);
846 }
847 crp->crp_etype = err;
848 crypto_done(crp);
849 return (err);
850 }
851