1 /*
2 * vim:sw=4 ts=8
3 */
4 /*
5 * Copyright (c) 2009 David McCullough <david.mccullough@securecomputing.com>
6 *
7 * Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights
8 * reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
12 * 1. Redistributions of source code must retain the above copyright notice,
13 * this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright notice,
15 * this list of conditions and the following disclaimer in the documentation
16 * and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 * must display the following acknowledgement:
19 * This product includes software developed by Cavium Networks
20 * 4. Cavium Networks' name may not be used to endorse or promote products
21 * derived from this software without specific prior written permission.
22 *
23 * This Software, including technical data, may be subject to U.S. export
24 * control laws, including the U.S. Export Administration Act and its
25 * associated regulations, and may be subject to export or import regulations
26 * in other countries. You warrant that You will comply strictly in all
27 * respects with all such regulations and acknowledge that you have the
28 * responsibility to obtain licenses to export, re-export or import the
29 * Software.
30 *
31 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" AND
32 * WITH ALL FAULTS AND CAVIUM MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES,
33 * EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE
34 * SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR
35 * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
36 * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
37 * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
38 * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
39 * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
40 * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
41 */
42 /****************************************************************************/
43
44 #include <sys/cdefs.h>
45 __FBSDID("$FreeBSD$");
46
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/kernel.h>
50 #include <sys/module.h>
51 #include <sys/malloc.h>
52 #include <sys/uio.h>
53
54 #include <opencrypto/cryptodev.h>
55
56 #include <contrib/octeon-sdk/cvmx.h>
57
58 #include <mips/cavium/cryptocteon/cryptocteonvar.h>
59
60 /****************************************************************************/
61
62 #define IOV_INIT(iov, ptr, idx, len) \
63 do { \
64 (idx) = 0; \
65 (ptr) = (iov)[(idx)].iov_base; \
66 (len) = (iov)[(idx)].iov_len; \
67 } while (0)
68
69 /*
70 * XXX
71 * It would be better if this were an IOV_READ/IOV_WRITE macro instead so
72 * that we could detect overflow before it happens rather than right after,
73 * which is especially bad since there is usually no IOV_CONSUME after the
74 * final read or write.
75 */
76 #define IOV_CONSUME(iov, ptr, idx, len) \
77 do { \
78 if ((len) > sizeof *(ptr)) { \
79 (len) -= sizeof *(ptr); \
80 (ptr)++; \
81 } else { \
82 if ((len) != sizeof *(ptr)) \
83 panic("%s: went past end of iovec.", __func__); \
84 (idx)++; \
85 (ptr) = (iov)[(idx)].iov_base; \
86 (len) = (iov)[(idx)].iov_len; \
87 } \
88 } while (0)
89
90 #define ESP_HEADER_LENGTH 8
91 #define DES_CBC_IV_LENGTH 8
92 #define AES_CBC_IV_LENGTH 16
93 #define ESP_HMAC_LEN 12
94
95 #define ESP_HEADER_LENGTH 8
96 #define DES_CBC_IV_LENGTH 8
97
98 /****************************************************************************/
99
100 #define CVM_LOAD_SHA_UNIT(dat, next) { \
101 if (next == 0) { \
102 next = 1; \
103 CVMX_MT_HSH_DAT (dat, 0); \
104 } else if (next == 1) { \
105 next = 2; \
106 CVMX_MT_HSH_DAT (dat, 1); \
107 } else if (next == 2) { \
108 next = 3; \
109 CVMX_MT_HSH_DAT (dat, 2); \
110 } else if (next == 3) { \
111 next = 4; \
112 CVMX_MT_HSH_DAT (dat, 3); \
113 } else if (next == 4) { \
114 next = 5; \
115 CVMX_MT_HSH_DAT (dat, 4); \
116 } else if (next == 5) { \
117 next = 6; \
118 CVMX_MT_HSH_DAT (dat, 5); \
119 } else if (next == 6) { \
120 next = 7; \
121 CVMX_MT_HSH_DAT (dat, 6); \
122 } else { \
123 CVMX_MT_HSH_STARTSHA (dat); \
124 next = 0; \
125 } \
126 }
127
128 #define CVM_LOAD2_SHA_UNIT(dat1, dat2, next) { \
129 if (next == 0) { \
130 CVMX_MT_HSH_DAT (dat1, 0); \
131 CVMX_MT_HSH_DAT (dat2, 1); \
132 next = 2; \
133 } else if (next == 1) { \
134 CVMX_MT_HSH_DAT (dat1, 1); \
135 CVMX_MT_HSH_DAT (dat2, 2); \
136 next = 3; \
137 } else if (next == 2) { \
138 CVMX_MT_HSH_DAT (dat1, 2); \
139 CVMX_MT_HSH_DAT (dat2, 3); \
140 next = 4; \
141 } else if (next == 3) { \
142 CVMX_MT_HSH_DAT (dat1, 3); \
143 CVMX_MT_HSH_DAT (dat2, 4); \
144 next = 5; \
145 } else if (next == 4) { \
146 CVMX_MT_HSH_DAT (dat1, 4); \
147 CVMX_MT_HSH_DAT (dat2, 5); \
148 next = 6; \
149 } else if (next == 5) { \
150 CVMX_MT_HSH_DAT (dat1, 5); \
151 CVMX_MT_HSH_DAT (dat2, 6); \
152 next = 7; \
153 } else if (next == 6) { \
154 CVMX_MT_HSH_DAT (dat1, 6); \
155 CVMX_MT_HSH_STARTSHA (dat2); \
156 next = 0; \
157 } else { \
158 CVMX_MT_HSH_STARTSHA (dat1); \
159 CVMX_MT_HSH_DAT (dat2, 0); \
160 next = 1; \
161 } \
162 }
163
164 /****************************************************************************/
165
166 #define CVM_LOAD_MD5_UNIT(dat, next) { \
167 if (next == 0) { \
168 next = 1; \
169 CVMX_MT_HSH_DAT (dat, 0); \
170 } else if (next == 1) { \
171 next = 2; \
172 CVMX_MT_HSH_DAT (dat, 1); \
173 } else if (next == 2) { \
174 next = 3; \
175 CVMX_MT_HSH_DAT (dat, 2); \
176 } else if (next == 3) { \
177 next = 4; \
178 CVMX_MT_HSH_DAT (dat, 3); \
179 } else if (next == 4) { \
180 next = 5; \
181 CVMX_MT_HSH_DAT (dat, 4); \
182 } else if (next == 5) { \
183 next = 6; \
184 CVMX_MT_HSH_DAT (dat, 5); \
185 } else if (next == 6) { \
186 next = 7; \
187 CVMX_MT_HSH_DAT (dat, 6); \
188 } else { \
189 CVMX_MT_HSH_STARTMD5 (dat); \
190 next = 0; \
191 } \
192 }
193
194 #define CVM_LOAD2_MD5_UNIT(dat1, dat2, next) { \
195 if (next == 0) { \
196 CVMX_MT_HSH_DAT (dat1, 0); \
197 CVMX_MT_HSH_DAT (dat2, 1); \
198 next = 2; \
199 } else if (next == 1) { \
200 CVMX_MT_HSH_DAT (dat1, 1); \
201 CVMX_MT_HSH_DAT (dat2, 2); \
202 next = 3; \
203 } else if (next == 2) { \
204 CVMX_MT_HSH_DAT (dat1, 2); \
205 CVMX_MT_HSH_DAT (dat2, 3); \
206 next = 4; \
207 } else if (next == 3) { \
208 CVMX_MT_HSH_DAT (dat1, 3); \
209 CVMX_MT_HSH_DAT (dat2, 4); \
210 next = 5; \
211 } else if (next == 4) { \
212 CVMX_MT_HSH_DAT (dat1, 4); \
213 CVMX_MT_HSH_DAT (dat2, 5); \
214 next = 6; \
215 } else if (next == 5) { \
216 CVMX_MT_HSH_DAT (dat1, 5); \
217 CVMX_MT_HSH_DAT (dat2, 6); \
218 next = 7; \
219 } else if (next == 6) { \
220 CVMX_MT_HSH_DAT (dat1, 6); \
221 CVMX_MT_HSH_STARTMD5 (dat2); \
222 next = 0; \
223 } else { \
224 CVMX_MT_HSH_STARTMD5 (dat1); \
225 CVMX_MT_HSH_DAT (dat2, 0); \
226 next = 1; \
227 } \
228 }
229
230 /****************************************************************************/
231
232 void
octo_calc_hash(uint8_t auth,unsigned char * key,uint64_t * inner,uint64_t * outer)233 octo_calc_hash(uint8_t auth, unsigned char *key, uint64_t *inner, uint64_t *outer)
234 {
235 uint8_t hash_key[64];
236 uint64_t *key1;
237 register uint64_t xor1 = 0x3636363636363636ULL;
238 register uint64_t xor2 = 0x5c5c5c5c5c5c5c5cULL;
239
240 dprintf("%s()\n", __func__);
241
242 memset(hash_key, 0, sizeof(hash_key));
243 memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16));
244 key1 = (uint64_t *) hash_key;
245 if (auth) {
246 CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0);
247 CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1);
248 CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2);
249 } else {
250 CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0);
251 CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1);
252 }
253
254 CVMX_MT_HSH_DAT((*key1 ^ xor1), 0);
255 key1++;
256 CVMX_MT_HSH_DAT((*key1 ^ xor1), 1);
257 key1++;
258 CVMX_MT_HSH_DAT((*key1 ^ xor1), 2);
259 key1++;
260 CVMX_MT_HSH_DAT((*key1 ^ xor1), 3);
261 key1++;
262 CVMX_MT_HSH_DAT((*key1 ^ xor1), 4);
263 key1++;
264 CVMX_MT_HSH_DAT((*key1 ^ xor1), 5);
265 key1++;
266 CVMX_MT_HSH_DAT((*key1 ^ xor1), 6);
267 key1++;
268 if (auth)
269 CVMX_MT_HSH_STARTSHA((*key1 ^ xor1));
270 else
271 CVMX_MT_HSH_STARTMD5((*key1 ^ xor1));
272
273 CVMX_MF_HSH_IV(inner[0], 0);
274 CVMX_MF_HSH_IV(inner[1], 1);
275 if (auth) {
276 inner[2] = 0;
277 CVMX_MF_HSH_IV(((uint64_t *) inner)[2], 2);
278 }
279
280 memset(hash_key, 0, sizeof(hash_key));
281 memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16));
282 key1 = (uint64_t *) hash_key;
283 if (auth) {
284 CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0);
285 CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1);
286 CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2);
287 } else {
288 CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0);
289 CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1);
290 }
291
292 CVMX_MT_HSH_DAT((*key1 ^ xor2), 0);
293 key1++;
294 CVMX_MT_HSH_DAT((*key1 ^ xor2), 1);
295 key1++;
296 CVMX_MT_HSH_DAT((*key1 ^ xor2), 2);
297 key1++;
298 CVMX_MT_HSH_DAT((*key1 ^ xor2), 3);
299 key1++;
300 CVMX_MT_HSH_DAT((*key1 ^ xor2), 4);
301 key1++;
302 CVMX_MT_HSH_DAT((*key1 ^ xor2), 5);
303 key1++;
304 CVMX_MT_HSH_DAT((*key1 ^ xor2), 6);
305 key1++;
306 if (auth)
307 CVMX_MT_HSH_STARTSHA((*key1 ^ xor2));
308 else
309 CVMX_MT_HSH_STARTMD5((*key1 ^ xor2));
310
311 CVMX_MF_HSH_IV(outer[0], 0);
312 CVMX_MF_HSH_IV(outer[1], 1);
313 if (auth) {
314 outer[2] = 0;
315 CVMX_MF_HSH_IV(outer[2], 2);
316 }
317 return;
318 }
319
320 /****************************************************************************/
321 /* DES functions */
322
323 int
octo_des_cbc_encrypt(struct octo_sess * od,struct iovec * iov,size_t iovcnt,size_t iovlen,int auth_off,int auth_len,int crypt_off,int crypt_len,int icv_off,uint8_t * ivp)324 octo_des_cbc_encrypt(
325 struct octo_sess *od,
326 struct iovec *iov, size_t iovcnt, size_t iovlen,
327 int auth_off, int auth_len,
328 int crypt_off, int crypt_len,
329 int icv_off, uint8_t *ivp)
330 {
331 uint64_t *data;
332 int data_i, data_l;
333
334 dprintf("%s()\n", __func__);
335
336 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
337 (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
338 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
339 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
340 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
341 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
342 return -EINVAL;
343 }
344
345 IOV_INIT(iov, data, data_i, data_l);
346
347 CVMX_PREFETCH0(ivp);
348 CVMX_PREFETCH0(od->octo_enckey);
349
350
351 /* load 3DES Key */
352 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
353 if (od->octo_encklen == 24) {
354 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
355 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
356 } else if (od->octo_encklen == 8) {
357 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
358 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
359 } else {
360 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
361 return -EINVAL;
362 }
363
364 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
365
366 while (crypt_off > 0) {
367 IOV_CONSUME(iov, data, data_i, data_l);
368 crypt_off -= 8;
369 }
370
371 while (crypt_len > 0) {
372 CVMX_MT_3DES_ENC_CBC(*data);
373 CVMX_MF_3DES_RESULT(*data);
374 IOV_CONSUME(iov, data, data_i, data_l);
375 crypt_len -= 8;
376 }
377
378 return 0;
379 }
380
381
382 int
octo_des_cbc_decrypt(struct octo_sess * od,struct iovec * iov,size_t iovcnt,size_t iovlen,int auth_off,int auth_len,int crypt_off,int crypt_len,int icv_off,uint8_t * ivp)383 octo_des_cbc_decrypt(
384 struct octo_sess *od,
385 struct iovec *iov, size_t iovcnt, size_t iovlen,
386 int auth_off, int auth_len,
387 int crypt_off, int crypt_len,
388 int icv_off, uint8_t *ivp)
389 {
390 uint64_t *data;
391 int data_i, data_l;
392
393 dprintf("%s()\n", __func__);
394
395 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
396 (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
397 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
398 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
399 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
400 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
401 return -EINVAL;
402 }
403
404 IOV_INIT(iov, data, data_i, data_l);
405
406 CVMX_PREFETCH0(ivp);
407 CVMX_PREFETCH0(od->octo_enckey);
408
409 /* load 3DES Key */
410 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
411 if (od->octo_encklen == 24) {
412 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
413 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
414 } else if (od->octo_encklen == 8) {
415 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
416 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
417 } else {
418 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
419 return -EINVAL;
420 }
421
422 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
423
424 while (crypt_off > 0) {
425 IOV_CONSUME(iov, data, data_i, data_l);
426 crypt_off -= 8;
427 }
428
429 while (crypt_len > 0) {
430 CVMX_MT_3DES_DEC_CBC(*data);
431 CVMX_MF_3DES_RESULT(*data);
432 IOV_CONSUME(iov, data, data_i, data_l);
433 crypt_len -= 8;
434 }
435
436 return 0;
437 }
438
439 /****************************************************************************/
440 /* AES functions */
441
442 int
octo_aes_cbc_encrypt(struct octo_sess * od,struct iovec * iov,size_t iovcnt,size_t iovlen,int auth_off,int auth_len,int crypt_off,int crypt_len,int icv_off,uint8_t * ivp)443 octo_aes_cbc_encrypt(
444 struct octo_sess *od,
445 struct iovec *iov, size_t iovcnt, size_t iovlen,
446 int auth_off, int auth_len,
447 int crypt_off, int crypt_len,
448 int icv_off, uint8_t *ivp)
449 {
450 uint64_t *data, *pdata;
451 int data_i, data_l;
452
453 dprintf("%s()\n", __func__);
454
455 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
456 (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
457 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
458 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
459 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
460 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
461 return -EINVAL;
462 }
463
464 IOV_INIT(iov, data, data_i, data_l);
465
466 CVMX_PREFETCH0(ivp);
467 CVMX_PREFETCH0(od->octo_enckey);
468
469 /* load AES Key */
470 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
471 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
472
473 if (od->octo_encklen == 16) {
474 CVMX_MT_AES_KEY(0x0, 2);
475 CVMX_MT_AES_KEY(0x0, 3);
476 } else if (od->octo_encklen == 24) {
477 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
478 CVMX_MT_AES_KEY(0x0, 3);
479 } else if (od->octo_encklen == 32) {
480 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
481 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
482 } else {
483 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
484 return -EINVAL;
485 }
486 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
487
488 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
489 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
490
491 while (crypt_off > 0) {
492 IOV_CONSUME(iov, data, data_i, data_l);
493 crypt_off -= 8;
494 }
495
496 while (crypt_len > 0) {
497 pdata = data;
498 CVMX_MT_AES_ENC_CBC0(*data);
499 IOV_CONSUME(iov, data, data_i, data_l);
500 CVMX_MT_AES_ENC_CBC1(*data);
501 CVMX_MF_AES_RESULT(*pdata, 0);
502 CVMX_MF_AES_RESULT(*data, 1);
503 IOV_CONSUME(iov, data, data_i, data_l);
504 crypt_len -= 16;
505 }
506
507 return 0;
508 }
509
510
511 int
octo_aes_cbc_decrypt(struct octo_sess * od,struct iovec * iov,size_t iovcnt,size_t iovlen,int auth_off,int auth_len,int crypt_off,int crypt_len,int icv_off,uint8_t * ivp)512 octo_aes_cbc_decrypt(
513 struct octo_sess *od,
514 struct iovec *iov, size_t iovcnt, size_t iovlen,
515 int auth_off, int auth_len,
516 int crypt_off, int crypt_len,
517 int icv_off, uint8_t *ivp)
518 {
519 uint64_t *data, *pdata;
520 int data_i, data_l;
521
522 dprintf("%s()\n", __func__);
523
524 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
525 (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
526 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
527 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
528 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
529 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
530 return -EINVAL;
531 }
532
533 IOV_INIT(iov, data, data_i, data_l);
534
535 CVMX_PREFETCH0(ivp);
536 CVMX_PREFETCH0(od->octo_enckey);
537
538 /* load AES Key */
539 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
540 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
541
542 if (od->octo_encklen == 16) {
543 CVMX_MT_AES_KEY(0x0, 2);
544 CVMX_MT_AES_KEY(0x0, 3);
545 } else if (od->octo_encklen == 24) {
546 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
547 CVMX_MT_AES_KEY(0x0, 3);
548 } else if (od->octo_encklen == 32) {
549 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
550 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
551 } else {
552 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
553 return -EINVAL;
554 }
555 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
556
557 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
558 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
559
560 while (crypt_off > 0) {
561 IOV_CONSUME(iov, data, data_i, data_l);
562 crypt_off -= 8;
563 }
564
565 while (crypt_len > 0) {
566 pdata = data;
567 CVMX_MT_AES_DEC_CBC0(*data);
568 IOV_CONSUME(iov, data, data_i, data_l);
569 CVMX_MT_AES_DEC_CBC1(*data);
570 CVMX_MF_AES_RESULT(*pdata, 0);
571 CVMX_MF_AES_RESULT(*data, 1);
572 IOV_CONSUME(iov, data, data_i, data_l);
573 crypt_len -= 16;
574 }
575
576 return 0;
577 }
578
579 /****************************************************************************/
580 /* MD5 */
581
582 int
octo_null_md5_encrypt(struct octo_sess * od,struct iovec * iov,size_t iovcnt,size_t iovlen,int auth_off,int auth_len,int crypt_off,int crypt_len,int icv_off,uint8_t * ivp)583 octo_null_md5_encrypt(
584 struct octo_sess *od,
585 struct iovec *iov, size_t iovcnt, size_t iovlen,
586 int auth_off, int auth_len,
587 int crypt_off, int crypt_len,
588 int icv_off, uint8_t *ivp)
589 {
590 register int next = 0;
591 uint64_t *data;
592 uint64_t tmp1, tmp2;
593 int data_i, data_l, alen = auth_len;
594
595 dprintf("%s()\n", __func__);
596
597 if (__predict_false(od == NULL || iov==NULL || iovlen==0 ||
598 (auth_off & 0x7) || (auth_off + auth_len > iovlen))) {
599 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
600 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
601 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
602 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
603 return -EINVAL;
604 }
605
606 IOV_INIT(iov, data, data_i, data_l);
607
608 /* Load MD5 IV */
609 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
610 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
611
612 while (auth_off > 0) {
613 IOV_CONSUME(iov, data, data_i, data_l);
614 auth_off -= 8;
615 }
616
617 while (auth_len > 0) {
618 CVM_LOAD_MD5_UNIT(*data, next);
619 auth_len -= 8;
620 IOV_CONSUME(iov, data, data_i, data_l);
621 }
622
623 /* finish the hash */
624 CVMX_PREFETCH0(od->octo_hmouter);
625 #if 0
626 if (__predict_false(inplen)) {
627 uint64_t tmp = 0;
628 uint8_t *p = (uint8_t *) & tmp;
629 p[inplen] = 0x80;
630 do {
631 inplen--;
632 p[inplen] = ((uint8_t *) data)[inplen];
633 } while (inplen);
634 CVM_LOAD_MD5_UNIT(tmp, next);
635 } else {
636 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
637 }
638 #else
639 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
640 #endif
641
642 /* Finish Inner hash */
643 while (next != 7) {
644 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
645 }
646 CVMX_ES64(tmp1, ((alen + 64) << 3));
647 CVM_LOAD_MD5_UNIT(tmp1, next);
648
649 /* Get the inner hash of HMAC */
650 CVMX_MF_HSH_IV(tmp1, 0);
651 CVMX_MF_HSH_IV(tmp2, 1);
652
653 /* Initialize hash unit */
654 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
655 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
656
657 CVMX_MT_HSH_DAT(tmp1, 0);
658 CVMX_MT_HSH_DAT(tmp2, 1);
659 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
660 CVMX_MT_HSH_DATZ(3);
661 CVMX_MT_HSH_DATZ(4);
662 CVMX_MT_HSH_DATZ(5);
663 CVMX_MT_HSH_DATZ(6);
664 CVMX_ES64(tmp1, ((64 + 16) << 3));
665 CVMX_MT_HSH_STARTMD5(tmp1);
666
667 /* save the HMAC */
668 IOV_INIT(iov, data, data_i, data_l);
669 while (icv_off > 0) {
670 IOV_CONSUME(iov, data, data_i, data_l);
671 icv_off -= 8;
672 }
673 CVMX_MF_HSH_IV(*data, 0);
674 IOV_CONSUME(iov, data, data_i, data_l);
675 CVMX_MF_HSH_IV(tmp1, 1);
676 *(uint32_t *)data = (uint32_t) (tmp1 >> 32);
677
678 return 0;
679 }
680
681 /****************************************************************************/
682 /* SHA1 */
683
684 int
octo_null_sha1_encrypt(struct octo_sess * od,struct iovec * iov,size_t iovcnt,size_t iovlen,int auth_off,int auth_len,int crypt_off,int crypt_len,int icv_off,uint8_t * ivp)685 octo_null_sha1_encrypt(
686 struct octo_sess *od,
687 struct iovec *iov, size_t iovcnt, size_t iovlen,
688 int auth_off, int auth_len,
689 int crypt_off, int crypt_len,
690 int icv_off, uint8_t *ivp)
691 {
692 register int next = 0;
693 uint64_t *data;
694 uint64_t tmp1, tmp2, tmp3;
695 int data_i, data_l, alen = auth_len;
696
697 dprintf("%s()\n", __func__);
698
699 if (__predict_false(od == NULL || iov==NULL || iovlen==0 ||
700 (auth_off & 0x7) || (auth_off + auth_len > iovlen))) {
701 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
702 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
703 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
704 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
705 return -EINVAL;
706 }
707
708 IOV_INIT(iov, data, data_i, data_l);
709
710 /* Load SHA1 IV */
711 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
712 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
713 CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
714
715 while (auth_off > 0) {
716 IOV_CONSUME(iov, data, data_i, data_l);
717 auth_off -= 8;
718 }
719
720 while (auth_len > 0) {
721 CVM_LOAD_SHA_UNIT(*data, next);
722 auth_len -= 8;
723 IOV_CONSUME(iov, data, data_i, data_l);
724 }
725
726 /* finish the hash */
727 CVMX_PREFETCH0(od->octo_hmouter);
728 #if 0
729 if (__predict_false(inplen)) {
730 uint64_t tmp = 0;
731 uint8_t *p = (uint8_t *) & tmp;
732 p[inplen] = 0x80;
733 do {
734 inplen--;
735 p[inplen] = ((uint8_t *) data)[inplen];
736 } while (inplen);
737 CVM_LOAD_MD5_UNIT(tmp, next);
738 } else {
739 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
740 }
741 #else
742 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
743 #endif
744
745 /* Finish Inner hash */
746 while (next != 7) {
747 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
748 }
749 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
750
751 /* Get the inner hash of HMAC */
752 CVMX_MF_HSH_IV(tmp1, 0);
753 CVMX_MF_HSH_IV(tmp2, 1);
754 tmp3 = 0;
755 CVMX_MF_HSH_IV(tmp3, 2);
756
757 /* Initialize hash unit */
758 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
759 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
760 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
761
762 CVMX_MT_HSH_DAT(tmp1, 0);
763 CVMX_MT_HSH_DAT(tmp2, 1);
764 tmp3 |= 0x0000000080000000;
765 CVMX_MT_HSH_DAT(tmp3, 2);
766 CVMX_MT_HSH_DATZ(3);
767 CVMX_MT_HSH_DATZ(4);
768 CVMX_MT_HSH_DATZ(5);
769 CVMX_MT_HSH_DATZ(6);
770 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
771
772 /* save the HMAC */
773 IOV_INIT(iov, data, data_i, data_l);
774 while (icv_off > 0) {
775 IOV_CONSUME(iov, data, data_i, data_l);
776 icv_off -= 8;
777 }
778 CVMX_MF_HSH_IV(*data, 0);
779 IOV_CONSUME(iov, data, data_i, data_l);
780 CVMX_MF_HSH_IV(tmp1, 1);
781 *(uint32_t *)data = (uint32_t) (tmp1 >> 32);
782
783 return 0;
784 }
785
786 /****************************************************************************/
787 /* DES MD5 */
788
789 int
octo_des_cbc_md5_encrypt(struct octo_sess * od,struct iovec * iov,size_t iovcnt,size_t iovlen,int auth_off,int auth_len,int crypt_off,int crypt_len,int icv_off,uint8_t * ivp)790 octo_des_cbc_md5_encrypt(
791 struct octo_sess *od,
792 struct iovec *iov, size_t iovcnt, size_t iovlen,
793 int auth_off, int auth_len,
794 int crypt_off, int crypt_len,
795 int icv_off, uint8_t *ivp)
796 {
797 register int next = 0;
798 union {
799 uint32_t data32[2];
800 uint64_t data64[1];
801 } mydata;
802 uint64_t *data = &mydata.data64[0];
803 uint32_t *data32;
804 uint64_t tmp1, tmp2;
805 int data_i, data_l, alen = auth_len;
806
807 dprintf("%s()\n", __func__);
808
809 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
810 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
811 (crypt_len & 0x7) ||
812 (auth_len & 0x7) ||
813 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
814 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
815 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
816 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
817 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
818 return -EINVAL;
819 }
820
821 IOV_INIT(iov, data32, data_i, data_l);
822
823 CVMX_PREFETCH0(ivp);
824 CVMX_PREFETCH0(od->octo_enckey);
825
826 /* load 3DES Key */
827 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
828 if (od->octo_encklen == 24) {
829 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
830 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
831 } else if (od->octo_encklen == 8) {
832 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
833 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
834 } else {
835 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
836 return -EINVAL;
837 }
838
839 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
840
841 /* Load MD5 IV */
842 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
843 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
844
845 while (crypt_off > 0 && auth_off > 0) {
846 IOV_CONSUME(iov, data32, data_i, data_l);
847 crypt_off -= 4;
848 auth_off -= 4;
849 }
850
851 while (crypt_len > 0 || auth_len > 0) {
852 uint32_t *first = data32;
853 mydata.data32[0] = *first;
854 IOV_CONSUME(iov, data32, data_i, data_l);
855 mydata.data32[1] = *data32;
856 if (crypt_off <= 0) {
857 if (crypt_len > 0) {
858 CVMX_MT_3DES_ENC_CBC(*data);
859 CVMX_MF_3DES_RESULT(*data);
860 crypt_len -= 8;
861 }
862 } else
863 crypt_off -= 8;
864 if (auth_off <= 0) {
865 if (auth_len > 0) {
866 CVM_LOAD_MD5_UNIT(*data, next);
867 auth_len -= 8;
868 }
869 } else
870 auth_off -= 8;
871 *first = mydata.data32[0];
872 *data32 = mydata.data32[1];
873 IOV_CONSUME(iov, data32, data_i, data_l);
874 }
875
876 /* finish the hash */
877 CVMX_PREFETCH0(od->octo_hmouter);
878 #if 0
879 if (__predict_false(inplen)) {
880 uint64_t tmp = 0;
881 uint8_t *p = (uint8_t *) & tmp;
882 p[inplen] = 0x80;
883 do {
884 inplen--;
885 p[inplen] = ((uint8_t *) data)[inplen];
886 } while (inplen);
887 CVM_LOAD_MD5_UNIT(tmp, next);
888 } else {
889 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
890 }
891 #else
892 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
893 #endif
894
895 /* Finish Inner hash */
896 while (next != 7) {
897 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
898 }
899 CVMX_ES64(tmp1, ((alen + 64) << 3));
900 CVM_LOAD_MD5_UNIT(tmp1, next);
901
902 /* Get the inner hash of HMAC */
903 CVMX_MF_HSH_IV(tmp1, 0);
904 CVMX_MF_HSH_IV(tmp2, 1);
905
906 /* Initialize hash unit */
907 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
908 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
909
910 CVMX_MT_HSH_DAT(tmp1, 0);
911 CVMX_MT_HSH_DAT(tmp2, 1);
912 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
913 CVMX_MT_HSH_DATZ(3);
914 CVMX_MT_HSH_DATZ(4);
915 CVMX_MT_HSH_DATZ(5);
916 CVMX_MT_HSH_DATZ(6);
917 CVMX_ES64(tmp1, ((64 + 16) << 3));
918 CVMX_MT_HSH_STARTMD5(tmp1);
919
920 /* save the HMAC */
921 IOV_INIT(iov, data32, data_i, data_l);
922 while (icv_off > 0) {
923 IOV_CONSUME(iov, data32, data_i, data_l);
924 icv_off -= 4;
925 }
926 CVMX_MF_HSH_IV(tmp1, 0);
927 *data32 = (uint32_t) (tmp1 >> 32);
928 IOV_CONSUME(iov, data32, data_i, data_l);
929 *data32 = (uint32_t) tmp1;
930 IOV_CONSUME(iov, data32, data_i, data_l);
931 CVMX_MF_HSH_IV(tmp1, 1);
932 *data32 = (uint32_t) (tmp1 >> 32);
933
934 return 0;
935 }
936
937 int
octo_des_cbc_md5_decrypt(struct octo_sess * od,struct iovec * iov,size_t iovcnt,size_t iovlen,int auth_off,int auth_len,int crypt_off,int crypt_len,int icv_off,uint8_t * ivp)938 octo_des_cbc_md5_decrypt(
939 struct octo_sess *od,
940 struct iovec *iov, size_t iovcnt, size_t iovlen,
941 int auth_off, int auth_len,
942 int crypt_off, int crypt_len,
943 int icv_off, uint8_t *ivp)
944 {
945 register int next = 0;
946 union {
947 uint32_t data32[2];
948 uint64_t data64[1];
949 } mydata;
950 uint64_t *data = &mydata.data64[0];
951 uint32_t *data32;
952 uint64_t tmp1, tmp2;
953 int data_i, data_l, alen = auth_len;
954
955 dprintf("%s()\n", __func__);
956
957 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
958 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
959 (crypt_len & 0x7) ||
960 (auth_len & 0x7) ||
961 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
962 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
963 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
964 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
965 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
966 return -EINVAL;
967 }
968
969 IOV_INIT(iov, data32, data_i, data_l);
970
971 CVMX_PREFETCH0(ivp);
972 CVMX_PREFETCH0(od->octo_enckey);
973
974 /* load 3DES Key */
975 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
976 if (od->octo_encklen == 24) {
977 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
978 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
979 } else if (od->octo_encklen == 8) {
980 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
981 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
982 } else {
983 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
984 return -EINVAL;
985 }
986
987 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
988
989 /* Load MD5 IV */
990 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
991 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
992
993 while (crypt_off > 0 && auth_off > 0) {
994 IOV_CONSUME(iov, data32, data_i, data_l);
995 crypt_off -= 4;
996 auth_off -= 4;
997 }
998
999 while (crypt_len > 0 || auth_len > 0) {
1000 uint32_t *first = data32;
1001 mydata.data32[0] = *first;
1002 IOV_CONSUME(iov, data32, data_i, data_l);
1003 mydata.data32[1] = *data32;
1004 if (auth_off <= 0) {
1005 if (auth_len > 0) {
1006 CVM_LOAD_MD5_UNIT(*data, next);
1007 auth_len -= 8;
1008 }
1009 } else
1010 auth_off -= 8;
1011 if (crypt_off <= 0) {
1012 if (crypt_len > 0) {
1013 CVMX_MT_3DES_DEC_CBC(*data);
1014 CVMX_MF_3DES_RESULT(*data);
1015 crypt_len -= 8;
1016 }
1017 } else
1018 crypt_off -= 8;
1019 *first = mydata.data32[0];
1020 *data32 = mydata.data32[1];
1021 IOV_CONSUME(iov, data32, data_i, data_l);
1022 }
1023
1024 /* finish the hash */
1025 CVMX_PREFETCH0(od->octo_hmouter);
1026 #if 0
1027 if (__predict_false(inplen)) {
1028 uint64_t tmp = 0;
1029 uint8_t *p = (uint8_t *) & tmp;
1030 p[inplen] = 0x80;
1031 do {
1032 inplen--;
1033 p[inplen] = ((uint8_t *) data)[inplen];
1034 } while (inplen);
1035 CVM_LOAD_MD5_UNIT(tmp, next);
1036 } else {
1037 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1038 }
1039 #else
1040 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1041 #endif
1042
1043 /* Finish Inner hash */
1044 while (next != 7) {
1045 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
1046 }
1047 CVMX_ES64(tmp1, ((alen + 64) << 3));
1048 CVM_LOAD_MD5_UNIT(tmp1, next);
1049
1050 /* Get the inner hash of HMAC */
1051 CVMX_MF_HSH_IV(tmp1, 0);
1052 CVMX_MF_HSH_IV(tmp2, 1);
1053
1054 /* Initialize hash unit */
1055 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1056 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1057
1058 CVMX_MT_HSH_DAT(tmp1, 0);
1059 CVMX_MT_HSH_DAT(tmp2, 1);
1060 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
1061 CVMX_MT_HSH_DATZ(3);
1062 CVMX_MT_HSH_DATZ(4);
1063 CVMX_MT_HSH_DATZ(5);
1064 CVMX_MT_HSH_DATZ(6);
1065 CVMX_ES64(tmp1, ((64 + 16) << 3));
1066 CVMX_MT_HSH_STARTMD5(tmp1);
1067
1068 /* save the HMAC */
1069 IOV_INIT(iov, data32, data_i, data_l);
1070 while (icv_off > 0) {
1071 IOV_CONSUME(iov, data32, data_i, data_l);
1072 icv_off -= 4;
1073 }
1074 CVMX_MF_HSH_IV(tmp1, 0);
1075 *data32 = (uint32_t) (tmp1 >> 32);
1076 IOV_CONSUME(iov, data32, data_i, data_l);
1077 *data32 = (uint32_t) tmp1;
1078 IOV_CONSUME(iov, data32, data_i, data_l);
1079 CVMX_MF_HSH_IV(tmp1, 1);
1080 *data32 = (uint32_t) (tmp1 >> 32);
1081
1082 return 0;
1083 }
1084
1085 /****************************************************************************/
1086 /* DES SHA */
1087
1088 int
octo_des_cbc_sha1_encrypt(struct octo_sess * od,struct iovec * iov,size_t iovcnt,size_t iovlen,int auth_off,int auth_len,int crypt_off,int crypt_len,int icv_off,uint8_t * ivp)1089 octo_des_cbc_sha1_encrypt(
1090 struct octo_sess *od,
1091 struct iovec *iov, size_t iovcnt, size_t iovlen,
1092 int auth_off, int auth_len,
1093 int crypt_off, int crypt_len,
1094 int icv_off, uint8_t *ivp)
1095 {
1096 register int next = 0;
1097 union {
1098 uint32_t data32[2];
1099 uint64_t data64[1];
1100 } mydata;
1101 uint64_t *data = &mydata.data64[0];
1102 uint32_t *data32;
1103 uint64_t tmp1, tmp2, tmp3;
1104 int data_i, data_l, alen = auth_len;
1105
1106 dprintf("%s()\n", __func__);
1107
1108 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
1109 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
1110 (crypt_len & 0x7) ||
1111 (auth_len & 0x7) ||
1112 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
1113 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
1114 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1115 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
1116 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1117 return -EINVAL;
1118 }
1119
1120 IOV_INIT(iov, data32, data_i, data_l);
1121
1122 CVMX_PREFETCH0(ivp);
1123 CVMX_PREFETCH0(od->octo_enckey);
1124
1125 /* load 3DES Key */
1126 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1127 if (od->octo_encklen == 24) {
1128 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1129 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1130 } else if (od->octo_encklen == 8) {
1131 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
1132 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
1133 } else {
1134 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
1135 return -EINVAL;
1136 }
1137
1138 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
1139
1140 /* Load SHA1 IV */
1141 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1142 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1143 CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
1144
1145 while (crypt_off > 0 && auth_off > 0) {
1146 IOV_CONSUME(iov, data32, data_i, data_l);
1147 crypt_off -= 4;
1148 auth_off -= 4;
1149 }
1150
1151 while (crypt_len > 0 || auth_len > 0) {
1152 uint32_t *first = data32;
1153 mydata.data32[0] = *first;
1154 IOV_CONSUME(iov, data32, data_i, data_l);
1155 mydata.data32[1] = *data32;
1156 if (crypt_off <= 0) {
1157 if (crypt_len > 0) {
1158 CVMX_MT_3DES_ENC_CBC(*data);
1159 CVMX_MF_3DES_RESULT(*data);
1160 crypt_len -= 8;
1161 }
1162 } else
1163 crypt_off -= 8;
1164 if (auth_off <= 0) {
1165 if (auth_len > 0) {
1166 CVM_LOAD_SHA_UNIT(*data, next);
1167 auth_len -= 8;
1168 }
1169 } else
1170 auth_off -= 8;
1171 *first = mydata.data32[0];
1172 *data32 = mydata.data32[1];
1173 IOV_CONSUME(iov, data32, data_i, data_l);
1174 }
1175
1176 /* finish the hash */
1177 CVMX_PREFETCH0(od->octo_hmouter);
1178 #if 0
1179 if (__predict_false(inplen)) {
1180 uint64_t tmp = 0;
1181 uint8_t *p = (uint8_t *) & tmp;
1182 p[inplen] = 0x80;
1183 do {
1184 inplen--;
1185 p[inplen] = ((uint8_t *) data)[inplen];
1186 } while (inplen);
1187 CVM_LOAD_SHA_UNIT(tmp, next);
1188 } else {
1189 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1190 }
1191 #else
1192 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1193 #endif
1194
1195 /* Finish Inner hash */
1196 while (next != 7) {
1197 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
1198 }
1199 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
1200
1201 /* Get the inner hash of HMAC */
1202 CVMX_MF_HSH_IV(tmp1, 0);
1203 CVMX_MF_HSH_IV(tmp2, 1);
1204 tmp3 = 0;
1205 CVMX_MF_HSH_IV(tmp3, 2);
1206
1207 /* Initialize hash unit */
1208 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1209 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1210 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
1211
1212 CVMX_MT_HSH_DAT(tmp1, 0);
1213 CVMX_MT_HSH_DAT(tmp2, 1);
1214 tmp3 |= 0x0000000080000000;
1215 CVMX_MT_HSH_DAT(tmp3, 2);
1216 CVMX_MT_HSH_DATZ(3);
1217 CVMX_MT_HSH_DATZ(4);
1218 CVMX_MT_HSH_DATZ(5);
1219 CVMX_MT_HSH_DATZ(6);
1220 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
1221
1222 /* save the HMAC */
1223 IOV_INIT(iov, data32, data_i, data_l);
1224 while (icv_off > 0) {
1225 IOV_CONSUME(iov, data32, data_i, data_l);
1226 icv_off -= 4;
1227 }
1228 CVMX_MF_HSH_IV(tmp1, 0);
1229 *data32 = (uint32_t) (tmp1 >> 32);
1230 IOV_CONSUME(iov, data32, data_i, data_l);
1231 *data32 = (uint32_t) tmp1;
1232 IOV_CONSUME(iov, data32, data_i, data_l);
1233 CVMX_MF_HSH_IV(tmp1, 1);
1234 *data32 = (uint32_t) (tmp1 >> 32);
1235
1236 return 0;
1237 }
1238
1239 int
octo_des_cbc_sha1_decrypt(struct octo_sess * od,struct iovec * iov,size_t iovcnt,size_t iovlen,int auth_off,int auth_len,int crypt_off,int crypt_len,int icv_off,uint8_t * ivp)1240 octo_des_cbc_sha1_decrypt(
1241 struct octo_sess *od,
1242 struct iovec *iov, size_t iovcnt, size_t iovlen,
1243 int auth_off, int auth_len,
1244 int crypt_off, int crypt_len,
1245 int icv_off, uint8_t *ivp)
1246 {
1247 register int next = 0;
1248 union {
1249 uint32_t data32[2];
1250 uint64_t data64[1];
1251 } mydata;
1252 uint64_t *data = &mydata.data64[0];
1253 uint32_t *data32;
1254 uint64_t tmp1, tmp2, tmp3;
1255 int data_i, data_l, alen = auth_len;
1256
1257 dprintf("%s()\n", __func__);
1258
1259 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
1260 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
1261 (crypt_len & 0x7) ||
1262 (auth_len & 0x7) ||
1263 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
1264 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
1265 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1266 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
1267 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1268 return -EINVAL;
1269 }
1270
1271 IOV_INIT(iov, data32, data_i, data_l);
1272
1273 CVMX_PREFETCH0(ivp);
1274 CVMX_PREFETCH0(od->octo_enckey);
1275
1276 /* load 3DES Key */
1277 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1278 if (od->octo_encklen == 24) {
1279 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1280 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1281 } else if (od->octo_encklen == 8) {
1282 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
1283 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
1284 } else {
1285 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
1286 return -EINVAL;
1287 }
1288
1289 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
1290
1291 /* Load SHA1 IV */
1292 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1293 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1294 CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
1295
1296 while (crypt_off > 0 && auth_off > 0) {
1297 IOV_CONSUME(iov, data32, data_i, data_l);
1298 crypt_off -= 4;
1299 auth_off -= 4;
1300 }
1301
1302 while (crypt_len > 0 || auth_len > 0) {
1303 uint32_t *first = data32;
1304 mydata.data32[0] = *first;
1305 IOV_CONSUME(iov, data32, data_i, data_l);
1306 mydata.data32[1] = *data32;
1307 if (auth_off <= 0) {
1308 if (auth_len > 0) {
1309 CVM_LOAD_SHA_UNIT(*data, next);
1310 auth_len -= 8;
1311 }
1312 } else
1313 auth_off -= 8;
1314 if (crypt_off <= 0) {
1315 if (crypt_len > 0) {
1316 CVMX_MT_3DES_DEC_CBC(*data);
1317 CVMX_MF_3DES_RESULT(*data);
1318 crypt_len -= 8;
1319 }
1320 } else
1321 crypt_off -= 8;
1322 *first = mydata.data32[0];
1323 *data32 = mydata.data32[1];
1324 IOV_CONSUME(iov, data32, data_i, data_l);
1325 }
1326
1327 /* finish the hash */
1328 CVMX_PREFETCH0(od->octo_hmouter);
1329 #if 0
1330 if (__predict_false(inplen)) {
1331 uint64_t tmp = 0;
1332 uint8_t *p = (uint8_t *) & tmp;
1333 p[inplen] = 0x80;
1334 do {
1335 inplen--;
1336 p[inplen] = ((uint8_t *) data)[inplen];
1337 } while (inplen);
1338 CVM_LOAD_SHA_UNIT(tmp, next);
1339 } else {
1340 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1341 }
1342 #else
1343 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1344 #endif
1345
1346 /* Finish Inner hash */
1347 while (next != 7) {
1348 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
1349 }
1350 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
1351
1352 /* Get the inner hash of HMAC */
1353 CVMX_MF_HSH_IV(tmp1, 0);
1354 CVMX_MF_HSH_IV(tmp2, 1);
1355 tmp3 = 0;
1356 CVMX_MF_HSH_IV(tmp3, 2);
1357
1358 /* Initialize hash unit */
1359 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1360 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1361 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
1362
1363 CVMX_MT_HSH_DAT(tmp1, 0);
1364 CVMX_MT_HSH_DAT(tmp2, 1);
1365 tmp3 |= 0x0000000080000000;
1366 CVMX_MT_HSH_DAT(tmp3, 2);
1367 CVMX_MT_HSH_DATZ(3);
1368 CVMX_MT_HSH_DATZ(4);
1369 CVMX_MT_HSH_DATZ(5);
1370 CVMX_MT_HSH_DATZ(6);
1371 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
1372 /* save the HMAC */
1373 IOV_INIT(iov, data32, data_i, data_l);
1374 while (icv_off > 0) {
1375 IOV_CONSUME(iov, data32, data_i, data_l);
1376 icv_off -= 4;
1377 }
1378 CVMX_MF_HSH_IV(tmp1, 0);
1379 *data32 = (uint32_t) (tmp1 >> 32);
1380 IOV_CONSUME(iov, data32, data_i, data_l);
1381 *data32 = (uint32_t) tmp1;
1382 IOV_CONSUME(iov, data32, data_i, data_l);
1383 CVMX_MF_HSH_IV(tmp1, 1);
1384 *data32 = (uint32_t) (tmp1 >> 32);
1385
1386 return 0;
1387 }
1388
1389 /****************************************************************************/
1390 /* AES MD5 */
1391
1392 int
octo_aes_cbc_md5_encrypt(struct octo_sess * od,struct iovec * iov,size_t iovcnt,size_t iovlen,int auth_off,int auth_len,int crypt_off,int crypt_len,int icv_off,uint8_t * ivp)1393 octo_aes_cbc_md5_encrypt(
1394 struct octo_sess *od,
1395 struct iovec *iov, size_t iovcnt, size_t iovlen,
1396 int auth_off, int auth_len,
1397 int crypt_off, int crypt_len,
1398 int icv_off, uint8_t *ivp)
1399 {
1400 register int next = 0;
1401 union {
1402 uint32_t data32[2];
1403 uint64_t data64[1];
1404 } mydata[2];
1405 uint64_t *pdata = &mydata[0].data64[0];
1406 uint64_t *data = &mydata[1].data64[0];
1407 uint32_t *data32;
1408 uint64_t tmp1, tmp2;
1409 int data_i, data_l, alen = auth_len;
1410
1411 dprintf("%s()\n", __func__);
1412
1413 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
1414 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
1415 (crypt_len & 0x7) ||
1416 (auth_len & 0x7) ||
1417 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
1418 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
1419 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1420 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
1421 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1422 return -EINVAL;
1423 }
1424
1425 IOV_INIT(iov, data32, data_i, data_l);
1426
1427 CVMX_PREFETCH0(ivp);
1428 CVMX_PREFETCH0(od->octo_enckey);
1429
1430 /* load AES Key */
1431 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1432 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1433
1434 if (od->octo_encklen == 16) {
1435 CVMX_MT_AES_KEY(0x0, 2);
1436 CVMX_MT_AES_KEY(0x0, 3);
1437 } else if (od->octo_encklen == 24) {
1438 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1439 CVMX_MT_AES_KEY(0x0, 3);
1440 } else if (od->octo_encklen == 32) {
1441 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1442 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
1443 } else {
1444 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
1445 return -EINVAL;
1446 }
1447 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
1448
1449 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
1450 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
1451
1452 /* Load MD5 IV */
1453 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1454 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1455
1456 while (crypt_off > 0 && auth_off > 0) {
1457 IOV_CONSUME(iov, data32, data_i, data_l);
1458 crypt_off -= 4;
1459 auth_off -= 4;
1460 }
1461
1462 while (crypt_len > 0 || auth_len > 0) {
1463 uint32_t *pdata32[3];
1464
1465 pdata32[0] = data32;
1466 mydata[0].data32[0] = *data32;
1467 IOV_CONSUME(iov, data32, data_i, data_l);
1468
1469 pdata32[1] = data32;
1470 mydata[0].data32[1] = *data32;
1471 IOV_CONSUME(iov, data32, data_i, data_l);
1472
1473 pdata32[2] = data32;
1474 mydata[1].data32[0] = *data32;
1475 IOV_CONSUME(iov, data32, data_i, data_l);
1476
1477 mydata[1].data32[1] = *data32;
1478
1479
1480 if (crypt_off <= 0) {
1481 if (crypt_len > 0) {
1482 CVMX_MT_AES_ENC_CBC0(*pdata);
1483 CVMX_MT_AES_ENC_CBC1(*data);
1484 CVMX_MF_AES_RESULT(*pdata, 0);
1485 CVMX_MF_AES_RESULT(*data, 1);
1486 crypt_len -= 16;
1487 }
1488 } else
1489 crypt_off -= 16;
1490
1491 if (auth_off <= 0) {
1492 if (auth_len > 0) {
1493 CVM_LOAD_MD5_UNIT(*pdata, next);
1494 CVM_LOAD_MD5_UNIT(*data, next);
1495 auth_len -= 16;
1496 }
1497 } else
1498 auth_off -= 16;
1499
1500 *pdata32[0] = mydata[0].data32[0];
1501 *pdata32[1] = mydata[0].data32[1];
1502 *pdata32[2] = mydata[1].data32[0];
1503 *data32 = mydata[1].data32[1];
1504
1505 IOV_CONSUME(iov, data32, data_i, data_l);
1506 }
1507
1508 /* finish the hash */
1509 CVMX_PREFETCH0(od->octo_hmouter);
1510 #if 0
1511 if (__predict_false(inplen)) {
1512 uint64_t tmp = 0;
1513 uint8_t *p = (uint8_t *) & tmp;
1514 p[inplen] = 0x80;
1515 do {
1516 inplen--;
1517 p[inplen] = ((uint8_t *) data)[inplen];
1518 } while (inplen);
1519 CVM_LOAD_MD5_UNIT(tmp, next);
1520 } else {
1521 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1522 }
1523 #else
1524 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1525 #endif
1526
1527 /* Finish Inner hash */
1528 while (next != 7) {
1529 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
1530 }
1531 CVMX_ES64(tmp1, ((alen + 64) << 3));
1532 CVM_LOAD_MD5_UNIT(tmp1, next);
1533
1534 /* Get the inner hash of HMAC */
1535 CVMX_MF_HSH_IV(tmp1, 0);
1536 CVMX_MF_HSH_IV(tmp2, 1);
1537
1538 /* Initialize hash unit */
1539 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1540 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1541
1542 CVMX_MT_HSH_DAT(tmp1, 0);
1543 CVMX_MT_HSH_DAT(tmp2, 1);
1544 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
1545 CVMX_MT_HSH_DATZ(3);
1546 CVMX_MT_HSH_DATZ(4);
1547 CVMX_MT_HSH_DATZ(5);
1548 CVMX_MT_HSH_DATZ(6);
1549 CVMX_ES64(tmp1, ((64 + 16) << 3));
1550 CVMX_MT_HSH_STARTMD5(tmp1);
1551
1552 /* save the HMAC */
1553 IOV_INIT(iov, data32, data_i, data_l);
1554 while (icv_off > 0) {
1555 IOV_CONSUME(iov, data32, data_i, data_l);
1556 icv_off -= 4;
1557 }
1558 CVMX_MF_HSH_IV(tmp1, 0);
1559 *data32 = (uint32_t) (tmp1 >> 32);
1560 IOV_CONSUME(iov, data32, data_i, data_l);
1561 *data32 = (uint32_t) tmp1;
1562 IOV_CONSUME(iov, data32, data_i, data_l);
1563 CVMX_MF_HSH_IV(tmp1, 1);
1564 *data32 = (uint32_t) (tmp1 >> 32);
1565
1566 return 0;
1567 }
1568
1569 int
octo_aes_cbc_md5_decrypt(struct octo_sess * od,struct iovec * iov,size_t iovcnt,size_t iovlen,int auth_off,int auth_len,int crypt_off,int crypt_len,int icv_off,uint8_t * ivp)1570 octo_aes_cbc_md5_decrypt(
1571 struct octo_sess *od,
1572 struct iovec *iov, size_t iovcnt, size_t iovlen,
1573 int auth_off, int auth_len,
1574 int crypt_off, int crypt_len,
1575 int icv_off, uint8_t *ivp)
1576 {
1577 register int next = 0;
1578 union {
1579 uint32_t data32[2];
1580 uint64_t data64[1];
1581 } mydata[2];
1582 uint64_t *pdata = &mydata[0].data64[0];
1583 uint64_t *data = &mydata[1].data64[0];
1584 uint32_t *data32;
1585 uint64_t tmp1, tmp2;
1586 int data_i, data_l, alen = auth_len;
1587
1588 dprintf("%s()\n", __func__);
1589
1590 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
1591 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
1592 (crypt_len & 0x7) ||
1593 (auth_len & 0x7) ||
1594 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
1595 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
1596 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1597 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
1598 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1599 return -EINVAL;
1600 }
1601
1602 IOV_INIT(iov, data32, data_i, data_l);
1603
1604 CVMX_PREFETCH0(ivp);
1605 CVMX_PREFETCH0(od->octo_enckey);
1606
1607 /* load AES Key */
1608 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1609 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1610
1611 if (od->octo_encklen == 16) {
1612 CVMX_MT_AES_KEY(0x0, 2);
1613 CVMX_MT_AES_KEY(0x0, 3);
1614 } else if (od->octo_encklen == 24) {
1615 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1616 CVMX_MT_AES_KEY(0x0, 3);
1617 } else if (od->octo_encklen == 32) {
1618 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1619 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
1620 } else {
1621 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
1622 return -EINVAL;
1623 }
1624 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
1625
1626 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
1627 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
1628
1629 /* Load MD5 IV */
1630 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1631 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1632
1633 while (crypt_off > 0 && auth_off > 0) {
1634 IOV_CONSUME(iov, data32, data_i, data_l);
1635 crypt_off -= 4;
1636 auth_off -= 4;
1637 }
1638
1639 while (crypt_len > 0 || auth_len > 0) {
1640 uint32_t *pdata32[3];
1641
1642 pdata32[0] = data32;
1643 mydata[0].data32[0] = *data32;
1644 IOV_CONSUME(iov, data32, data_i, data_l);
1645 pdata32[1] = data32;
1646 mydata[0].data32[1] = *data32;
1647 IOV_CONSUME(iov, data32, data_i, data_l);
1648 pdata32[2] = data32;
1649 mydata[1].data32[0] = *data32;
1650 IOV_CONSUME(iov, data32, data_i, data_l);
1651 mydata[1].data32[1] = *data32;
1652
1653 if (auth_off <= 0) {
1654 if (auth_len > 0) {
1655 CVM_LOAD_MD5_UNIT(*pdata, next);
1656 CVM_LOAD_MD5_UNIT(*data, next);
1657 auth_len -= 16;
1658 }
1659 } else
1660 auth_off -= 16;
1661
1662 if (crypt_off <= 0) {
1663 if (crypt_len > 0) {
1664 CVMX_MT_AES_DEC_CBC0(*pdata);
1665 CVMX_MT_AES_DEC_CBC1(*data);
1666 CVMX_MF_AES_RESULT(*pdata, 0);
1667 CVMX_MF_AES_RESULT(*data, 1);
1668 crypt_len -= 16;
1669 }
1670 } else
1671 crypt_off -= 16;
1672
1673 *pdata32[0] = mydata[0].data32[0];
1674 *pdata32[1] = mydata[0].data32[1];
1675 *pdata32[2] = mydata[1].data32[0];
1676 *data32 = mydata[1].data32[1];
1677
1678 IOV_CONSUME(iov, data32, data_i, data_l);
1679 }
1680
1681 /* finish the hash */
1682 CVMX_PREFETCH0(od->octo_hmouter);
1683 #if 0
1684 if (__predict_false(inplen)) {
1685 uint64_t tmp = 0;
1686 uint8_t *p = (uint8_t *) & tmp;
1687 p[inplen] = 0x80;
1688 do {
1689 inplen--;
1690 p[inplen] = ((uint8_t *) data)[inplen];
1691 } while (inplen);
1692 CVM_LOAD_MD5_UNIT(tmp, next);
1693 } else {
1694 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1695 }
1696 #else
1697 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1698 #endif
1699
1700 /* Finish Inner hash */
1701 while (next != 7) {
1702 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
1703 }
1704 CVMX_ES64(tmp1, ((alen + 64) << 3));
1705 CVM_LOAD_MD5_UNIT(tmp1, next);
1706
1707 /* Get the inner hash of HMAC */
1708 CVMX_MF_HSH_IV(tmp1, 0);
1709 CVMX_MF_HSH_IV(tmp2, 1);
1710
1711 /* Initialize hash unit */
1712 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1713 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1714
1715 CVMX_MT_HSH_DAT(tmp1, 0);
1716 CVMX_MT_HSH_DAT(tmp2, 1);
1717 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
1718 CVMX_MT_HSH_DATZ(3);
1719 CVMX_MT_HSH_DATZ(4);
1720 CVMX_MT_HSH_DATZ(5);
1721 CVMX_MT_HSH_DATZ(6);
1722 CVMX_ES64(tmp1, ((64 + 16) << 3));
1723 CVMX_MT_HSH_STARTMD5(tmp1);
1724
1725 /* save the HMAC */
1726 IOV_INIT(iov, data32, data_i, data_l);
1727 while (icv_off > 0) {
1728 IOV_CONSUME(iov, data32, data_i, data_l);
1729 icv_off -= 4;
1730 }
1731 CVMX_MF_HSH_IV(tmp1, 0);
1732 *data32 = (uint32_t) (tmp1 >> 32);
1733 IOV_CONSUME(iov, data32, data_i, data_l);
1734 *data32 = (uint32_t) tmp1;
1735 IOV_CONSUME(iov, data32, data_i, data_l);
1736 CVMX_MF_HSH_IV(tmp1, 1);
1737 *data32 = (uint32_t) (tmp1 >> 32);
1738
1739 return 0;
1740 }
1741
1742 /****************************************************************************/
1743 /* AES SHA1 */
1744
1745 int
octo_aes_cbc_sha1_encrypt(struct octo_sess * od,struct iovec * iov,size_t iovcnt,size_t iovlen,int auth_off,int auth_len,int crypt_off,int crypt_len,int icv_off,uint8_t * ivp)1746 octo_aes_cbc_sha1_encrypt(
1747 struct octo_sess *od,
1748 struct iovec *iov, size_t iovcnt, size_t iovlen,
1749 int auth_off, int auth_len,
1750 int crypt_off, int crypt_len,
1751 int icv_off, uint8_t *ivp)
1752 {
1753 register int next = 0;
1754 union {
1755 uint32_t data32[2];
1756 uint64_t data64[1];
1757 } mydata[2];
1758 uint64_t *pdata = &mydata[0].data64[0];
1759 uint64_t *data = &mydata[1].data64[0];
1760 uint32_t *data32;
1761 uint64_t tmp1, tmp2, tmp3;
1762 int data_i, data_l, alen = auth_len;
1763
1764 dprintf("%s()\n", __func__);
1765
1766 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
1767 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
1768 (crypt_len & 0x7) ||
1769 (auth_len & 0x7) ||
1770 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
1771 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
1772 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1773 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
1774 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1775 return -EINVAL;
1776 }
1777
1778 IOV_INIT(iov, data32, data_i, data_l);
1779
1780 CVMX_PREFETCH0(ivp);
1781 CVMX_PREFETCH0(od->octo_enckey);
1782
1783 /* load AES Key */
1784 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1785 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1786
1787 if (od->octo_encklen == 16) {
1788 CVMX_MT_AES_KEY(0x0, 2);
1789 CVMX_MT_AES_KEY(0x0, 3);
1790 } else if (od->octo_encklen == 24) {
1791 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1792 CVMX_MT_AES_KEY(0x0, 3);
1793 } else if (od->octo_encklen == 32) {
1794 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1795 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
1796 } else {
1797 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
1798 return -EINVAL;
1799 }
1800 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
1801
1802 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
1803 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
1804
1805 /* Load SHA IV */
1806 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1807 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1808 CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
1809
1810 while (crypt_off > 0 && auth_off > 0) {
1811 IOV_CONSUME(iov, data32, data_i, data_l);
1812 crypt_off -= 4;
1813 auth_off -= 4;
1814 }
1815
1816 while (crypt_len > 0 || auth_len > 0) {
1817 uint32_t *pdata32[3];
1818
1819 pdata32[0] = data32;
1820 mydata[0].data32[0] = *data32;
1821 IOV_CONSUME(iov, data32, data_i, data_l);
1822 pdata32[1] = data32;
1823 mydata[0].data32[1] = *data32;
1824 IOV_CONSUME(iov, data32, data_i, data_l);
1825 pdata32[2] = data32;
1826 mydata[1].data32[0] = *data32;
1827 IOV_CONSUME(iov, data32, data_i, data_l);
1828 mydata[1].data32[1] = *data32;
1829
1830
1831 if (crypt_off <= 0) {
1832 if (crypt_len > 0) {
1833 CVMX_MT_AES_ENC_CBC0(*pdata);
1834 CVMX_MT_AES_ENC_CBC1(*data);
1835 CVMX_MF_AES_RESULT(*pdata, 0);
1836 CVMX_MF_AES_RESULT(*data, 1);
1837 crypt_len -= 16;
1838 }
1839 } else
1840 crypt_off -= 16;
1841
1842 if (auth_off <= 0) {
1843 if (auth_len > 0) {
1844 CVM_LOAD_SHA_UNIT(*pdata, next);
1845 CVM_LOAD_SHA_UNIT(*data, next);
1846 auth_len -= 16;
1847 }
1848 } else
1849 auth_off -= 16;
1850
1851 *pdata32[0] = mydata[0].data32[0];
1852 *pdata32[1] = mydata[0].data32[1];
1853 *pdata32[2] = mydata[1].data32[0];
1854 *data32 = mydata[1].data32[1];
1855
1856 IOV_CONSUME(iov, data32, data_i, data_l);
1857 }
1858
1859 /* finish the hash */
1860 CVMX_PREFETCH0(od->octo_hmouter);
1861 #if 0
1862 if (__predict_false(inplen)) {
1863 uint64_t tmp = 0;
1864 uint8_t *p = (uint8_t *) & tmp;
1865 p[inplen] = 0x80;
1866 do {
1867 inplen--;
1868 p[inplen] = ((uint8_t *) data)[inplen];
1869 } while (inplen);
1870 CVM_LOAD_SHA_UNIT(tmp, next);
1871 } else {
1872 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1873 }
1874 #else
1875 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1876 #endif
1877
1878 /* Finish Inner hash */
1879 while (next != 7) {
1880 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
1881 }
1882 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
1883
1884 /* Get the inner hash of HMAC */
1885 CVMX_MF_HSH_IV(tmp1, 0);
1886 CVMX_MF_HSH_IV(tmp2, 1);
1887 tmp3 = 0;
1888 CVMX_MF_HSH_IV(tmp3, 2);
1889
1890 /* Initialize hash unit */
1891 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1892 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1893 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
1894
1895 CVMX_MT_HSH_DAT(tmp1, 0);
1896 CVMX_MT_HSH_DAT(tmp2, 1);
1897 tmp3 |= 0x0000000080000000;
1898 CVMX_MT_HSH_DAT(tmp3, 2);
1899 CVMX_MT_HSH_DATZ(3);
1900 CVMX_MT_HSH_DATZ(4);
1901 CVMX_MT_HSH_DATZ(5);
1902 CVMX_MT_HSH_DATZ(6);
1903 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
1904
1905 /* finish the hash */
1906 CVMX_PREFETCH0(od->octo_hmouter);
1907 #if 0
1908 if (__predict_false(inplen)) {
1909 uint64_t tmp = 0;
1910 uint8_t *p = (uint8_t *) & tmp;
1911 p[inplen] = 0x80;
1912 do {
1913 inplen--;
1914 p[inplen] = ((uint8_t *) data)[inplen];
1915 } while (inplen);
1916 CVM_LOAD_MD5_UNIT(tmp, next);
1917 } else {
1918 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1919 }
1920 #else
1921 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1922 #endif
1923
1924 /* save the HMAC */
1925 IOV_INIT(iov, data32, data_i, data_l);
1926 while (icv_off > 0) {
1927 IOV_CONSUME(iov, data32, data_i, data_l);
1928 icv_off -= 4;
1929 }
1930 CVMX_MF_HSH_IV(tmp1, 0);
1931 *data32 = (uint32_t) (tmp1 >> 32);
1932 IOV_CONSUME(iov, data32, data_i, data_l);
1933 *data32 = (uint32_t) tmp1;
1934 IOV_CONSUME(iov, data32, data_i, data_l);
1935 CVMX_MF_HSH_IV(tmp1, 1);
1936 *data32 = (uint32_t) (tmp1 >> 32);
1937
1938 return 0;
1939 }
1940
1941 int
octo_aes_cbc_sha1_decrypt(struct octo_sess * od,struct iovec * iov,size_t iovcnt,size_t iovlen,int auth_off,int auth_len,int crypt_off,int crypt_len,int icv_off,uint8_t * ivp)1942 octo_aes_cbc_sha1_decrypt(
1943 struct octo_sess *od,
1944 struct iovec *iov, size_t iovcnt, size_t iovlen,
1945 int auth_off, int auth_len,
1946 int crypt_off, int crypt_len,
1947 int icv_off, uint8_t *ivp)
1948 {
1949 register int next = 0;
1950 union {
1951 uint32_t data32[2];
1952 uint64_t data64[1];
1953 } mydata[2];
1954 uint64_t *pdata = &mydata[0].data64[0];
1955 uint64_t *data = &mydata[1].data64[0];
1956 uint32_t *data32;
1957 uint64_t tmp1, tmp2, tmp3;
1958 int data_i, data_l, alen = auth_len;
1959
1960 dprintf("%s()\n", __func__);
1961
1962 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
1963 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
1964 (crypt_len & 0x7) ||
1965 (auth_len & 0x7) ||
1966 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
1967 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
1968 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1969 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
1970 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1971 return -EINVAL;
1972 }
1973
1974 IOV_INIT(iov, data32, data_i, data_l);
1975
1976 CVMX_PREFETCH0(ivp);
1977 CVMX_PREFETCH0(od->octo_enckey);
1978
1979 /* load AES Key */
1980 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1981 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1982
1983 if (od->octo_encklen == 16) {
1984 CVMX_MT_AES_KEY(0x0, 2);
1985 CVMX_MT_AES_KEY(0x0, 3);
1986 } else if (od->octo_encklen == 24) {
1987 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1988 CVMX_MT_AES_KEY(0x0, 3);
1989 } else if (od->octo_encklen == 32) {
1990 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1991 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
1992 } else {
1993 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
1994 return -EINVAL;
1995 }
1996 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
1997
1998 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
1999 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
2000
2001 /* Load MD5 IV */
2002 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
2003 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
2004 CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
2005
2006 while (crypt_off > 0 && auth_off > 0) {
2007 IOV_CONSUME(iov, data32, data_i, data_l);
2008 crypt_off -= 4;
2009 auth_off -= 4;
2010 }
2011
2012 while (crypt_len > 0 || auth_len > 0) {
2013 uint32_t *pdata32[3];
2014
2015 pdata32[0] = data32;
2016 mydata[0].data32[0] = *data32;
2017 IOV_CONSUME(iov, data32, data_i, data_l);
2018 pdata32[1] = data32;
2019 mydata[0].data32[1] = *data32;
2020 IOV_CONSUME(iov, data32, data_i, data_l);
2021 pdata32[2] = data32;
2022 mydata[1].data32[0] = *data32;
2023 IOV_CONSUME(iov, data32, data_i, data_l);
2024 mydata[1].data32[1] = *data32;
2025
2026 if (auth_off <= 0) {
2027 if (auth_len > 0) {
2028 CVM_LOAD_SHA_UNIT(*pdata, next);
2029 CVM_LOAD_SHA_UNIT(*data, next);
2030 auth_len -= 16;
2031 }
2032 } else
2033 auth_off -= 16;
2034
2035 if (crypt_off <= 0) {
2036 if (crypt_len > 0) {
2037 CVMX_MT_AES_DEC_CBC0(*pdata);
2038 CVMX_MT_AES_DEC_CBC1(*data);
2039 CVMX_MF_AES_RESULT(*pdata, 0);
2040 CVMX_MF_AES_RESULT(*data, 1);
2041 crypt_len -= 16;
2042 }
2043 } else
2044 crypt_off -= 16;
2045
2046 *pdata32[0] = mydata[0].data32[0];
2047 *pdata32[1] = mydata[0].data32[1];
2048 *pdata32[2] = mydata[1].data32[0];
2049 *data32 = mydata[1].data32[1];
2050
2051 IOV_CONSUME(iov, data32, data_i, data_l);
2052 }
2053
2054 /* finish the hash */
2055 CVMX_PREFETCH0(od->octo_hmouter);
2056 #if 0
2057 if (__predict_false(inplen)) {
2058 uint64_t tmp = 0;
2059 uint8_t *p = (uint8_t *) & tmp;
2060 p[inplen] = 0x80;
2061 do {
2062 inplen--;
2063 p[inplen] = ((uint8_t *) data)[inplen];
2064 } while (inplen);
2065 CVM_LOAD_SHA_UNIT(tmp, next);
2066 } else {
2067 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
2068 }
2069 #else
2070 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
2071 #endif
2072
2073 /* Finish Inner hash */
2074 while (next != 7) {
2075 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
2076 }
2077 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
2078
2079 /* Get the inner hash of HMAC */
2080 CVMX_MF_HSH_IV(tmp1, 0);
2081 CVMX_MF_HSH_IV(tmp2, 1);
2082 tmp3 = 0;
2083 CVMX_MF_HSH_IV(tmp3, 2);
2084
2085 /* Initialize hash unit */
2086 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
2087 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
2088 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
2089
2090 CVMX_MT_HSH_DAT(tmp1, 0);
2091 CVMX_MT_HSH_DAT(tmp2, 1);
2092 tmp3 |= 0x0000000080000000;
2093 CVMX_MT_HSH_DAT(tmp3, 2);
2094 CVMX_MT_HSH_DATZ(3);
2095 CVMX_MT_HSH_DATZ(4);
2096 CVMX_MT_HSH_DATZ(5);
2097 CVMX_MT_HSH_DATZ(6);
2098 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
2099
2100 /* finish the hash */
2101 CVMX_PREFETCH0(od->octo_hmouter);
2102 #if 0
2103 if (__predict_false(inplen)) {
2104 uint64_t tmp = 0;
2105 uint8_t *p = (uint8_t *) & tmp;
2106 p[inplen] = 0x80;
2107 do {
2108 inplen--;
2109 p[inplen] = ((uint8_t *) data)[inplen];
2110 } while (inplen);
2111 CVM_LOAD_MD5_UNIT(tmp, next);
2112 } else {
2113 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
2114 }
2115 #else
2116 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
2117 #endif
2118
2119 /* save the HMAC */
2120 IOV_INIT(iov, data32, data_i, data_l);
2121 while (icv_off > 0) {
2122 IOV_CONSUME(iov, data32, data_i, data_l);
2123 icv_off -= 4;
2124 }
2125 CVMX_MF_HSH_IV(tmp1, 0);
2126 *data32 = (uint32_t) (tmp1 >> 32);
2127 IOV_CONSUME(iov, data32, data_i, data_l);
2128 *data32 = (uint32_t) tmp1;
2129 IOV_CONSUME(iov, data32, data_i, data_l);
2130 CVMX_MF_HSH_IV(tmp1, 1);
2131 *data32 = (uint32_t) (tmp1 >> 32);
2132
2133 return 0;
2134 }
2135
2136 /****************************************************************************/
2137