1 /*-
2 * Copyright (c) 2017 Chelsio Communications, Inc.
3 * Copyright (c) 2021 The FreeBSD Foundation
4 * All rights reserved.
5 * Written by: John Baldwin <jhb@FreeBSD.org>
6 *
7 * Portions of this software were developed by Ararat River
8 * Consulting, LLC under sponsorship of the FreeBSD Foundation.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 #include <sys/cdefs.h>
33 #include <sys/types.h>
34 #include <sys/bus.h>
35 #include <sys/lock.h>
36 #include <sys/malloc.h>
37 #include <sys/mutex.h>
38 #include <sys/module.h>
39 #include <sys/sglist.h>
40
41 #include <opencrypto/cryptodev.h>
42 #include <opencrypto/xform.h>
43
44 #include "cryptodev_if.h"
45
46 #include "common/common.h"
47 #include "crypto/t4_crypto.h"
48
49 /*
50 * Requests consist of:
51 *
52 * +-------------------------------+
53 * | struct fw_crypto_lookaside_wr |
54 * +-------------------------------+
55 * | struct ulp_txpkt |
56 * +-------------------------------+
57 * | struct ulptx_idata |
58 * +-------------------------------+
59 * | struct cpl_tx_sec_pdu |
60 * +-------------------------------+
61 * | struct cpl_tls_tx_scmd_fmt |
62 * +-------------------------------+
63 * | key context header |
64 * +-------------------------------+
65 * | AES key | ----- For requests with AES
66 * +-------------------------------+
67 * | Hash state | ----- For hash-only requests
68 * +-------------------------------+ -
69 * | IPAD (16-byte aligned) | \
70 * +-------------------------------+ +---- For requests with HMAC
71 * | OPAD (16-byte aligned) | /
72 * +-------------------------------+ -
73 * | GMAC H | ----- For AES-GCM
74 * +-------------------------------+ -
75 * | struct cpl_rx_phys_dsgl | \
76 * +-------------------------------+ +---- Destination buffer for
77 * | PHYS_DSGL entries | / non-hash-only requests
78 * +-------------------------------+ -
79 * | 16 dummy bytes | ----- Only for HMAC/hash-only requests
80 * +-------------------------------+
81 * | IV | ----- If immediate IV
82 * +-------------------------------+
83 * | Payload | ----- If immediate Payload
84 * +-------------------------------+ -
85 * | struct ulptx_sgl | \
86 * +-------------------------------+ +---- If payload via SGL
87 * | SGL entries | /
88 * +-------------------------------+ -
89 *
90 * Note that the key context must be padded to ensure 16-byte alignment.
91 * For HMAC requests, the key consists of the partial hash of the IPAD
92 * followed by the partial hash of the OPAD.
93 *
94 * Replies consist of:
95 *
96 * +-------------------------------+
97 * | struct cpl_fw6_pld |
98 * +-------------------------------+
99 * | hash digest | ----- For HMAC request with
100 * +-------------------------------+ 'hash_size' set in work request
101 *
102 * A 32-bit big-endian error status word is supplied in the last 4
103 * bytes of data[0] in the CPL_FW6_PLD message. bit 0 indicates a
104 * "MAC" error and bit 1 indicates a "PAD" error.
105 *
106 * The 64-bit 'cookie' field from the fw_crypto_lookaside_wr message
107 * in the request is returned in data[1] of the CPL_FW6_PLD message.
108 *
109 * For block cipher replies, the updated IV is supplied in data[2] and
110 * data[3] of the CPL_FW6_PLD message.
111 *
112 * For hash replies where the work request set 'hash_size' to request
113 * a copy of the hash in the reply, the hash digest is supplied
114 * immediately following the CPL_FW6_PLD message.
115 */
116
117 /*
118 * The crypto engine supports a maximum AAD size of 511 bytes.
119 */
120 #define MAX_AAD_LEN 511
121
122 /*
123 * The documentation for CPL_RX_PHYS_DSGL claims a maximum of 32 SG
124 * entries. While the CPL includes a 16-bit length field, the T6 can
125 * sometimes hang if an error occurs while processing a request with a
126 * single DSGL entry larger than 2k.
127 */
128 #define MAX_RX_PHYS_DSGL_SGE 32
129 #define DSGL_SGE_MAXLEN 2048
130
131 /*
132 * The adapter only supports requests with a total input or output
133 * length of 64k-1 or smaller. Longer requests either result in hung
134 * requests or incorrect results.
135 */
136 #define MAX_REQUEST_SIZE 65535
137
138 static MALLOC_DEFINE(M_CCR, "ccr", "Chelsio T6 crypto");
139
140 struct ccr_session_hmac {
141 const struct auth_hash *auth_hash;
142 int hash_len;
143 unsigned int partial_digest_len;
144 unsigned int auth_mode;
145 unsigned int mk_size;
146 char pads[CHCR_HASH_MAX_BLOCK_SIZE_128 * 2];
147 };
148
149 struct ccr_session_gmac {
150 int hash_len;
151 char ghash_h[GMAC_BLOCK_LEN];
152 };
153
154 struct ccr_session_ccm_mac {
155 int hash_len;
156 };
157
158 struct ccr_session_cipher {
159 unsigned int cipher_mode;
160 unsigned int key_len;
161 unsigned int iv_len;
162 __be32 key_ctx_hdr;
163 char enckey[CHCR_AES_MAX_KEY_LEN];
164 char deckey[CHCR_AES_MAX_KEY_LEN];
165 };
166
167 struct ccr_port {
168 struct sge_wrq *txq;
169 struct sge_rxq *rxq;
170 int rx_channel_id;
171 int tx_channel_id;
172 u_int active_sessions;
173
174 counter_u64_t stats_queued;
175 counter_u64_t stats_completed;
176 };
177
178 struct ccr_softc {
179 struct adapter *adapter;
180 device_t dev;
181 uint32_t cid;
182 struct mtx lock;
183 bool detaching;
184 struct ccr_port ports[MAX_NPORTS];
185 u_int port_mask;
186 int first_rxq_id;
187
188 /*
189 * Pre-allocate a dummy output buffer for the IV and AAD for
190 * AEAD requests.
191 */
192 char *iv_aad_buf;
193 struct sglist *sg_iv_aad;
194
195 /* Statistics. */
196 counter_u64_t stats_cipher_encrypt;
197 counter_u64_t stats_cipher_decrypt;
198 counter_u64_t stats_hash;
199 counter_u64_t stats_hmac;
200 counter_u64_t stats_eta_encrypt;
201 counter_u64_t stats_eta_decrypt;
202 counter_u64_t stats_gcm_encrypt;
203 counter_u64_t stats_gcm_decrypt;
204 counter_u64_t stats_ccm_encrypt;
205 counter_u64_t stats_ccm_decrypt;
206 counter_u64_t stats_wr_nomem;
207 counter_u64_t stats_inflight;
208 counter_u64_t stats_mac_error;
209 counter_u64_t stats_pad_error;
210 counter_u64_t stats_sglist_error;
211 counter_u64_t stats_process_error;
212 counter_u64_t stats_sw_fallback;
213
214 struct sysctl_ctx_list ctx;
215 };
216
217 struct ccr_session {
218 #ifdef INVARIANTS
219 int pending;
220 #endif
221 enum { HASH, HMAC, CIPHER, ETA, GCM, CCM } mode;
222 struct ccr_softc *sc;
223 struct ccr_port *port;
224 union {
225 struct ccr_session_hmac hmac;
226 struct ccr_session_gmac gmac;
227 struct ccr_session_ccm_mac ccm_mac;
228 };
229 struct ccr_session_cipher cipher;
230 struct mtx lock;
231
232 /*
233 * A fallback software session is used for certain GCM/CCM
234 * requests that the hardware can't handle such as requests
235 * with only AAD and no payload.
236 */
237 crypto_session_t sw_session;
238
239 /*
240 * Pre-allocate S/G lists used when preparing a work request.
241 * 'sg_input' contains an sglist describing the entire input
242 * buffer for a 'struct cryptop'. 'sg_output' contains an
243 * sglist describing the entire output buffer. 'sg_ulptx' is
244 * used to describe the data the engine should DMA as input
245 * via ULPTX_SGL. 'sg_dsgl' is used to describe the
246 * destination that cipher text and a tag should be written
247 * to.
248 */
249 struct sglist *sg_input;
250 struct sglist *sg_output;
251 struct sglist *sg_ulptx;
252 struct sglist *sg_dsgl;
253 };
254
255 /*
256 * Crypto requests involve two kind of scatter/gather lists.
257 *
258 * Non-hash-only requests require a PHYS_DSGL that describes the
259 * location to store the results of the encryption or decryption
260 * operation. This SGL uses a different format (PHYS_DSGL) and should
261 * exclude the skip bytes at the start of the data as well as any AAD
262 * or IV. For authenticated encryption requests it should include the
263 * destination of the hash or tag.
264 *
265 * The input payload may either be supplied inline as immediate data,
266 * or via a standard ULP_TX SGL. This SGL should include AAD,
267 * ciphertext, and the hash or tag for authenticated decryption
268 * requests.
269 *
270 * These scatter/gather lists can describe different subsets of the
271 * buffers described by the crypto operation. ccr_populate_sglist()
272 * generates a scatter/gather list that covers an entire crypto
273 * operation buffer that is then used to construct the other
274 * scatter/gather lists.
275 */
276 static int
ccr_populate_sglist(struct sglist * sg,struct crypto_buffer * cb)277 ccr_populate_sglist(struct sglist *sg, struct crypto_buffer *cb)
278 {
279 int error;
280
281 sglist_reset(sg);
282 switch (cb->cb_type) {
283 case CRYPTO_BUF_MBUF:
284 error = sglist_append_mbuf(sg, cb->cb_mbuf);
285 break;
286 case CRYPTO_BUF_SINGLE_MBUF:
287 error = sglist_append_single_mbuf(sg, cb->cb_mbuf);
288 break;
289 case CRYPTO_BUF_UIO:
290 error = sglist_append_uio(sg, cb->cb_uio);
291 break;
292 case CRYPTO_BUF_CONTIG:
293 error = sglist_append(sg, cb->cb_buf, cb->cb_buf_len);
294 break;
295 case CRYPTO_BUF_VMPAGE:
296 error = sglist_append_vmpages(sg, cb->cb_vm_page,
297 cb->cb_vm_page_offset, cb->cb_vm_page_len);
298 break;
299 default:
300 error = EINVAL;
301 }
302 return (error);
303 }
304
305 /*
306 * Segments in 'sg' larger than 'maxsegsize' are counted as multiple
307 * segments.
308 */
309 static int
ccr_count_sgl(struct sglist * sg,int maxsegsize)310 ccr_count_sgl(struct sglist *sg, int maxsegsize)
311 {
312 int i, nsegs;
313
314 nsegs = 0;
315 for (i = 0; i < sg->sg_nseg; i++)
316 nsegs += howmany(sg->sg_segs[i].ss_len, maxsegsize);
317 return (nsegs);
318 }
319
320 /* These functions deal with PHYS_DSGL for the reply buffer. */
321 static inline int
ccr_phys_dsgl_len(int nsegs)322 ccr_phys_dsgl_len(int nsegs)
323 {
324 int len;
325
326 len = (nsegs / 8) * sizeof(struct phys_sge_pairs);
327 if ((nsegs % 8) != 0) {
328 len += sizeof(uint16_t) * 8;
329 len += roundup2(nsegs % 8, 2) * sizeof(uint64_t);
330 }
331 return (len);
332 }
333
334 static void
ccr_write_phys_dsgl(struct ccr_session * s,void * dst,int nsegs)335 ccr_write_phys_dsgl(struct ccr_session *s, void *dst, int nsegs)
336 {
337 struct sglist *sg;
338 struct cpl_rx_phys_dsgl *cpl;
339 struct phys_sge_pairs *sgl;
340 vm_paddr_t paddr;
341 size_t seglen;
342 u_int i, j;
343
344 sg = s->sg_dsgl;
345 cpl = dst;
346 cpl->op_to_tid = htobe32(V_CPL_RX_PHYS_DSGL_OPCODE(CPL_RX_PHYS_DSGL) |
347 V_CPL_RX_PHYS_DSGL_ISRDMA(0));
348 cpl->pcirlxorder_to_noofsgentr = htobe32(
349 V_CPL_RX_PHYS_DSGL_PCIRLXORDER(0) |
350 V_CPL_RX_PHYS_DSGL_PCINOSNOOP(0) |
351 V_CPL_RX_PHYS_DSGL_PCITPHNTENB(0) | V_CPL_RX_PHYS_DSGL_DCAID(0) |
352 V_CPL_RX_PHYS_DSGL_NOOFSGENTR(nsegs));
353 cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
354 cpl->rss_hdr_int.qid = htobe16(s->port->rxq->iq.abs_id);
355 cpl->rss_hdr_int.hash_val = 0;
356 cpl->rss_hdr_int.channel = s->port->rx_channel_id;
357 sgl = (struct phys_sge_pairs *)(cpl + 1);
358 j = 0;
359 for (i = 0; i < sg->sg_nseg; i++) {
360 seglen = sg->sg_segs[i].ss_len;
361 paddr = sg->sg_segs[i].ss_paddr;
362 do {
363 sgl->addr[j] = htobe64(paddr);
364 if (seglen > DSGL_SGE_MAXLEN) {
365 sgl->len[j] = htobe16(DSGL_SGE_MAXLEN);
366 paddr += DSGL_SGE_MAXLEN;
367 seglen -= DSGL_SGE_MAXLEN;
368 } else {
369 sgl->len[j] = htobe16(seglen);
370 seglen = 0;
371 }
372 j++;
373 if (j == 8) {
374 sgl++;
375 j = 0;
376 }
377 } while (seglen != 0);
378 }
379 MPASS(j + 8 * (sgl - (struct phys_sge_pairs *)(cpl + 1)) == nsegs);
380 }
381
382 /* These functions deal with the ULPTX_SGL for input payload. */
383 static inline int
ccr_ulptx_sgl_len(int nsegs)384 ccr_ulptx_sgl_len(int nsegs)
385 {
386 u_int n;
387
388 nsegs--; /* first segment is part of ulptx_sgl */
389 n = sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1));
390 return (roundup2(n, 16));
391 }
392
393 static void
ccr_write_ulptx_sgl(struct ccr_session * s,void * dst,int nsegs)394 ccr_write_ulptx_sgl(struct ccr_session *s, void *dst, int nsegs)
395 {
396 struct ulptx_sgl *usgl;
397 struct sglist *sg;
398 struct sglist_seg *ss;
399 int i;
400
401 sg = s->sg_ulptx;
402 MPASS(nsegs == sg->sg_nseg);
403 ss = &sg->sg_segs[0];
404 usgl = dst;
405 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
406 V_ULPTX_NSGE(nsegs));
407 usgl->len0 = htobe32(ss->ss_len);
408 usgl->addr0 = htobe64(ss->ss_paddr);
409 ss++;
410 for (i = 0; i < sg->sg_nseg - 1; i++) {
411 usgl->sge[i / 2].len[i & 1] = htobe32(ss->ss_len);
412 usgl->sge[i / 2].addr[i & 1] = htobe64(ss->ss_paddr);
413 ss++;
414 }
415 }
416
417 static bool
ccr_use_imm_data(u_int transhdr_len,u_int input_len)418 ccr_use_imm_data(u_int transhdr_len, u_int input_len)
419 {
420
421 if (input_len > CRYPTO_MAX_IMM_TX_PKT_LEN)
422 return (false);
423 if (roundup2(transhdr_len, 16) + roundup2(input_len, 16) >
424 SGE_MAX_WR_LEN)
425 return (false);
426 return (true);
427 }
428
429 static void
ccr_populate_wreq(struct ccr_softc * sc,struct ccr_session * s,struct chcr_wr * crwr,u_int kctx_len,u_int wr_len,u_int imm_len,u_int sgl_len,u_int hash_size,struct cryptop * crp)430 ccr_populate_wreq(struct ccr_softc *sc, struct ccr_session *s,
431 struct chcr_wr *crwr, u_int kctx_len, u_int wr_len, u_int imm_len,
432 u_int sgl_len, u_int hash_size, struct cryptop *crp)
433 {
434 u_int cctx_size, idata_len;
435
436 cctx_size = sizeof(struct _key_ctx) + kctx_len;
437 crwr->wreq.op_to_cctx_size = htobe32(
438 V_FW_CRYPTO_LOOKASIDE_WR_OPCODE(FW_CRYPTO_LOOKASIDE_WR) |
439 V_FW_CRYPTO_LOOKASIDE_WR_COMPL(0) |
440 V_FW_CRYPTO_LOOKASIDE_WR_IMM_LEN(imm_len) |
441 V_FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC(1) |
442 V_FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE(cctx_size >> 4));
443 crwr->wreq.len16_pkd = htobe32(
444 V_FW_CRYPTO_LOOKASIDE_WR_LEN16(wr_len / 16));
445 crwr->wreq.session_id = 0;
446 crwr->wreq.rx_chid_to_rx_q_id = htobe32(
447 V_FW_CRYPTO_LOOKASIDE_WR_RX_CHID(s->port->rx_channel_id) |
448 V_FW_CRYPTO_LOOKASIDE_WR_LCB(0) |
449 V_FW_CRYPTO_LOOKASIDE_WR_PHASH(0) |
450 V_FW_CRYPTO_LOOKASIDE_WR_IV(IV_NOP) |
451 V_FW_CRYPTO_LOOKASIDE_WR_FQIDX(0) |
452 V_FW_CRYPTO_LOOKASIDE_WR_TX_CH(0) | /* unused in firmware */
453 V_FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID(s->port->rxq->iq.abs_id));
454 crwr->wreq.key_addr = 0;
455 crwr->wreq.pld_size_hash_size = htobe32(
456 V_FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE(sgl_len) |
457 V_FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE(hash_size));
458 crwr->wreq.cookie = htobe64((uintptr_t)crp);
459
460 crwr->ulptx.cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) |
461 V_ULP_TXPKT_DATAMODIFY(0) |
462 V_ULP_TXPKT_CHANNELID(s->port->tx_channel_id) |
463 V_ULP_TXPKT_DEST(0) |
464 V_ULP_TXPKT_FID(sc->first_rxq_id) | V_ULP_TXPKT_RO(1));
465 crwr->ulptx.len = htobe32(
466 ((wr_len - sizeof(struct fw_crypto_lookaside_wr)) / 16));
467
468 crwr->sc_imm.cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) |
469 V_ULP_TX_SC_MORE(sgl_len != 0 ? 1 : 0));
470 idata_len = wr_len - offsetof(struct chcr_wr, sec_cpl) - sgl_len;
471 if (imm_len % 16 != 0)
472 idata_len -= 16 - imm_len % 16;
473 crwr->sc_imm.len = htobe32(idata_len);
474 }
475
476 static int
ccr_hash(struct ccr_softc * sc,struct ccr_session * s,struct cryptop * crp)477 ccr_hash(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
478 {
479 struct chcr_wr *crwr;
480 struct wrqe *wr;
481 const struct auth_hash *axf;
482 char *dst;
483 u_int hash_size_in_response, kctx_flits, kctx_len, transhdr_len, wr_len;
484 u_int hmac_ctrl, imm_len, iopad_size;
485 int error, sgl_nsegs, sgl_len, use_opad;
486
487 /* Reject requests with too large of an input buffer. */
488 if (crp->crp_payload_length > MAX_REQUEST_SIZE)
489 return (EFBIG);
490
491 axf = s->hmac.auth_hash;
492
493 if (s->mode == HMAC) {
494 use_opad = 1;
495 hmac_ctrl = SCMD_HMAC_CTRL_NO_TRUNC;
496 } else {
497 use_opad = 0;
498 hmac_ctrl = SCMD_HMAC_CTRL_NOP;
499 }
500
501 /* PADs must be 128-bit aligned. */
502 iopad_size = roundup2(s->hmac.partial_digest_len, 16);
503
504 /*
505 * The 'key' part of the context includes the aligned IPAD and
506 * OPAD.
507 */
508 kctx_len = iopad_size;
509 if (use_opad)
510 kctx_len += iopad_size;
511 hash_size_in_response = axf->hashsize;
512 transhdr_len = HASH_TRANSHDR_SIZE(kctx_len);
513
514 if (crp->crp_payload_length == 0) {
515 imm_len = axf->blocksize;
516 sgl_nsegs = 0;
517 sgl_len = 0;
518 } else if (ccr_use_imm_data(transhdr_len, crp->crp_payload_length)) {
519 imm_len = crp->crp_payload_length;
520 sgl_nsegs = 0;
521 sgl_len = 0;
522 } else {
523 imm_len = 0;
524 sglist_reset(s->sg_ulptx);
525 error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
526 crp->crp_payload_start, crp->crp_payload_length);
527 if (error)
528 return (error);
529 sgl_nsegs = s->sg_ulptx->sg_nseg;
530 sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
531 }
532
533 wr_len = roundup2(transhdr_len, 16) + roundup2(imm_len, 16) + sgl_len;
534 if (wr_len > SGE_MAX_WR_LEN)
535 return (EFBIG);
536 wr = alloc_wrqe(wr_len, s->port->txq);
537 if (wr == NULL) {
538 counter_u64_add(sc->stats_wr_nomem, 1);
539 return (ENOMEM);
540 }
541 crwr = wrtod(wr);
542 memset(crwr, 0, wr_len);
543
544 ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len,
545 hash_size_in_response, crp);
546
547 crwr->sec_cpl.op_ivinsrtofst = htobe32(
548 V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) |
549 V_CPL_TX_SEC_PDU_RXCHID(s->port->rx_channel_id) |
550 V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) |
551 V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
552 V_CPL_TX_SEC_PDU_IVINSRTOFST(0));
553
554 crwr->sec_cpl.pldlen = htobe32(crp->crp_payload_length == 0 ?
555 axf->blocksize : crp->crp_payload_length);
556
557 crwr->sec_cpl.cipherstop_lo_authinsert = htobe32(
558 V_CPL_TX_SEC_PDU_AUTHSTART(1) | V_CPL_TX_SEC_PDU_AUTHSTOP(0));
559
560 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
561 crwr->sec_cpl.seqno_numivs = htobe32(
562 V_SCMD_SEQ_NO_CTRL(0) |
563 V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) |
564 V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_NOP) |
565 V_SCMD_AUTH_MODE(s->hmac.auth_mode) |
566 V_SCMD_HMAC_CTRL(hmac_ctrl));
567 crwr->sec_cpl.ivgen_hdrlen = htobe32(
568 V_SCMD_LAST_FRAG(0) |
569 V_SCMD_MORE_FRAGS(crp->crp_payload_length == 0 ? 1 : 0) |
570 V_SCMD_MAC_ONLY(1));
571
572 memcpy(crwr->key_ctx.key, s->hmac.pads, kctx_len);
573
574 /* XXX: F_KEY_CONTEXT_SALT_PRESENT set, but 'salt' not set. */
575 kctx_flits = (sizeof(struct _key_ctx) + kctx_len) / 16;
576 crwr->key_ctx.ctx_hdr = htobe32(V_KEY_CONTEXT_CTX_LEN(kctx_flits) |
577 V_KEY_CONTEXT_OPAD_PRESENT(use_opad) |
578 V_KEY_CONTEXT_SALT_PRESENT(1) |
579 V_KEY_CONTEXT_CK_SIZE(CHCR_KEYCTX_NO_KEY) |
580 V_KEY_CONTEXT_MK_SIZE(s->hmac.mk_size) | V_KEY_CONTEXT_VALID(1));
581
582 dst = (char *)(crwr + 1) + kctx_len + DUMMY_BYTES;
583 if (crp->crp_payload_length == 0) {
584 dst[0] = 0x80;
585 if (s->mode == HMAC)
586 *(uint64_t *)(dst + axf->blocksize - sizeof(uint64_t)) =
587 htobe64(axf->blocksize << 3);
588 } else if (imm_len != 0)
589 crypto_copydata(crp, crp->crp_payload_start,
590 crp->crp_payload_length, dst);
591 else
592 ccr_write_ulptx_sgl(s, dst, sgl_nsegs);
593
594 /* XXX: TODO backpressure */
595 t4_wrq_tx(sc->adapter, wr);
596
597 return (0);
598 }
599
600 static int
ccr_hash_done(struct ccr_softc * sc,struct ccr_session * s,struct cryptop * crp,const struct cpl_fw6_pld * cpl,int error)601 ccr_hash_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
602 const struct cpl_fw6_pld *cpl, int error)
603 {
604 uint8_t hash[HASH_MAX_LEN];
605
606 if (error)
607 return (error);
608
609 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
610 crypto_copydata(crp, crp->crp_digest_start, s->hmac.hash_len,
611 hash);
612 if (timingsafe_bcmp((cpl + 1), hash, s->hmac.hash_len) != 0)
613 return (EBADMSG);
614 } else
615 crypto_copyback(crp, crp->crp_digest_start, s->hmac.hash_len,
616 (cpl + 1));
617 return (0);
618 }
619
620 static int
ccr_cipher(struct ccr_softc * sc,struct ccr_session * s,struct cryptop * crp)621 ccr_cipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
622 {
623 char iv[CHCR_MAX_CRYPTO_IV_LEN];
624 struct chcr_wr *crwr;
625 struct wrqe *wr;
626 char *dst;
627 u_int kctx_len, key_half, op_type, transhdr_len, wr_len;
628 u_int imm_len, iv_len;
629 int dsgl_nsegs, dsgl_len;
630 int sgl_nsegs, sgl_len;
631 int error;
632
633 if (s->cipher.key_len == 0 || crp->crp_payload_length == 0)
634 return (EINVAL);
635 if (s->cipher.cipher_mode == SCMD_CIPH_MODE_AES_CBC &&
636 (crp->crp_payload_length % AES_BLOCK_LEN) != 0)
637 return (EINVAL);
638
639 /* Reject requests with too large of an input buffer. */
640 if (crp->crp_payload_length > MAX_REQUEST_SIZE)
641 return (EFBIG);
642
643 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
644 op_type = CHCR_ENCRYPT_OP;
645 else
646 op_type = CHCR_DECRYPT_OP;
647
648 sglist_reset(s->sg_dsgl);
649 if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
650 error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
651 crp->crp_payload_output_start, crp->crp_payload_length);
652 else
653 error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
654 crp->crp_payload_start, crp->crp_payload_length);
655 if (error)
656 return (error);
657 dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN);
658 if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
659 return (EFBIG);
660 dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
661
662 /* The 'key' must be 128-bit aligned. */
663 kctx_len = roundup2(s->cipher.key_len, 16);
664 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len);
665
666 /* For AES-XTS we send a 16-byte IV in the work request. */
667 if (s->cipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS)
668 iv_len = AES_BLOCK_LEN;
669 else
670 iv_len = s->cipher.iv_len;
671
672 if (ccr_use_imm_data(transhdr_len, crp->crp_payload_length + iv_len)) {
673 imm_len = crp->crp_payload_length;
674 sgl_nsegs = 0;
675 sgl_len = 0;
676 } else {
677 imm_len = 0;
678 sglist_reset(s->sg_ulptx);
679 error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
680 crp->crp_payload_start, crp->crp_payload_length);
681 if (error)
682 return (error);
683 sgl_nsegs = s->sg_ulptx->sg_nseg;
684 sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
685 }
686
687 wr_len = roundup2(transhdr_len, 16) + iv_len +
688 roundup2(imm_len, 16) + sgl_len;
689 if (wr_len > SGE_MAX_WR_LEN)
690 return (EFBIG);
691 wr = alloc_wrqe(wr_len, s->port->txq);
692 if (wr == NULL) {
693 counter_u64_add(sc->stats_wr_nomem, 1);
694 return (ENOMEM);
695 }
696 crwr = wrtod(wr);
697 memset(crwr, 0, wr_len);
698
699 crypto_read_iv(crp, iv);
700
701 /* Zero the remainder of the IV for AES-XTS. */
702 memset(iv + s->cipher.iv_len, 0, iv_len - s->cipher.iv_len);
703
704 ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len, 0,
705 crp);
706
707 crwr->sec_cpl.op_ivinsrtofst = htobe32(
708 V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) |
709 V_CPL_TX_SEC_PDU_RXCHID(s->port->rx_channel_id) |
710 V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) |
711 V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
712 V_CPL_TX_SEC_PDU_IVINSRTOFST(1));
713
714 crwr->sec_cpl.pldlen = htobe32(iv_len + crp->crp_payload_length);
715
716 crwr->sec_cpl.aadstart_cipherstop_hi = htobe32(
717 V_CPL_TX_SEC_PDU_CIPHERSTART(iv_len + 1) |
718 V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0));
719 crwr->sec_cpl.cipherstop_lo_authinsert = htobe32(
720 V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0));
721
722 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
723 crwr->sec_cpl.seqno_numivs = htobe32(
724 V_SCMD_SEQ_NO_CTRL(0) |
725 V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) |
726 V_SCMD_ENC_DEC_CTRL(op_type) |
727 V_SCMD_CIPH_MODE(s->cipher.cipher_mode) |
728 V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_NOP) |
729 V_SCMD_HMAC_CTRL(SCMD_HMAC_CTRL_NOP) |
730 V_SCMD_IV_SIZE(iv_len / 2) |
731 V_SCMD_NUM_IVS(0));
732 crwr->sec_cpl.ivgen_hdrlen = htobe32(
733 V_SCMD_IV_GEN_CTRL(0) |
734 V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) |
735 V_SCMD_AADIVDROP(1) | V_SCMD_HDR_LEN(dsgl_len));
736
737 crwr->key_ctx.ctx_hdr = s->cipher.key_ctx_hdr;
738 switch (s->cipher.cipher_mode) {
739 case SCMD_CIPH_MODE_AES_CBC:
740 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
741 memcpy(crwr->key_ctx.key, s->cipher.enckey,
742 s->cipher.key_len);
743 else
744 memcpy(crwr->key_ctx.key, s->cipher.deckey,
745 s->cipher.key_len);
746 break;
747 case SCMD_CIPH_MODE_AES_CTR:
748 memcpy(crwr->key_ctx.key, s->cipher.enckey,
749 s->cipher.key_len);
750 break;
751 case SCMD_CIPH_MODE_AES_XTS:
752 key_half = s->cipher.key_len / 2;
753 memcpy(crwr->key_ctx.key, s->cipher.enckey + key_half,
754 key_half);
755 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
756 memcpy(crwr->key_ctx.key + key_half,
757 s->cipher.enckey, key_half);
758 else
759 memcpy(crwr->key_ctx.key + key_half,
760 s->cipher.deckey, key_half);
761 break;
762 }
763
764 dst = (char *)(crwr + 1) + kctx_len;
765 ccr_write_phys_dsgl(s, dst, dsgl_nsegs);
766 dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
767 memcpy(dst, iv, iv_len);
768 dst += iv_len;
769 if (imm_len != 0)
770 crypto_copydata(crp, crp->crp_payload_start,
771 crp->crp_payload_length, dst);
772 else
773 ccr_write_ulptx_sgl(s, dst, sgl_nsegs);
774
775 /* XXX: TODO backpressure */
776 t4_wrq_tx(sc->adapter, wr);
777
778 explicit_bzero(iv, sizeof(iv));
779 return (0);
780 }
781
782 static int
ccr_cipher_done(struct ccr_softc * sc,struct ccr_session * s,struct cryptop * crp,const struct cpl_fw6_pld * cpl,int error)783 ccr_cipher_done(struct ccr_softc *sc, struct ccr_session *s,
784 struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error)
785 {
786
787 /*
788 * The updated IV to permit chained requests is at
789 * cpl->data[2], but OCF doesn't permit chained requests.
790 */
791 return (error);
792 }
793
794 /*
795 * 'hashsize' is the length of a full digest. 'authsize' is the
796 * requested digest length for this operation which may be less
797 * than 'hashsize'.
798 */
799 static int
ccr_hmac_ctrl(unsigned int hashsize,unsigned int authsize)800 ccr_hmac_ctrl(unsigned int hashsize, unsigned int authsize)
801 {
802
803 if (authsize == 10)
804 return (SCMD_HMAC_CTRL_TRUNC_RFC4366);
805 if (authsize == 12)
806 return (SCMD_HMAC_CTRL_IPSEC_96BIT);
807 if (authsize == hashsize / 2)
808 return (SCMD_HMAC_CTRL_DIV2);
809 return (SCMD_HMAC_CTRL_NO_TRUNC);
810 }
811
812 static int
ccr_eta(struct ccr_softc * sc,struct ccr_session * s,struct cryptop * crp)813 ccr_eta(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
814 {
815 char iv[CHCR_MAX_CRYPTO_IV_LEN];
816 struct chcr_wr *crwr;
817 struct wrqe *wr;
818 const struct auth_hash *axf;
819 char *dst;
820 u_int kctx_len, key_half, op_type, transhdr_len, wr_len;
821 u_int hash_size_in_response, imm_len, iopad_size, iv_len;
822 u_int aad_start, aad_stop;
823 u_int auth_insert;
824 u_int cipher_start, cipher_stop;
825 u_int hmac_ctrl, input_len;
826 int dsgl_nsegs, dsgl_len;
827 int sgl_nsegs, sgl_len;
828 int error;
829
830 /*
831 * If there is a need in the future, requests with an empty
832 * payload could be supported as HMAC-only requests.
833 */
834 if (s->cipher.key_len == 0 || crp->crp_payload_length == 0)
835 return (EINVAL);
836 if (s->cipher.cipher_mode == SCMD_CIPH_MODE_AES_CBC &&
837 (crp->crp_payload_length % AES_BLOCK_LEN) != 0)
838 return (EINVAL);
839
840 /* For AES-XTS we send a 16-byte IV in the work request. */
841 if (s->cipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS)
842 iv_len = AES_BLOCK_LEN;
843 else
844 iv_len = s->cipher.iv_len;
845
846 if (crp->crp_aad_length + iv_len > MAX_AAD_LEN)
847 return (EINVAL);
848
849 axf = s->hmac.auth_hash;
850 hash_size_in_response = s->hmac.hash_len;
851 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
852 op_type = CHCR_ENCRYPT_OP;
853 else
854 op_type = CHCR_DECRYPT_OP;
855
856 /*
857 * The output buffer consists of the cipher text followed by
858 * the hash when encrypting. For decryption it only contains
859 * the plain text.
860 *
861 * Due to a firmware bug, the output buffer must include a
862 * dummy output buffer for the IV and AAD prior to the real
863 * output buffer.
864 */
865 if (op_type == CHCR_ENCRYPT_OP) {
866 if (iv_len + crp->crp_aad_length + crp->crp_payload_length +
867 hash_size_in_response > MAX_REQUEST_SIZE)
868 return (EFBIG);
869 } else {
870 if (iv_len + crp->crp_aad_length + crp->crp_payload_length >
871 MAX_REQUEST_SIZE)
872 return (EFBIG);
873 }
874 sglist_reset(s->sg_dsgl);
875 error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0,
876 iv_len + crp->crp_aad_length);
877 if (error)
878 return (error);
879 if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
880 error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
881 crp->crp_payload_output_start, crp->crp_payload_length);
882 else
883 error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
884 crp->crp_payload_start, crp->crp_payload_length);
885 if (error)
886 return (error);
887 if (op_type == CHCR_ENCRYPT_OP) {
888 if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
889 error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
890 crp->crp_digest_start, hash_size_in_response);
891 else
892 error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
893 crp->crp_digest_start, hash_size_in_response);
894 if (error)
895 return (error);
896 }
897 dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN);
898 if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
899 return (EFBIG);
900 dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
901
902 /* PADs must be 128-bit aligned. */
903 iopad_size = roundup2(s->hmac.partial_digest_len, 16);
904
905 /*
906 * The 'key' part of the key context consists of the key followed
907 * by the IPAD and OPAD.
908 */
909 kctx_len = roundup2(s->cipher.key_len, 16) + iopad_size * 2;
910 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len);
911
912 /*
913 * The input buffer consists of the IV, any AAD, and then the
914 * cipher/plain text. For decryption requests the hash is
915 * appended after the cipher text.
916 *
917 * The IV is always stored at the start of the input buffer
918 * even though it may be duplicated in the payload. The
919 * crypto engine doesn't work properly if the IV offset points
920 * inside of the AAD region, so a second copy is always
921 * required.
922 */
923 input_len = crp->crp_aad_length + crp->crp_payload_length;
924
925 /*
926 * The firmware hangs if sent a request which is a
927 * bit smaller than MAX_REQUEST_SIZE. In particular, the
928 * firmware appears to require 512 - 16 bytes of spare room
929 * along with the size of the hash even if the hash isn't
930 * included in the input buffer.
931 */
932 if (input_len + roundup2(axf->hashsize, 16) + (512 - 16) >
933 MAX_REQUEST_SIZE)
934 return (EFBIG);
935 if (op_type == CHCR_DECRYPT_OP)
936 input_len += hash_size_in_response;
937
938 if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) {
939 imm_len = input_len;
940 sgl_nsegs = 0;
941 sgl_len = 0;
942 } else {
943 imm_len = 0;
944 sglist_reset(s->sg_ulptx);
945 if (crp->crp_aad_length != 0) {
946 if (crp->crp_aad != NULL)
947 error = sglist_append(s->sg_ulptx,
948 crp->crp_aad, crp->crp_aad_length);
949 else
950 error = sglist_append_sglist(s->sg_ulptx,
951 s->sg_input, crp->crp_aad_start,
952 crp->crp_aad_length);
953 if (error)
954 return (error);
955 }
956 error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
957 crp->crp_payload_start, crp->crp_payload_length);
958 if (error)
959 return (error);
960 if (op_type == CHCR_DECRYPT_OP) {
961 error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
962 crp->crp_digest_start, hash_size_in_response);
963 if (error)
964 return (error);
965 }
966 sgl_nsegs = s->sg_ulptx->sg_nseg;
967 sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
968 }
969
970 /* Any AAD comes after the IV. */
971 if (crp->crp_aad_length != 0) {
972 aad_start = iv_len + 1;
973 aad_stop = aad_start + crp->crp_aad_length - 1;
974 } else {
975 aad_start = 0;
976 aad_stop = 0;
977 }
978 cipher_start = iv_len + crp->crp_aad_length + 1;
979 if (op_type == CHCR_DECRYPT_OP)
980 cipher_stop = hash_size_in_response;
981 else
982 cipher_stop = 0;
983 if (op_type == CHCR_DECRYPT_OP)
984 auth_insert = hash_size_in_response;
985 else
986 auth_insert = 0;
987
988 wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) +
989 sgl_len;
990 if (wr_len > SGE_MAX_WR_LEN)
991 return (EFBIG);
992 wr = alloc_wrqe(wr_len, s->port->txq);
993 if (wr == NULL) {
994 counter_u64_add(sc->stats_wr_nomem, 1);
995 return (ENOMEM);
996 }
997 crwr = wrtod(wr);
998 memset(crwr, 0, wr_len);
999
1000 crypto_read_iv(crp, iv);
1001
1002 /* Zero the remainder of the IV for AES-XTS. */
1003 memset(iv + s->cipher.iv_len, 0, iv_len - s->cipher.iv_len);
1004
1005 ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len,
1006 op_type == CHCR_DECRYPT_OP ? hash_size_in_response : 0, crp);
1007
1008 crwr->sec_cpl.op_ivinsrtofst = htobe32(
1009 V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) |
1010 V_CPL_TX_SEC_PDU_RXCHID(s->port->rx_channel_id) |
1011 V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) |
1012 V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
1013 V_CPL_TX_SEC_PDU_IVINSRTOFST(1));
1014
1015 crwr->sec_cpl.pldlen = htobe32(iv_len + input_len);
1016
1017 crwr->sec_cpl.aadstart_cipherstop_hi = htobe32(
1018 V_CPL_TX_SEC_PDU_AADSTART(aad_start) |
1019 V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) |
1020 V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) |
1021 V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(cipher_stop >> 4));
1022 crwr->sec_cpl.cipherstop_lo_authinsert = htobe32(
1023 V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(cipher_stop & 0xf) |
1024 V_CPL_TX_SEC_PDU_AUTHSTART(cipher_start) |
1025 V_CPL_TX_SEC_PDU_AUTHSTOP(cipher_stop) |
1026 V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert));
1027
1028 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1029 hmac_ctrl = ccr_hmac_ctrl(axf->hashsize, hash_size_in_response);
1030 crwr->sec_cpl.seqno_numivs = htobe32(
1031 V_SCMD_SEQ_NO_CTRL(0) |
1032 V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) |
1033 V_SCMD_ENC_DEC_CTRL(op_type) |
1034 V_SCMD_CIPH_AUTH_SEQ_CTRL(op_type == CHCR_ENCRYPT_OP ? 1 : 0) |
1035 V_SCMD_CIPH_MODE(s->cipher.cipher_mode) |
1036 V_SCMD_AUTH_MODE(s->hmac.auth_mode) |
1037 V_SCMD_HMAC_CTRL(hmac_ctrl) |
1038 V_SCMD_IV_SIZE(iv_len / 2) |
1039 V_SCMD_NUM_IVS(0));
1040 crwr->sec_cpl.ivgen_hdrlen = htobe32(
1041 V_SCMD_IV_GEN_CTRL(0) |
1042 V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) |
1043 V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len));
1044
1045 crwr->key_ctx.ctx_hdr = s->cipher.key_ctx_hdr;
1046 switch (s->cipher.cipher_mode) {
1047 case SCMD_CIPH_MODE_AES_CBC:
1048 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
1049 memcpy(crwr->key_ctx.key, s->cipher.enckey,
1050 s->cipher.key_len);
1051 else
1052 memcpy(crwr->key_ctx.key, s->cipher.deckey,
1053 s->cipher.key_len);
1054 break;
1055 case SCMD_CIPH_MODE_AES_CTR:
1056 memcpy(crwr->key_ctx.key, s->cipher.enckey,
1057 s->cipher.key_len);
1058 break;
1059 case SCMD_CIPH_MODE_AES_XTS:
1060 key_half = s->cipher.key_len / 2;
1061 memcpy(crwr->key_ctx.key, s->cipher.enckey + key_half,
1062 key_half);
1063 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
1064 memcpy(crwr->key_ctx.key + key_half,
1065 s->cipher.enckey, key_half);
1066 else
1067 memcpy(crwr->key_ctx.key + key_half,
1068 s->cipher.deckey, key_half);
1069 break;
1070 }
1071
1072 dst = crwr->key_ctx.key + roundup2(s->cipher.key_len, 16);
1073 memcpy(dst, s->hmac.pads, iopad_size * 2);
1074
1075 dst = (char *)(crwr + 1) + kctx_len;
1076 ccr_write_phys_dsgl(s, dst, dsgl_nsegs);
1077 dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
1078 memcpy(dst, iv, iv_len);
1079 dst += iv_len;
1080 if (imm_len != 0) {
1081 if (crp->crp_aad_length != 0) {
1082 if (crp->crp_aad != NULL)
1083 memcpy(dst, crp->crp_aad, crp->crp_aad_length);
1084 else
1085 crypto_copydata(crp, crp->crp_aad_start,
1086 crp->crp_aad_length, dst);
1087 dst += crp->crp_aad_length;
1088 }
1089 crypto_copydata(crp, crp->crp_payload_start,
1090 crp->crp_payload_length, dst);
1091 dst += crp->crp_payload_length;
1092 if (op_type == CHCR_DECRYPT_OP)
1093 crypto_copydata(crp, crp->crp_digest_start,
1094 hash_size_in_response, dst);
1095 } else
1096 ccr_write_ulptx_sgl(s, dst, sgl_nsegs);
1097
1098 /* XXX: TODO backpressure */
1099 t4_wrq_tx(sc->adapter, wr);
1100
1101 explicit_bzero(iv, sizeof(iv));
1102 return (0);
1103 }
1104
1105 static int
ccr_eta_done(struct ccr_softc * sc,struct ccr_session * s,struct cryptop * crp,const struct cpl_fw6_pld * cpl,int error)1106 ccr_eta_done(struct ccr_softc *sc, struct ccr_session *s,
1107 struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error)
1108 {
1109
1110 /*
1111 * The updated IV to permit chained requests is at
1112 * cpl->data[2], but OCF doesn't permit chained requests.
1113 */
1114 return (error);
1115 }
1116
1117 static int
ccr_gcm(struct ccr_softc * sc,struct ccr_session * s,struct cryptop * crp)1118 ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
1119 {
1120 char iv[CHCR_MAX_CRYPTO_IV_LEN];
1121 struct chcr_wr *crwr;
1122 struct wrqe *wr;
1123 char *dst;
1124 u_int iv_len, kctx_len, op_type, transhdr_len, wr_len;
1125 u_int hash_size_in_response, imm_len;
1126 u_int aad_start, aad_stop, cipher_start, cipher_stop, auth_insert;
1127 u_int hmac_ctrl, input_len;
1128 int dsgl_nsegs, dsgl_len;
1129 int sgl_nsegs, sgl_len;
1130 int error;
1131
1132 if (s->cipher.key_len == 0)
1133 return (EINVAL);
1134
1135 /*
1136 * The crypto engine doesn't handle GCM requests with an empty
1137 * payload, so handle those in software instead.
1138 */
1139 if (crp->crp_payload_length == 0)
1140 return (EMSGSIZE);
1141
1142 if (crp->crp_aad_length + AES_BLOCK_LEN > MAX_AAD_LEN)
1143 return (EMSGSIZE);
1144
1145 hash_size_in_response = s->gmac.hash_len;
1146 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
1147 op_type = CHCR_ENCRYPT_OP;
1148 else
1149 op_type = CHCR_DECRYPT_OP;
1150
1151 iv_len = AES_BLOCK_LEN;
1152
1153 /*
1154 * GCM requests should always provide an explicit IV.
1155 */
1156 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
1157 return (EINVAL);
1158
1159 /*
1160 * The output buffer consists of the cipher text followed by
1161 * the tag when encrypting. For decryption it only contains
1162 * the plain text.
1163 *
1164 * Due to a firmware bug, the output buffer must include a
1165 * dummy output buffer for the IV and AAD prior to the real
1166 * output buffer.
1167 */
1168 if (op_type == CHCR_ENCRYPT_OP) {
1169 if (iv_len + crp->crp_aad_length + crp->crp_payload_length +
1170 hash_size_in_response > MAX_REQUEST_SIZE)
1171 return (EFBIG);
1172 } else {
1173 if (iv_len + crp->crp_aad_length + crp->crp_payload_length >
1174 MAX_REQUEST_SIZE)
1175 return (EFBIG);
1176 }
1177 sglist_reset(s->sg_dsgl);
1178 error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0, iv_len +
1179 crp->crp_aad_length);
1180 if (error)
1181 return (error);
1182 if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
1183 error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
1184 crp->crp_payload_output_start, crp->crp_payload_length);
1185 else
1186 error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
1187 crp->crp_payload_start, crp->crp_payload_length);
1188 if (error)
1189 return (error);
1190 if (op_type == CHCR_ENCRYPT_OP) {
1191 if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
1192 error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
1193 crp->crp_digest_start, hash_size_in_response);
1194 else
1195 error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
1196 crp->crp_digest_start, hash_size_in_response);
1197 if (error)
1198 return (error);
1199 }
1200 dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN);
1201 if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
1202 return (EFBIG);
1203 dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
1204
1205 /*
1206 * The 'key' part of the key context consists of the key followed
1207 * by the Galois hash key.
1208 */
1209 kctx_len = roundup2(s->cipher.key_len, 16) + GMAC_BLOCK_LEN;
1210 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len);
1211
1212 /*
1213 * The input buffer consists of the IV, any AAD, and then the
1214 * cipher/plain text. For decryption requests the hash is
1215 * appended after the cipher text.
1216 *
1217 * The IV is always stored at the start of the input buffer
1218 * even though it may be duplicated in the payload. The
1219 * crypto engine doesn't work properly if the IV offset points
1220 * inside of the AAD region, so a second copy is always
1221 * required.
1222 */
1223 input_len = crp->crp_aad_length + crp->crp_payload_length;
1224 if (op_type == CHCR_DECRYPT_OP)
1225 input_len += hash_size_in_response;
1226 if (input_len > MAX_REQUEST_SIZE)
1227 return (EFBIG);
1228 if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) {
1229 imm_len = input_len;
1230 sgl_nsegs = 0;
1231 sgl_len = 0;
1232 } else {
1233 imm_len = 0;
1234 sglist_reset(s->sg_ulptx);
1235 if (crp->crp_aad_length != 0) {
1236 if (crp->crp_aad != NULL)
1237 error = sglist_append(s->sg_ulptx,
1238 crp->crp_aad, crp->crp_aad_length);
1239 else
1240 error = sglist_append_sglist(s->sg_ulptx,
1241 s->sg_input, crp->crp_aad_start,
1242 crp->crp_aad_length);
1243 if (error)
1244 return (error);
1245 }
1246 error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
1247 crp->crp_payload_start, crp->crp_payload_length);
1248 if (error)
1249 return (error);
1250 if (op_type == CHCR_DECRYPT_OP) {
1251 error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
1252 crp->crp_digest_start, hash_size_in_response);
1253 if (error)
1254 return (error);
1255 }
1256 sgl_nsegs = s->sg_ulptx->sg_nseg;
1257 sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
1258 }
1259
1260 if (crp->crp_aad_length != 0) {
1261 aad_start = iv_len + 1;
1262 aad_stop = aad_start + crp->crp_aad_length - 1;
1263 } else {
1264 aad_start = 0;
1265 aad_stop = 0;
1266 }
1267 cipher_start = iv_len + crp->crp_aad_length + 1;
1268 if (op_type == CHCR_DECRYPT_OP)
1269 cipher_stop = hash_size_in_response;
1270 else
1271 cipher_stop = 0;
1272 if (op_type == CHCR_DECRYPT_OP)
1273 auth_insert = hash_size_in_response;
1274 else
1275 auth_insert = 0;
1276
1277 wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) +
1278 sgl_len;
1279 if (wr_len > SGE_MAX_WR_LEN)
1280 return (EFBIG);
1281 wr = alloc_wrqe(wr_len, s->port->txq);
1282 if (wr == NULL) {
1283 counter_u64_add(sc->stats_wr_nomem, 1);
1284 return (ENOMEM);
1285 }
1286 crwr = wrtod(wr);
1287 memset(crwr, 0, wr_len);
1288
1289 crypto_read_iv(crp, iv);
1290 *(uint32_t *)&iv[12] = htobe32(1);
1291
1292 ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len, 0,
1293 crp);
1294
1295 crwr->sec_cpl.op_ivinsrtofst = htobe32(
1296 V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) |
1297 V_CPL_TX_SEC_PDU_RXCHID(s->port->rx_channel_id) |
1298 V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) |
1299 V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
1300 V_CPL_TX_SEC_PDU_IVINSRTOFST(1));
1301
1302 crwr->sec_cpl.pldlen = htobe32(iv_len + input_len);
1303
1304 /*
1305 * NB: cipherstop is explicitly set to 0. On encrypt it
1306 * should normally be set to 0 anyway. However, for decrypt
1307 * the cipher ends before the tag in the ETA case (and
1308 * authstop is set to stop before the tag), but for GCM the
1309 * cipher still runs to the end of the buffer. Not sure if
1310 * this is intentional or a firmware quirk, but it is required
1311 * for working tag validation with GCM decryption.
1312 */
1313 crwr->sec_cpl.aadstart_cipherstop_hi = htobe32(
1314 V_CPL_TX_SEC_PDU_AADSTART(aad_start) |
1315 V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) |
1316 V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) |
1317 V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0));
1318 crwr->sec_cpl.cipherstop_lo_authinsert = htobe32(
1319 V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0) |
1320 V_CPL_TX_SEC_PDU_AUTHSTART(cipher_start) |
1321 V_CPL_TX_SEC_PDU_AUTHSTOP(cipher_stop) |
1322 V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert));
1323
1324 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1325 hmac_ctrl = ccr_hmac_ctrl(AES_GMAC_HASH_LEN, hash_size_in_response);
1326 crwr->sec_cpl.seqno_numivs = htobe32(
1327 V_SCMD_SEQ_NO_CTRL(0) |
1328 V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) |
1329 V_SCMD_ENC_DEC_CTRL(op_type) |
1330 V_SCMD_CIPH_AUTH_SEQ_CTRL(op_type == CHCR_ENCRYPT_OP ? 1 : 0) |
1331 V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_AES_GCM) |
1332 V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_GHASH) |
1333 V_SCMD_HMAC_CTRL(hmac_ctrl) |
1334 V_SCMD_IV_SIZE(iv_len / 2) |
1335 V_SCMD_NUM_IVS(0));
1336 crwr->sec_cpl.ivgen_hdrlen = htobe32(
1337 V_SCMD_IV_GEN_CTRL(0) |
1338 V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) |
1339 V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len));
1340
1341 crwr->key_ctx.ctx_hdr = s->cipher.key_ctx_hdr;
1342 memcpy(crwr->key_ctx.key, s->cipher.enckey, s->cipher.key_len);
1343 dst = crwr->key_ctx.key + roundup2(s->cipher.key_len, 16);
1344 memcpy(dst, s->gmac.ghash_h, GMAC_BLOCK_LEN);
1345
1346 dst = (char *)(crwr + 1) + kctx_len;
1347 ccr_write_phys_dsgl(s, dst, dsgl_nsegs);
1348 dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
1349 memcpy(dst, iv, iv_len);
1350 dst += iv_len;
1351 if (imm_len != 0) {
1352 if (crp->crp_aad_length != 0) {
1353 if (crp->crp_aad != NULL)
1354 memcpy(dst, crp->crp_aad, crp->crp_aad_length);
1355 else
1356 crypto_copydata(crp, crp->crp_aad_start,
1357 crp->crp_aad_length, dst);
1358 dst += crp->crp_aad_length;
1359 }
1360 crypto_copydata(crp, crp->crp_payload_start,
1361 crp->crp_payload_length, dst);
1362 dst += crp->crp_payload_length;
1363 if (op_type == CHCR_DECRYPT_OP)
1364 crypto_copydata(crp, crp->crp_digest_start,
1365 hash_size_in_response, dst);
1366 } else
1367 ccr_write_ulptx_sgl(s, dst, sgl_nsegs);
1368
1369 /* XXX: TODO backpressure */
1370 t4_wrq_tx(sc->adapter, wr);
1371
1372 explicit_bzero(iv, sizeof(iv));
1373 return (0);
1374 }
1375
1376 static int
ccr_gcm_done(struct ccr_softc * sc,struct ccr_session * s,struct cryptop * crp,const struct cpl_fw6_pld * cpl,int error)1377 ccr_gcm_done(struct ccr_softc *sc, struct ccr_session *s,
1378 struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error)
1379 {
1380
1381 /*
1382 * The updated IV to permit chained requests is at
1383 * cpl->data[2], but OCF doesn't permit chained requests.
1384 *
1385 * Note that the hardware should always verify the GMAC hash.
1386 */
1387 return (error);
1388 }
1389
1390 static int
ccr_ccm_hmac_ctrl(unsigned int authsize)1391 ccr_ccm_hmac_ctrl(unsigned int authsize)
1392 {
1393 switch (authsize) {
1394 case 4:
1395 return (SCMD_HMAC_CTRL_PL1);
1396 case 6:
1397 return (SCMD_HMAC_CTRL_PL2);
1398 case 8:
1399 return (SCMD_HMAC_CTRL_DIV2);
1400 case 10:
1401 return (SCMD_HMAC_CTRL_TRUNC_RFC4366);
1402 case 12:
1403 return (SCMD_HMAC_CTRL_IPSEC_96BIT);
1404 case 14:
1405 return (SCMD_HMAC_CTRL_PL3);
1406 case 16:
1407 return (SCMD_HMAC_CTRL_NO_TRUNC);
1408 default:
1409 __assert_unreachable();
1410 }
1411 }
1412
1413 static void
generate_ccm_b0(struct cryptop * crp,u_int hash_size_in_response,const char * iv,char * b0)1414 generate_ccm_b0(struct cryptop *crp, u_int hash_size_in_response,
1415 const char *iv, char *b0)
1416 {
1417 u_int i, payload_len, L;
1418
1419 /* NB: L is already set in the first byte of the IV. */
1420 memcpy(b0, iv, CCM_B0_SIZE);
1421 L = iv[0] + 1;
1422
1423 /* Set length of hash in bits 3 - 5. */
1424 b0[0] |= (((hash_size_in_response - 2) / 2) << 3);
1425
1426 /* Store the payload length as a big-endian value. */
1427 payload_len = crp->crp_payload_length;
1428 for (i = 0; i < L; i++) {
1429 b0[CCM_CBC_BLOCK_LEN - 1 - i] = payload_len;
1430 payload_len >>= 8;
1431 }
1432
1433 /*
1434 * If there is AAD in the request, set bit 6 in the flags
1435 * field and store the AAD length as a big-endian value at the
1436 * start of block 1. This only assumes a 16-bit AAD length
1437 * since T6 doesn't support large AAD sizes.
1438 */
1439 if (crp->crp_aad_length != 0) {
1440 b0[0] |= (1 << 6);
1441 *(uint16_t *)(b0 + CCM_B0_SIZE) = htobe16(crp->crp_aad_length);
1442 }
1443 }
1444
1445 static int
ccr_ccm(struct ccr_softc * sc,struct ccr_session * s,struct cryptop * crp)1446 ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
1447 {
1448 char iv[CHCR_MAX_CRYPTO_IV_LEN];
1449 const struct crypto_session_params *csp;
1450 struct ulptx_idata *idata;
1451 struct chcr_wr *crwr;
1452 struct wrqe *wr;
1453 char *dst;
1454 u_int iv_len, kctx_len, op_type, transhdr_len, wr_len;
1455 u_int aad_len, b0_len, hash_size_in_response, imm_len;
1456 u_int aad_start, aad_stop, cipher_start, cipher_stop, auth_insert;
1457 u_int hmac_ctrl, input_len;
1458 int dsgl_nsegs, dsgl_len;
1459 int sgl_nsegs, sgl_len;
1460 int error;
1461
1462 csp = crypto_get_params(crp->crp_session);
1463
1464 if (s->cipher.key_len == 0)
1465 return (EINVAL);
1466
1467 /*
1468 * The crypto engine doesn't handle CCM requests with an empty
1469 * payload, so handle those in software instead.
1470 */
1471 if (crp->crp_payload_length == 0)
1472 return (EMSGSIZE);
1473
1474 /* The length has to fit within the length field in block 0. */
1475 if (crp->crp_payload_length > ccm_max_payload_length(csp))
1476 return (EMSGSIZE);
1477
1478 /*
1479 * CCM always includes block 0 in the AAD before AAD from the
1480 * request.
1481 */
1482 b0_len = CCM_B0_SIZE;
1483 if (crp->crp_aad_length != 0)
1484 b0_len += CCM_AAD_FIELD_SIZE;
1485 aad_len = b0_len + crp->crp_aad_length;
1486
1487 /*
1488 * CCM requests should always provide an explicit IV (really
1489 * the nonce).
1490 */
1491 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
1492 return (EINVAL);
1493
1494 /*
1495 * The IV in the work request is 16 bytes and not just the
1496 * nonce.
1497 */
1498 iv_len = AES_BLOCK_LEN;
1499
1500 if (iv_len + aad_len > MAX_AAD_LEN)
1501 return (EMSGSIZE);
1502
1503 hash_size_in_response = s->ccm_mac.hash_len;
1504 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
1505 op_type = CHCR_ENCRYPT_OP;
1506 else
1507 op_type = CHCR_DECRYPT_OP;
1508
1509 /*
1510 * The output buffer consists of the cipher text followed by
1511 * the tag when encrypting. For decryption it only contains
1512 * the plain text.
1513 *
1514 * Due to a firmware bug, the output buffer must include a
1515 * dummy output buffer for the IV and AAD prior to the real
1516 * output buffer.
1517 */
1518 if (op_type == CHCR_ENCRYPT_OP) {
1519 if (iv_len + aad_len + crp->crp_payload_length +
1520 hash_size_in_response > MAX_REQUEST_SIZE)
1521 return (EFBIG);
1522 } else {
1523 if (iv_len + aad_len + crp->crp_payload_length >
1524 MAX_REQUEST_SIZE)
1525 return (EFBIG);
1526 }
1527 sglist_reset(s->sg_dsgl);
1528 error = sglist_append_sglist(s->sg_dsgl, sc->sg_iv_aad, 0, iv_len +
1529 aad_len);
1530 if (error)
1531 return (error);
1532 if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
1533 error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
1534 crp->crp_payload_output_start, crp->crp_payload_length);
1535 else
1536 error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
1537 crp->crp_payload_start, crp->crp_payload_length);
1538 if (error)
1539 return (error);
1540 if (op_type == CHCR_ENCRYPT_OP) {
1541 if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
1542 error = sglist_append_sglist(s->sg_dsgl, s->sg_output,
1543 crp->crp_digest_start, hash_size_in_response);
1544 else
1545 error = sglist_append_sglist(s->sg_dsgl, s->sg_input,
1546 crp->crp_digest_start, hash_size_in_response);
1547 if (error)
1548 return (error);
1549 }
1550 dsgl_nsegs = ccr_count_sgl(s->sg_dsgl, DSGL_SGE_MAXLEN);
1551 if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
1552 return (EFBIG);
1553 dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
1554
1555 /*
1556 * The 'key' part of the key context consists of two copies of
1557 * the AES key.
1558 */
1559 kctx_len = roundup2(s->cipher.key_len, 16) * 2;
1560 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len);
1561
1562 /*
1563 * The input buffer consists of the IV, AAD (including block
1564 * 0), and then the cipher/plain text. For decryption
1565 * requests the hash is appended after the cipher text.
1566 *
1567 * The IV is always stored at the start of the input buffer
1568 * even though it may be duplicated in the payload. The
1569 * crypto engine doesn't work properly if the IV offset points
1570 * inside of the AAD region, so a second copy is always
1571 * required.
1572 */
1573 input_len = aad_len + crp->crp_payload_length;
1574 if (op_type == CHCR_DECRYPT_OP)
1575 input_len += hash_size_in_response;
1576 if (input_len > MAX_REQUEST_SIZE)
1577 return (EFBIG);
1578 if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) {
1579 imm_len = input_len;
1580 sgl_nsegs = 0;
1581 sgl_len = 0;
1582 } else {
1583 /* Block 0 is passed as immediate data. */
1584 imm_len = b0_len;
1585
1586 sglist_reset(s->sg_ulptx);
1587 if (crp->crp_aad_length != 0) {
1588 if (crp->crp_aad != NULL)
1589 error = sglist_append(s->sg_ulptx,
1590 crp->crp_aad, crp->crp_aad_length);
1591 else
1592 error = sglist_append_sglist(s->sg_ulptx,
1593 s->sg_input, crp->crp_aad_start,
1594 crp->crp_aad_length);
1595 if (error)
1596 return (error);
1597 }
1598 error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
1599 crp->crp_payload_start, crp->crp_payload_length);
1600 if (error)
1601 return (error);
1602 if (op_type == CHCR_DECRYPT_OP) {
1603 error = sglist_append_sglist(s->sg_ulptx, s->sg_input,
1604 crp->crp_digest_start, hash_size_in_response);
1605 if (error)
1606 return (error);
1607 }
1608 sgl_nsegs = s->sg_ulptx->sg_nseg;
1609 sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
1610 }
1611
1612 aad_start = iv_len + 1;
1613 aad_stop = aad_start + aad_len - 1;
1614 cipher_start = aad_stop + 1;
1615 if (op_type == CHCR_DECRYPT_OP)
1616 cipher_stop = hash_size_in_response;
1617 else
1618 cipher_stop = 0;
1619 if (op_type == CHCR_DECRYPT_OP)
1620 auth_insert = hash_size_in_response;
1621 else
1622 auth_insert = 0;
1623
1624 wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) +
1625 sgl_len;
1626 if (wr_len > SGE_MAX_WR_LEN)
1627 return (EFBIG);
1628 wr = alloc_wrqe(wr_len, s->port->txq);
1629 if (wr == NULL) {
1630 counter_u64_add(sc->stats_wr_nomem, 1);
1631 return (ENOMEM);
1632 }
1633 crwr = wrtod(wr);
1634 memset(crwr, 0, wr_len);
1635
1636 /*
1637 * Read the nonce from the request. Use the nonce to generate
1638 * the full IV with the counter set to 0.
1639 */
1640 memset(iv, 0, iv_len);
1641 iv[0] = (15 - csp->csp_ivlen) - 1;
1642 crypto_read_iv(crp, iv + 1);
1643
1644 ccr_populate_wreq(sc, s, crwr, kctx_len, wr_len, imm_len, sgl_len, 0,
1645 crp);
1646
1647 crwr->sec_cpl.op_ivinsrtofst = htobe32(
1648 V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) |
1649 V_CPL_TX_SEC_PDU_RXCHID(s->port->rx_channel_id) |
1650 V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) |
1651 V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
1652 V_CPL_TX_SEC_PDU_IVINSRTOFST(1));
1653
1654 crwr->sec_cpl.pldlen = htobe32(iv_len + input_len);
1655
1656 /*
1657 * NB: cipherstop is explicitly set to 0. See comments above
1658 * in ccr_gcm().
1659 */
1660 crwr->sec_cpl.aadstart_cipherstop_hi = htobe32(
1661 V_CPL_TX_SEC_PDU_AADSTART(aad_start) |
1662 V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) |
1663 V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) |
1664 V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0));
1665 crwr->sec_cpl.cipherstop_lo_authinsert = htobe32(
1666 V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0) |
1667 V_CPL_TX_SEC_PDU_AUTHSTART(cipher_start) |
1668 V_CPL_TX_SEC_PDU_AUTHSTOP(cipher_stop) |
1669 V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert));
1670
1671 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1672 hmac_ctrl = ccr_ccm_hmac_ctrl(hash_size_in_response);
1673 crwr->sec_cpl.seqno_numivs = htobe32(
1674 V_SCMD_SEQ_NO_CTRL(0) |
1675 V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) |
1676 V_SCMD_ENC_DEC_CTRL(op_type) |
1677 V_SCMD_CIPH_AUTH_SEQ_CTRL(op_type == CHCR_ENCRYPT_OP ? 0 : 1) |
1678 V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_AES_CCM) |
1679 V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_CBCMAC) |
1680 V_SCMD_HMAC_CTRL(hmac_ctrl) |
1681 V_SCMD_IV_SIZE(iv_len / 2) |
1682 V_SCMD_NUM_IVS(0));
1683 crwr->sec_cpl.ivgen_hdrlen = htobe32(
1684 V_SCMD_IV_GEN_CTRL(0) |
1685 V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) |
1686 V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len));
1687
1688 crwr->key_ctx.ctx_hdr = s->cipher.key_ctx_hdr;
1689 memcpy(crwr->key_ctx.key, s->cipher.enckey, s->cipher.key_len);
1690 memcpy(crwr->key_ctx.key + roundup(s->cipher.key_len, 16),
1691 s->cipher.enckey, s->cipher.key_len);
1692
1693 dst = (char *)(crwr + 1) + kctx_len;
1694 ccr_write_phys_dsgl(s, dst, dsgl_nsegs);
1695 dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
1696 memcpy(dst, iv, iv_len);
1697 dst += iv_len;
1698 generate_ccm_b0(crp, hash_size_in_response, iv, dst);
1699 if (sgl_nsegs == 0) {
1700 dst += b0_len;
1701 if (crp->crp_aad_length != 0) {
1702 if (crp->crp_aad != NULL)
1703 memcpy(dst, crp->crp_aad, crp->crp_aad_length);
1704 else
1705 crypto_copydata(crp, crp->crp_aad_start,
1706 crp->crp_aad_length, dst);
1707 dst += crp->crp_aad_length;
1708 }
1709 crypto_copydata(crp, crp->crp_payload_start,
1710 crp->crp_payload_length, dst);
1711 dst += crp->crp_payload_length;
1712 if (op_type == CHCR_DECRYPT_OP)
1713 crypto_copydata(crp, crp->crp_digest_start,
1714 hash_size_in_response, dst);
1715 } else {
1716 dst += CCM_B0_SIZE;
1717 if (b0_len > CCM_B0_SIZE) {
1718 /*
1719 * If there is AAD, insert padding including a
1720 * ULP_TX_SC_NOOP so that the ULP_TX_SC_DSGL
1721 * is 16-byte aligned.
1722 */
1723 KASSERT(b0_len - CCM_B0_SIZE == CCM_AAD_FIELD_SIZE,
1724 ("b0_len mismatch"));
1725 memset(dst + CCM_AAD_FIELD_SIZE, 0,
1726 8 - CCM_AAD_FIELD_SIZE);
1727 idata = (void *)(dst + 8);
1728 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
1729 idata->len = htobe32(0);
1730 dst = (void *)(idata + 1);
1731 }
1732 ccr_write_ulptx_sgl(s, dst, sgl_nsegs);
1733 }
1734
1735 /* XXX: TODO backpressure */
1736 t4_wrq_tx(sc->adapter, wr);
1737
1738 explicit_bzero(iv, sizeof(iv));
1739 return (0);
1740 }
1741
1742 static int
ccr_ccm_done(struct ccr_softc * sc,struct ccr_session * s,struct cryptop * crp,const struct cpl_fw6_pld * cpl,int error)1743 ccr_ccm_done(struct ccr_softc *sc, struct ccr_session *s,
1744 struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error)
1745 {
1746
1747 /*
1748 * The updated IV to permit chained requests is at
1749 * cpl->data[2], but OCF doesn't permit chained requests.
1750 *
1751 * Note that the hardware should always verify the CBC MAC
1752 * hash.
1753 */
1754 return (error);
1755 }
1756
1757 /*
1758 * Use the software session for requests not supported by the crypto
1759 * engine (e.g. CCM and GCM requests with an empty payload).
1760 */
1761 static int
ccr_soft_done(struct cryptop * crp)1762 ccr_soft_done(struct cryptop *crp)
1763 {
1764 struct cryptop *orig;
1765
1766 orig = crp->crp_opaque;
1767 orig->crp_etype = crp->crp_etype;
1768 crypto_freereq(crp);
1769 crypto_done(orig);
1770 return (0);
1771 }
1772
1773 static void
ccr_soft(struct ccr_session * s,struct cryptop * crp)1774 ccr_soft(struct ccr_session *s, struct cryptop *crp)
1775 {
1776 struct cryptop *new;
1777 int error;
1778
1779 new = crypto_clonereq(crp, s->sw_session, M_NOWAIT);
1780 if (new == NULL) {
1781 crp->crp_etype = ENOMEM;
1782 crypto_done(crp);
1783 return;
1784 }
1785
1786 /*
1787 * XXX: This only really needs CRYPTO_ASYNC_ORDERED if the
1788 * original request was dispatched that way. There is no way
1789 * to know that though since crypto_dispatch_async() discards
1790 * the flag for async backends (such as ccr(4)).
1791 */
1792 new->crp_opaque = crp;
1793 new->crp_callback = ccr_soft_done;
1794 error = crypto_dispatch_async(new, CRYPTO_ASYNC_ORDERED);
1795 if (error != 0) {
1796 crp->crp_etype = error;
1797 crypto_done(crp);
1798 }
1799 }
1800
1801 static void
ccr_identify(driver_t * driver,device_t parent)1802 ccr_identify(driver_t *driver, device_t parent)
1803 {
1804 struct adapter *sc;
1805
1806 sc = device_get_softc(parent);
1807 if (sc->cryptocaps & FW_CAPS_CONFIG_CRYPTO_LOOKASIDE &&
1808 device_find_child(parent, "ccr", -1) == NULL)
1809 device_add_child(parent, "ccr", -1);
1810 }
1811
1812 static int
ccr_probe(device_t dev)1813 ccr_probe(device_t dev)
1814 {
1815
1816 device_set_desc(dev, "Chelsio Crypto Accelerator");
1817 return (BUS_PROBE_DEFAULT);
1818 }
1819
1820 static void
ccr_sysctls(struct ccr_softc * sc)1821 ccr_sysctls(struct ccr_softc *sc)
1822 {
1823 struct sysctl_ctx_list *ctx = &sc->ctx;
1824 struct sysctl_oid *oid, *port_oid;
1825 struct sysctl_oid_list *children;
1826 char buf[16];
1827 int i;
1828
1829 /*
1830 * dev.ccr.X.
1831 */
1832 oid = device_get_sysctl_tree(sc->dev);
1833 children = SYSCTL_CHILDREN(oid);
1834
1835 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "port_mask", CTLFLAG_RW,
1836 &sc->port_mask, 0, "Mask of enabled ports");
1837
1838 /*
1839 * dev.ccr.X.stats.
1840 */
1841 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats",
1842 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "statistics");
1843 children = SYSCTL_CHILDREN(oid);
1844
1845 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "hash", CTLFLAG_RD,
1846 &sc->stats_hash, "Hash requests submitted");
1847 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "hmac", CTLFLAG_RD,
1848 &sc->stats_hmac, "HMAC requests submitted");
1849 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "cipher_encrypt",
1850 CTLFLAG_RD, &sc->stats_cipher_encrypt,
1851 "Cipher encryption requests submitted");
1852 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "cipher_decrypt",
1853 CTLFLAG_RD, &sc->stats_cipher_decrypt,
1854 "Cipher decryption requests submitted");
1855 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "eta_encrypt",
1856 CTLFLAG_RD, &sc->stats_eta_encrypt,
1857 "Combined AES+HMAC encryption requests submitted");
1858 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "eta_decrypt",
1859 CTLFLAG_RD, &sc->stats_eta_decrypt,
1860 "Combined AES+HMAC decryption requests submitted");
1861 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_encrypt",
1862 CTLFLAG_RD, &sc->stats_gcm_encrypt,
1863 "AES-GCM encryption requests submitted");
1864 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_decrypt",
1865 CTLFLAG_RD, &sc->stats_gcm_decrypt,
1866 "AES-GCM decryption requests submitted");
1867 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ccm_encrypt",
1868 CTLFLAG_RD, &sc->stats_ccm_encrypt,
1869 "AES-CCM encryption requests submitted");
1870 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ccm_decrypt",
1871 CTLFLAG_RD, &sc->stats_ccm_decrypt,
1872 "AES-CCM decryption requests submitted");
1873 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "wr_nomem", CTLFLAG_RD,
1874 &sc->stats_wr_nomem, "Work request memory allocation failures");
1875 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "inflight", CTLFLAG_RD,
1876 &sc->stats_inflight, "Requests currently pending");
1877 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "mac_error", CTLFLAG_RD,
1878 &sc->stats_mac_error, "MAC errors");
1879 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "pad_error", CTLFLAG_RD,
1880 &sc->stats_pad_error, "Padding errors");
1881 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sglist_error",
1882 CTLFLAG_RD, &sc->stats_sglist_error,
1883 "Requests for which DMA mapping failed");
1884 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "process_error",
1885 CTLFLAG_RD, &sc->stats_process_error,
1886 "Requests failed during queueing");
1887 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sw_fallback",
1888 CTLFLAG_RD, &sc->stats_sw_fallback,
1889 "Requests processed by falling back to software");
1890
1891 /*
1892 * dev.ccr.X.stats.port
1893 */
1894 port_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "port",
1895 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Per-port statistics");
1896
1897 for (i = 0; i < nitems(sc->ports); i++) {
1898 if (sc->ports[i].rxq == NULL)
1899 continue;
1900
1901 /*
1902 * dev.ccr.X.stats.port.Y
1903 */
1904 snprintf(buf, sizeof(buf), "%d", i);
1905 oid = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(port_oid), OID_AUTO,
1906 buf, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, buf);
1907 children = SYSCTL_CHILDREN(oid);
1908
1909 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "active_sessions",
1910 CTLFLAG_RD, &sc->ports[i].active_sessions, 0,
1911 "Count of active sessions");
1912 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "queued",
1913 CTLFLAG_RD, &sc->ports[i].stats_queued, "Requests queued");
1914 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "completed",
1915 CTLFLAG_RD, &sc->ports[i].stats_completed,
1916 "Requests completed");
1917 }
1918 }
1919
1920 static void
ccr_init_port(struct ccr_softc * sc,int port)1921 ccr_init_port(struct ccr_softc *sc, int port)
1922 {
1923 struct port_info *pi;
1924
1925 pi = sc->adapter->port[port];
1926 sc->ports[port].txq = &sc->adapter->sge.ctrlq[port];
1927 sc->ports[port].rxq = &sc->adapter->sge.rxq[pi->vi->first_rxq];
1928 sc->ports[port].rx_channel_id = pi->rx_chan;
1929 sc->ports[port].tx_channel_id = pi->tx_chan;
1930 sc->ports[port].stats_queued = counter_u64_alloc(M_WAITOK);
1931 sc->ports[port].stats_completed = counter_u64_alloc(M_WAITOK);
1932 _Static_assert(sizeof(sc->port_mask) * NBBY >= MAX_NPORTS - 1,
1933 "Too many ports to fit in port_mask");
1934
1935 /*
1936 * Completions for crypto requests on port 1 can sometimes
1937 * return a stale cookie value due to a firmware bug. Disable
1938 * requests on port 1 by default on affected firmware.
1939 */
1940 if (sc->adapter->params.fw_vers >= FW_VERSION32(1, 25, 4, 0) ||
1941 port == 0)
1942 sc->port_mask |= 1u << port;
1943 }
1944
1945 static int
ccr_attach(device_t dev)1946 ccr_attach(device_t dev)
1947 {
1948 struct ccr_softc *sc;
1949 int32_t cid;
1950 int i;
1951
1952 sc = device_get_softc(dev);
1953 sc->dev = dev;
1954 sysctl_ctx_init(&sc->ctx);
1955 sc->adapter = device_get_softc(device_get_parent(dev));
1956 for_each_port(sc->adapter, i) {
1957 ccr_init_port(sc, i);
1958 }
1959 cid = crypto_get_driverid(dev, sizeof(struct ccr_session),
1960 CRYPTOCAP_F_HARDWARE);
1961 if (cid < 0) {
1962 device_printf(dev, "could not get crypto driver id\n");
1963 return (ENXIO);
1964 }
1965 sc->cid = cid;
1966
1967 /*
1968 * The FID must be the first RXQ for port 0 regardless of
1969 * which port is used to service the request.
1970 */
1971 sc->first_rxq_id = sc->adapter->sge.rxq[0].iq.abs_id;
1972
1973 mtx_init(&sc->lock, "ccr", NULL, MTX_DEF);
1974 sc->iv_aad_buf = malloc(MAX_AAD_LEN, M_CCR, M_WAITOK);
1975 sc->sg_iv_aad = sglist_build(sc->iv_aad_buf, MAX_AAD_LEN, M_WAITOK);
1976 sc->stats_cipher_encrypt = counter_u64_alloc(M_WAITOK);
1977 sc->stats_cipher_decrypt = counter_u64_alloc(M_WAITOK);
1978 sc->stats_hash = counter_u64_alloc(M_WAITOK);
1979 sc->stats_hmac = counter_u64_alloc(M_WAITOK);
1980 sc->stats_eta_encrypt = counter_u64_alloc(M_WAITOK);
1981 sc->stats_eta_decrypt = counter_u64_alloc(M_WAITOK);
1982 sc->stats_gcm_encrypt = counter_u64_alloc(M_WAITOK);
1983 sc->stats_gcm_decrypt = counter_u64_alloc(M_WAITOK);
1984 sc->stats_ccm_encrypt = counter_u64_alloc(M_WAITOK);
1985 sc->stats_ccm_decrypt = counter_u64_alloc(M_WAITOK);
1986 sc->stats_wr_nomem = counter_u64_alloc(M_WAITOK);
1987 sc->stats_inflight = counter_u64_alloc(M_WAITOK);
1988 sc->stats_mac_error = counter_u64_alloc(M_WAITOK);
1989 sc->stats_pad_error = counter_u64_alloc(M_WAITOK);
1990 sc->stats_sglist_error = counter_u64_alloc(M_WAITOK);
1991 sc->stats_process_error = counter_u64_alloc(M_WAITOK);
1992 sc->stats_sw_fallback = counter_u64_alloc(M_WAITOK);
1993 ccr_sysctls(sc);
1994
1995 return (0);
1996 }
1997
1998 static void
ccr_free_port(struct ccr_softc * sc,int port)1999 ccr_free_port(struct ccr_softc *sc, int port)
2000 {
2001
2002 counter_u64_free(sc->ports[port].stats_queued);
2003 counter_u64_free(sc->ports[port].stats_completed);
2004 }
2005
2006 static int
ccr_detach(device_t dev)2007 ccr_detach(device_t dev)
2008 {
2009 struct ccr_softc *sc;
2010 int i;
2011
2012 sc = device_get_softc(dev);
2013
2014 mtx_lock(&sc->lock);
2015 sc->detaching = true;
2016 mtx_unlock(&sc->lock);
2017
2018 crypto_unregister_all(sc->cid);
2019
2020 sysctl_ctx_free(&sc->ctx);
2021 mtx_destroy(&sc->lock);
2022 counter_u64_free(sc->stats_cipher_encrypt);
2023 counter_u64_free(sc->stats_cipher_decrypt);
2024 counter_u64_free(sc->stats_hash);
2025 counter_u64_free(sc->stats_hmac);
2026 counter_u64_free(sc->stats_eta_encrypt);
2027 counter_u64_free(sc->stats_eta_decrypt);
2028 counter_u64_free(sc->stats_gcm_encrypt);
2029 counter_u64_free(sc->stats_gcm_decrypt);
2030 counter_u64_free(sc->stats_ccm_encrypt);
2031 counter_u64_free(sc->stats_ccm_decrypt);
2032 counter_u64_free(sc->stats_wr_nomem);
2033 counter_u64_free(sc->stats_inflight);
2034 counter_u64_free(sc->stats_mac_error);
2035 counter_u64_free(sc->stats_pad_error);
2036 counter_u64_free(sc->stats_sglist_error);
2037 counter_u64_free(sc->stats_process_error);
2038 counter_u64_free(sc->stats_sw_fallback);
2039 for_each_port(sc->adapter, i) {
2040 ccr_free_port(sc, i);
2041 }
2042 sglist_free(sc->sg_iv_aad);
2043 free(sc->iv_aad_buf, M_CCR);
2044 return (0);
2045 }
2046
2047 static void
ccr_init_hash_digest(struct ccr_session * s)2048 ccr_init_hash_digest(struct ccr_session *s)
2049 {
2050 union authctx auth_ctx;
2051 const struct auth_hash *axf;
2052
2053 axf = s->hmac.auth_hash;
2054 axf->Init(&auth_ctx);
2055 t4_copy_partial_hash(axf->type, &auth_ctx, s->hmac.pads);
2056 }
2057
2058 static bool
ccr_aes_check_keylen(int alg,int klen)2059 ccr_aes_check_keylen(int alg, int klen)
2060 {
2061
2062 switch (klen * 8) {
2063 case 128:
2064 case 192:
2065 if (alg == CRYPTO_AES_XTS)
2066 return (false);
2067 break;
2068 case 256:
2069 break;
2070 case 512:
2071 if (alg != CRYPTO_AES_XTS)
2072 return (false);
2073 break;
2074 default:
2075 return (false);
2076 }
2077 return (true);
2078 }
2079
2080 static void
ccr_aes_setkey(struct ccr_session * s,const void * key,int klen)2081 ccr_aes_setkey(struct ccr_session *s, const void *key, int klen)
2082 {
2083 unsigned int ck_size, iopad_size, kctx_flits, kctx_len, kbits, mk_size;
2084 unsigned int opad_present;
2085
2086 if (s->cipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS)
2087 kbits = (klen / 2) * 8;
2088 else
2089 kbits = klen * 8;
2090 switch (kbits) {
2091 case 128:
2092 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2093 break;
2094 case 192:
2095 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
2096 break;
2097 case 256:
2098 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
2099 break;
2100 default:
2101 panic("should not get here");
2102 }
2103
2104 s->cipher.key_len = klen;
2105 memcpy(s->cipher.enckey, key, s->cipher.key_len);
2106 switch (s->cipher.cipher_mode) {
2107 case SCMD_CIPH_MODE_AES_CBC:
2108 case SCMD_CIPH_MODE_AES_XTS:
2109 t4_aes_getdeckey(s->cipher.deckey, key, kbits);
2110 break;
2111 }
2112
2113 kctx_len = roundup2(s->cipher.key_len, 16);
2114 switch (s->mode) {
2115 case ETA:
2116 mk_size = s->hmac.mk_size;
2117 opad_present = 1;
2118 iopad_size = roundup2(s->hmac.partial_digest_len, 16);
2119 kctx_len += iopad_size * 2;
2120 break;
2121 case GCM:
2122 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
2123 opad_present = 0;
2124 kctx_len += GMAC_BLOCK_LEN;
2125 break;
2126 case CCM:
2127 switch (kbits) {
2128 case 128:
2129 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
2130 break;
2131 case 192:
2132 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192;
2133 break;
2134 case 256:
2135 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
2136 break;
2137 default:
2138 panic("should not get here");
2139 }
2140 opad_present = 0;
2141 kctx_len *= 2;
2142 break;
2143 default:
2144 mk_size = CHCR_KEYCTX_NO_KEY;
2145 opad_present = 0;
2146 break;
2147 }
2148 kctx_flits = (sizeof(struct _key_ctx) + kctx_len) / 16;
2149 s->cipher.key_ctx_hdr = htobe32(V_KEY_CONTEXT_CTX_LEN(kctx_flits) |
2150 V_KEY_CONTEXT_DUAL_CK(s->cipher.cipher_mode ==
2151 SCMD_CIPH_MODE_AES_XTS) |
2152 V_KEY_CONTEXT_OPAD_PRESENT(opad_present) |
2153 V_KEY_CONTEXT_SALT_PRESENT(1) | V_KEY_CONTEXT_CK_SIZE(ck_size) |
2154 V_KEY_CONTEXT_MK_SIZE(mk_size) | V_KEY_CONTEXT_VALID(1));
2155 }
2156
2157 static bool
ccr_auth_supported(const struct crypto_session_params * csp)2158 ccr_auth_supported(const struct crypto_session_params *csp)
2159 {
2160
2161 switch (csp->csp_auth_alg) {
2162 case CRYPTO_SHA1:
2163 case CRYPTO_SHA2_224:
2164 case CRYPTO_SHA2_256:
2165 case CRYPTO_SHA2_384:
2166 case CRYPTO_SHA2_512:
2167 case CRYPTO_SHA1_HMAC:
2168 case CRYPTO_SHA2_224_HMAC:
2169 case CRYPTO_SHA2_256_HMAC:
2170 case CRYPTO_SHA2_384_HMAC:
2171 case CRYPTO_SHA2_512_HMAC:
2172 break;
2173 default:
2174 return (false);
2175 }
2176 return (true);
2177 }
2178
2179 static bool
ccr_cipher_supported(const struct crypto_session_params * csp)2180 ccr_cipher_supported(const struct crypto_session_params *csp)
2181 {
2182
2183 switch (csp->csp_cipher_alg) {
2184 case CRYPTO_AES_CBC:
2185 if (csp->csp_ivlen != AES_BLOCK_LEN)
2186 return (false);
2187 break;
2188 case CRYPTO_AES_ICM:
2189 if (csp->csp_ivlen != AES_BLOCK_LEN)
2190 return (false);
2191 break;
2192 case CRYPTO_AES_XTS:
2193 if (csp->csp_ivlen != AES_XTS_IV_LEN)
2194 return (false);
2195 break;
2196 default:
2197 return (false);
2198 }
2199 return (ccr_aes_check_keylen(csp->csp_cipher_alg,
2200 csp->csp_cipher_klen));
2201 }
2202
2203 static int
ccr_cipher_mode(const struct crypto_session_params * csp)2204 ccr_cipher_mode(const struct crypto_session_params *csp)
2205 {
2206
2207 switch (csp->csp_cipher_alg) {
2208 case CRYPTO_AES_CBC:
2209 return (SCMD_CIPH_MODE_AES_CBC);
2210 case CRYPTO_AES_ICM:
2211 return (SCMD_CIPH_MODE_AES_CTR);
2212 case CRYPTO_AES_NIST_GCM_16:
2213 return (SCMD_CIPH_MODE_AES_GCM);
2214 case CRYPTO_AES_XTS:
2215 return (SCMD_CIPH_MODE_AES_XTS);
2216 case CRYPTO_AES_CCM_16:
2217 return (SCMD_CIPH_MODE_AES_CCM);
2218 default:
2219 return (SCMD_CIPH_MODE_NOP);
2220 }
2221 }
2222
2223 static int
ccr_probesession(device_t dev,const struct crypto_session_params * csp)2224 ccr_probesession(device_t dev, const struct crypto_session_params *csp)
2225 {
2226 unsigned int cipher_mode;
2227
2228 if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) !=
2229 0)
2230 return (EINVAL);
2231 switch (csp->csp_mode) {
2232 case CSP_MODE_DIGEST:
2233 if (!ccr_auth_supported(csp))
2234 return (EINVAL);
2235 break;
2236 case CSP_MODE_CIPHER:
2237 if (!ccr_cipher_supported(csp))
2238 return (EINVAL);
2239 break;
2240 case CSP_MODE_AEAD:
2241 switch (csp->csp_cipher_alg) {
2242 case CRYPTO_AES_NIST_GCM_16:
2243 case CRYPTO_AES_CCM_16:
2244 break;
2245 default:
2246 return (EINVAL);
2247 }
2248 break;
2249 case CSP_MODE_ETA:
2250 if (!ccr_auth_supported(csp) || !ccr_cipher_supported(csp))
2251 return (EINVAL);
2252 break;
2253 default:
2254 return (EINVAL);
2255 }
2256
2257 if (csp->csp_cipher_klen != 0) {
2258 cipher_mode = ccr_cipher_mode(csp);
2259 if (cipher_mode == SCMD_CIPH_MODE_NOP)
2260 return (EINVAL);
2261 }
2262
2263 return (CRYPTODEV_PROBE_HARDWARE);
2264 }
2265
2266 /*
2267 * Select an available port with the lowest number of active sessions.
2268 */
2269 static struct ccr_port *
ccr_choose_port(struct ccr_softc * sc)2270 ccr_choose_port(struct ccr_softc *sc)
2271 {
2272 struct ccr_port *best, *p;
2273 int i;
2274
2275 mtx_assert(&sc->lock, MA_OWNED);
2276 best = NULL;
2277 for (i = 0; i < nitems(sc->ports); i++) {
2278 p = &sc->ports[i];
2279
2280 /* Ignore non-existent ports. */
2281 if (p->rxq == NULL)
2282 continue;
2283
2284 /*
2285 * XXX: Ignore ports whose queues aren't initialized.
2286 * This is racy as the rxq can be destroyed by the
2287 * associated VI detaching. Eventually ccr should use
2288 * dedicated queues.
2289 */
2290 if (p->rxq->iq.adapter == NULL || p->txq->adapter == NULL)
2291 continue;
2292
2293 if ((sc->port_mask & (1u << i)) == 0)
2294 continue;
2295
2296 if (best == NULL ||
2297 p->active_sessions < best->active_sessions)
2298 best = p;
2299 }
2300 return (best);
2301 }
2302
2303 static void
ccr_delete_session(struct ccr_session * s)2304 ccr_delete_session(struct ccr_session *s)
2305 {
2306 crypto_freesession(s->sw_session);
2307 sglist_free(s->sg_input);
2308 sglist_free(s->sg_output);
2309 sglist_free(s->sg_ulptx);
2310 sglist_free(s->sg_dsgl);
2311 mtx_destroy(&s->lock);
2312 }
2313
2314 static int
ccr_newsession(device_t dev,crypto_session_t cses,const struct crypto_session_params * csp)2315 ccr_newsession(device_t dev, crypto_session_t cses,
2316 const struct crypto_session_params *csp)
2317 {
2318 struct ccr_softc *sc;
2319 struct ccr_session *s;
2320 const struct auth_hash *auth_hash;
2321 unsigned int auth_mode, cipher_mode, mk_size;
2322 unsigned int partial_digest_len;
2323 int error;
2324
2325 switch (csp->csp_auth_alg) {
2326 case CRYPTO_SHA1:
2327 case CRYPTO_SHA1_HMAC:
2328 auth_hash = &auth_hash_hmac_sha1;
2329 auth_mode = SCMD_AUTH_MODE_SHA1;
2330 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
2331 partial_digest_len = SHA1_HASH_LEN;
2332 break;
2333 case CRYPTO_SHA2_224:
2334 case CRYPTO_SHA2_224_HMAC:
2335 auth_hash = &auth_hash_hmac_sha2_224;
2336 auth_mode = SCMD_AUTH_MODE_SHA224;
2337 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
2338 partial_digest_len = SHA2_256_HASH_LEN;
2339 break;
2340 case CRYPTO_SHA2_256:
2341 case CRYPTO_SHA2_256_HMAC:
2342 auth_hash = &auth_hash_hmac_sha2_256;
2343 auth_mode = SCMD_AUTH_MODE_SHA256;
2344 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
2345 partial_digest_len = SHA2_256_HASH_LEN;
2346 break;
2347 case CRYPTO_SHA2_384:
2348 case CRYPTO_SHA2_384_HMAC:
2349 auth_hash = &auth_hash_hmac_sha2_384;
2350 auth_mode = SCMD_AUTH_MODE_SHA512_384;
2351 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
2352 partial_digest_len = SHA2_512_HASH_LEN;
2353 break;
2354 case CRYPTO_SHA2_512:
2355 case CRYPTO_SHA2_512_HMAC:
2356 auth_hash = &auth_hash_hmac_sha2_512;
2357 auth_mode = SCMD_AUTH_MODE_SHA512_512;
2358 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
2359 partial_digest_len = SHA2_512_HASH_LEN;
2360 break;
2361 default:
2362 auth_hash = NULL;
2363 auth_mode = SCMD_AUTH_MODE_NOP;
2364 mk_size = 0;
2365 partial_digest_len = 0;
2366 break;
2367 }
2368
2369 cipher_mode = ccr_cipher_mode(csp);
2370
2371 #ifdef INVARIANTS
2372 switch (csp->csp_mode) {
2373 case CSP_MODE_CIPHER:
2374 if (cipher_mode == SCMD_CIPH_MODE_NOP ||
2375 cipher_mode == SCMD_CIPH_MODE_AES_GCM ||
2376 cipher_mode == SCMD_CIPH_MODE_AES_CCM)
2377 panic("invalid cipher algo");
2378 break;
2379 case CSP_MODE_DIGEST:
2380 if (auth_mode == SCMD_AUTH_MODE_NOP)
2381 panic("invalid auth algo");
2382 break;
2383 case CSP_MODE_AEAD:
2384 if (cipher_mode != SCMD_CIPH_MODE_AES_GCM &&
2385 cipher_mode != SCMD_CIPH_MODE_AES_CCM)
2386 panic("invalid aead cipher algo");
2387 if (auth_mode != SCMD_AUTH_MODE_NOP)
2388 panic("invalid aead auth aglo");
2389 break;
2390 case CSP_MODE_ETA:
2391 if (cipher_mode == SCMD_CIPH_MODE_NOP ||
2392 cipher_mode == SCMD_CIPH_MODE_AES_GCM ||
2393 cipher_mode == SCMD_CIPH_MODE_AES_CCM)
2394 panic("invalid cipher algo");
2395 if (auth_mode == SCMD_AUTH_MODE_NOP)
2396 panic("invalid auth algo");
2397 break;
2398 default:
2399 panic("invalid csp mode");
2400 }
2401 #endif
2402
2403 s = crypto_get_driver_session(cses);
2404 mtx_init(&s->lock, "ccr session", NULL, MTX_DEF);
2405 s->sg_input = sglist_alloc(TX_SGL_SEGS, M_NOWAIT);
2406 s->sg_output = sglist_alloc(TX_SGL_SEGS, M_NOWAIT);
2407 s->sg_ulptx = sglist_alloc(TX_SGL_SEGS, M_NOWAIT);
2408 s->sg_dsgl = sglist_alloc(MAX_RX_PHYS_DSGL_SGE, M_NOWAIT);
2409 if (s->sg_input == NULL || s->sg_output == NULL ||
2410 s->sg_ulptx == NULL || s->sg_dsgl == NULL) {
2411 ccr_delete_session(s);
2412 return (ENOMEM);
2413 }
2414
2415 if (csp->csp_mode == CSP_MODE_AEAD) {
2416 error = crypto_newsession(&s->sw_session, csp,
2417 CRYPTOCAP_F_SOFTWARE);
2418 if (error) {
2419 ccr_delete_session(s);
2420 return (error);
2421 }
2422 }
2423
2424 sc = device_get_softc(dev);
2425 s->sc = sc;
2426
2427 mtx_lock(&sc->lock);
2428 if (sc->detaching) {
2429 mtx_unlock(&sc->lock);
2430 ccr_delete_session(s);
2431 return (ENXIO);
2432 }
2433
2434 s->port = ccr_choose_port(sc);
2435 if (s->port == NULL) {
2436 mtx_unlock(&sc->lock);
2437 ccr_delete_session(s);
2438 return (ENXIO);
2439 }
2440
2441 switch (csp->csp_mode) {
2442 case CSP_MODE_AEAD:
2443 if (cipher_mode == SCMD_CIPH_MODE_AES_CCM)
2444 s->mode = CCM;
2445 else
2446 s->mode = GCM;
2447 break;
2448 case CSP_MODE_ETA:
2449 s->mode = ETA;
2450 break;
2451 case CSP_MODE_DIGEST:
2452 if (csp->csp_auth_klen != 0)
2453 s->mode = HMAC;
2454 else
2455 s->mode = HASH;
2456 break;
2457 case CSP_MODE_CIPHER:
2458 s->mode = CIPHER;
2459 break;
2460 }
2461
2462 if (s->mode == GCM) {
2463 if (csp->csp_auth_mlen == 0)
2464 s->gmac.hash_len = AES_GMAC_HASH_LEN;
2465 else
2466 s->gmac.hash_len = csp->csp_auth_mlen;
2467 t4_init_gmac_hash(csp->csp_cipher_key, csp->csp_cipher_klen,
2468 s->gmac.ghash_h);
2469 } else if (s->mode == CCM) {
2470 if (csp->csp_auth_mlen == 0)
2471 s->ccm_mac.hash_len = AES_CBC_MAC_HASH_LEN;
2472 else
2473 s->ccm_mac.hash_len = csp->csp_auth_mlen;
2474 } else if (auth_mode != SCMD_AUTH_MODE_NOP) {
2475 s->hmac.auth_hash = auth_hash;
2476 s->hmac.auth_mode = auth_mode;
2477 s->hmac.mk_size = mk_size;
2478 s->hmac.partial_digest_len = partial_digest_len;
2479 if (csp->csp_auth_mlen == 0)
2480 s->hmac.hash_len = auth_hash->hashsize;
2481 else
2482 s->hmac.hash_len = csp->csp_auth_mlen;
2483 if (csp->csp_auth_key != NULL)
2484 t4_init_hmac_digest(auth_hash, partial_digest_len,
2485 csp->csp_auth_key, csp->csp_auth_klen,
2486 s->hmac.pads);
2487 else
2488 ccr_init_hash_digest(s);
2489 }
2490 if (cipher_mode != SCMD_CIPH_MODE_NOP) {
2491 s->cipher.cipher_mode = cipher_mode;
2492 s->cipher.iv_len = csp->csp_ivlen;
2493 if (csp->csp_cipher_key != NULL)
2494 ccr_aes_setkey(s, csp->csp_cipher_key,
2495 csp->csp_cipher_klen);
2496 }
2497
2498 s->port->active_sessions++;
2499 mtx_unlock(&sc->lock);
2500 return (0);
2501 }
2502
2503 static void
ccr_freesession(device_t dev,crypto_session_t cses)2504 ccr_freesession(device_t dev, crypto_session_t cses)
2505 {
2506 struct ccr_softc *sc;
2507 struct ccr_session *s;
2508
2509 sc = device_get_softc(dev);
2510 s = crypto_get_driver_session(cses);
2511 #ifdef INVARIANTS
2512 if (s->pending != 0)
2513 device_printf(dev,
2514 "session %p freed with %d pending requests\n", s,
2515 s->pending);
2516 #endif
2517 mtx_lock(&sc->lock);
2518 s->port->active_sessions--;
2519 mtx_unlock(&sc->lock);
2520 ccr_delete_session(s);
2521 }
2522
2523 static int
ccr_process(device_t dev,struct cryptop * crp,int hint)2524 ccr_process(device_t dev, struct cryptop *crp, int hint)
2525 {
2526 const struct crypto_session_params *csp;
2527 struct ccr_softc *sc;
2528 struct ccr_session *s;
2529 int error;
2530
2531 csp = crypto_get_params(crp->crp_session);
2532 s = crypto_get_driver_session(crp->crp_session);
2533 sc = device_get_softc(dev);
2534
2535 mtx_lock(&s->lock);
2536 error = ccr_populate_sglist(s->sg_input, &crp->crp_buf);
2537 if (error == 0 && CRYPTO_HAS_OUTPUT_BUFFER(crp))
2538 error = ccr_populate_sglist(s->sg_output, &crp->crp_obuf);
2539 if (error) {
2540 counter_u64_add(sc->stats_sglist_error, 1);
2541 goto out;
2542 }
2543
2544 switch (s->mode) {
2545 case HASH:
2546 error = ccr_hash(sc, s, crp);
2547 if (error == 0)
2548 counter_u64_add(sc->stats_hash, 1);
2549 break;
2550 case HMAC:
2551 if (crp->crp_auth_key != NULL)
2552 t4_init_hmac_digest(s->hmac.auth_hash,
2553 s->hmac.partial_digest_len, crp->crp_auth_key,
2554 csp->csp_auth_klen, s->hmac.pads);
2555 error = ccr_hash(sc, s, crp);
2556 if (error == 0)
2557 counter_u64_add(sc->stats_hmac, 1);
2558 break;
2559 case CIPHER:
2560 if (crp->crp_cipher_key != NULL)
2561 ccr_aes_setkey(s, crp->crp_cipher_key,
2562 csp->csp_cipher_klen);
2563 error = ccr_cipher(sc, s, crp);
2564 if (error == 0) {
2565 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
2566 counter_u64_add(sc->stats_cipher_encrypt, 1);
2567 else
2568 counter_u64_add(sc->stats_cipher_decrypt, 1);
2569 }
2570 break;
2571 case ETA:
2572 if (crp->crp_auth_key != NULL)
2573 t4_init_hmac_digest(s->hmac.auth_hash,
2574 s->hmac.partial_digest_len, crp->crp_auth_key,
2575 csp->csp_auth_klen, s->hmac.pads);
2576 if (crp->crp_cipher_key != NULL)
2577 ccr_aes_setkey(s, crp->crp_cipher_key,
2578 csp->csp_cipher_klen);
2579 error = ccr_eta(sc, s, crp);
2580 if (error == 0) {
2581 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
2582 counter_u64_add(sc->stats_eta_encrypt, 1);
2583 else
2584 counter_u64_add(sc->stats_eta_decrypt, 1);
2585 }
2586 break;
2587 case GCM:
2588 if (crp->crp_cipher_key != NULL) {
2589 t4_init_gmac_hash(crp->crp_cipher_key,
2590 csp->csp_cipher_klen, s->gmac.ghash_h);
2591 ccr_aes_setkey(s, crp->crp_cipher_key,
2592 csp->csp_cipher_klen);
2593 }
2594 error = ccr_gcm(sc, s, crp);
2595 if (error == EMSGSIZE || error == EFBIG) {
2596 counter_u64_add(sc->stats_sw_fallback, 1);
2597 mtx_unlock(&s->lock);
2598 ccr_soft(s, crp);
2599 return (0);
2600 }
2601 if (error == 0) {
2602 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
2603 counter_u64_add(sc->stats_gcm_encrypt, 1);
2604 else
2605 counter_u64_add(sc->stats_gcm_decrypt, 1);
2606 }
2607 break;
2608 case CCM:
2609 if (crp->crp_cipher_key != NULL) {
2610 ccr_aes_setkey(s, crp->crp_cipher_key,
2611 csp->csp_cipher_klen);
2612 }
2613 error = ccr_ccm(sc, s, crp);
2614 if (error == EMSGSIZE || error == EFBIG) {
2615 counter_u64_add(sc->stats_sw_fallback, 1);
2616 mtx_unlock(&s->lock);
2617 ccr_soft(s, crp);
2618 return (0);
2619 }
2620 if (error == 0) {
2621 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
2622 counter_u64_add(sc->stats_ccm_encrypt, 1);
2623 else
2624 counter_u64_add(sc->stats_ccm_decrypt, 1);
2625 }
2626 break;
2627 }
2628
2629 if (error == 0) {
2630 #ifdef INVARIANTS
2631 s->pending++;
2632 #endif
2633 counter_u64_add(sc->stats_inflight, 1);
2634 counter_u64_add(s->port->stats_queued, 1);
2635 } else
2636 counter_u64_add(sc->stats_process_error, 1);
2637
2638 out:
2639 mtx_unlock(&s->lock);
2640
2641 if (error) {
2642 crp->crp_etype = error;
2643 crypto_done(crp);
2644 }
2645
2646 return (0);
2647 }
2648
2649 static int
do_cpl6_fw_pld(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)2650 do_cpl6_fw_pld(struct sge_iq *iq, const struct rss_header *rss,
2651 struct mbuf *m)
2652 {
2653 struct ccr_softc *sc;
2654 struct ccr_session *s;
2655 const struct cpl_fw6_pld *cpl;
2656 struct cryptop *crp;
2657 uint32_t status;
2658 int error;
2659
2660 if (m != NULL)
2661 cpl = mtod(m, const void *);
2662 else
2663 cpl = (const void *)(rss + 1);
2664
2665 crp = (struct cryptop *)(uintptr_t)be64toh(cpl->data[1]);
2666 s = crypto_get_driver_session(crp->crp_session);
2667 status = be64toh(cpl->data[0]);
2668 if (CHK_MAC_ERR_BIT(status) || CHK_PAD_ERR_BIT(status))
2669 error = EBADMSG;
2670 else
2671 error = 0;
2672
2673 sc = s->sc;
2674 #ifdef INVARIANTS
2675 mtx_lock(&s->lock);
2676 s->pending--;
2677 mtx_unlock(&s->lock);
2678 #endif
2679 counter_u64_add(sc->stats_inflight, -1);
2680 counter_u64_add(s->port->stats_completed, 1);
2681
2682 switch (s->mode) {
2683 case HASH:
2684 case HMAC:
2685 error = ccr_hash_done(sc, s, crp, cpl, error);
2686 break;
2687 case CIPHER:
2688 error = ccr_cipher_done(sc, s, crp, cpl, error);
2689 break;
2690 case ETA:
2691 error = ccr_eta_done(sc, s, crp, cpl, error);
2692 break;
2693 case GCM:
2694 error = ccr_gcm_done(sc, s, crp, cpl, error);
2695 break;
2696 case CCM:
2697 error = ccr_ccm_done(sc, s, crp, cpl, error);
2698 break;
2699 }
2700
2701 if (error == EBADMSG) {
2702 if (CHK_MAC_ERR_BIT(status))
2703 counter_u64_add(sc->stats_mac_error, 1);
2704 if (CHK_PAD_ERR_BIT(status))
2705 counter_u64_add(sc->stats_pad_error, 1);
2706 }
2707 crp->crp_etype = error;
2708 crypto_done(crp);
2709 m_freem(m);
2710 return (0);
2711 }
2712
2713 static int
ccr_modevent(module_t mod,int cmd,void * arg)2714 ccr_modevent(module_t mod, int cmd, void *arg)
2715 {
2716
2717 switch (cmd) {
2718 case MOD_LOAD:
2719 t4_register_cpl_handler(CPL_FW6_PLD, do_cpl6_fw_pld);
2720 return (0);
2721 case MOD_UNLOAD:
2722 t4_register_cpl_handler(CPL_FW6_PLD, NULL);
2723 return (0);
2724 default:
2725 return (EOPNOTSUPP);
2726 }
2727 }
2728
2729 static device_method_t ccr_methods[] = {
2730 DEVMETHOD(device_identify, ccr_identify),
2731 DEVMETHOD(device_probe, ccr_probe),
2732 DEVMETHOD(device_attach, ccr_attach),
2733 DEVMETHOD(device_detach, ccr_detach),
2734
2735 DEVMETHOD(cryptodev_probesession, ccr_probesession),
2736 DEVMETHOD(cryptodev_newsession, ccr_newsession),
2737 DEVMETHOD(cryptodev_freesession, ccr_freesession),
2738 DEVMETHOD(cryptodev_process, ccr_process),
2739
2740 DEVMETHOD_END
2741 };
2742
2743 static driver_t ccr_driver = {
2744 "ccr",
2745 ccr_methods,
2746 sizeof(struct ccr_softc)
2747 };
2748
2749 DRIVER_MODULE(ccr, t6nex, ccr_driver, ccr_modevent, NULL);
2750 MODULE_VERSION(ccr, 1);
2751 MODULE_DEPEND(ccr, crypto, 1, 1, 1);
2752 MODULE_DEPEND(ccr, t6nex, 1, 1, 1);
2753