1 /* SPDX-License-Identifier: BSD-2-Clause AND BSD-3-Clause */
2 /* $NetBSD: qat.c,v 1.6 2020/06/14 23:23:12 riastradh Exp $ */
3
4 /*
5 * Copyright (c) 2019 Internet Initiative Japan, Inc.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 * POSSIBILITY OF SUCH DAMAGE.
28 */
29
30 /*
31 * Copyright(c) 2007-2019 Intel Corporation. All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 *
37 * * Redistributions of source code must retain the above copyright
38 * notice, this list of conditions and the following disclaimer.
39 * * Redistributions in binary form must reproduce the above copyright
40 * notice, this list of conditions and the following disclaimer in
41 * the documentation and/or other materials provided with the
42 * distribution.
43 * * Neither the name of Intel Corporation nor the names of its
44 * contributors may be used to endorse or promote products derived
45 * from this software without specific prior written permission.
46 *
47 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
48 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
49 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
50 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
51 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
52 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
53 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
54 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
55 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
56 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
57 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
58 */
59
60 #include <sys/cdefs.h>
61 #if 0
62 __KERNEL_RCSID(0, "$NetBSD: qat.c,v 1.6 2020/06/14 23:23:12 riastradh Exp $");
63 #endif
64
65 #include <sys/param.h>
66 #include <sys/systm.h>
67 #include <sys/bus.h>
68 #include <sys/cpu.h>
69 #include <sys/firmware.h>
70 #include <sys/kernel.h>
71 #include <sys/mbuf.h>
72 #include <sys/md5.h>
73 #include <sys/module.h>
74 #include <sys/mutex.h>
75 #include <sys/smp.h>
76 #include <sys/sysctl.h>
77 #include <sys/rman.h>
78
79 #include <machine/bus.h>
80
81 #include <opencrypto/cryptodev.h>
82 #include <opencrypto/xform.h>
83
84 #include "cryptodev_if.h"
85
86 #include <dev/pci/pcireg.h>
87 #include <dev/pci/pcivar.h>
88
89 #include "qatreg.h"
90 #include "qatvar.h"
91 #include "qat_aevar.h"
92
93 extern struct qat_hw qat_hw_c2xxx;
94 extern struct qat_hw qat_hw_c3xxx;
95 extern struct qat_hw qat_hw_c62x;
96 extern struct qat_hw qat_hw_d15xx;
97 extern struct qat_hw qat_hw_dh895xcc;
98
99 #define PCI_VENDOR_INTEL 0x8086
100 #define PCI_PRODUCT_INTEL_C2000_IQIA_PHYS 0x1f18
101 #define PCI_PRODUCT_INTEL_C3K_QAT 0x19e2
102 #define PCI_PRODUCT_INTEL_C3K_QAT_VF 0x19e3
103 #define PCI_PRODUCT_INTEL_C620_QAT 0x37c8
104 #define PCI_PRODUCT_INTEL_C620_QAT_VF 0x37c9
105 #define PCI_PRODUCT_INTEL_XEOND_QAT 0x6f54
106 #define PCI_PRODUCT_INTEL_XEOND_QAT_VF 0x6f55
107 #define PCI_PRODUCT_INTEL_DH895XCC_QAT 0x0435
108 #define PCI_PRODUCT_INTEL_DH895XCC_QAT_VF 0x0443
109
110 static const struct qat_product {
111 uint16_t qatp_vendor;
112 uint16_t qatp_product;
113 const char *qatp_name;
114 enum qat_chip_type qatp_chip;
115 const struct qat_hw *qatp_hw;
116 } qat_products[] = {
117 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_C2000_IQIA_PHYS,
118 "Intel C2000 QuickAssist PF",
119 QAT_CHIP_C2XXX, &qat_hw_c2xxx },
120 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_C3K_QAT,
121 "Intel C3000 QuickAssist PF",
122 QAT_CHIP_C3XXX, &qat_hw_c3xxx },
123 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_C620_QAT,
124 "Intel C620/Xeon D-2100 QuickAssist PF",
125 QAT_CHIP_C62X, &qat_hw_c62x },
126 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XEOND_QAT,
127 "Intel Xeon D-1500 QuickAssist PF",
128 QAT_CHIP_D15XX, &qat_hw_d15xx },
129 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_DH895XCC_QAT,
130 "Intel 8950 QuickAssist PCIe Adapter PF",
131 QAT_CHIP_DH895XCC, &qat_hw_dh895xcc },
132 { 0, 0, NULL, 0, NULL },
133 };
134
135 /* Hash Algorithm specific structure */
136
137 /* SHA1 - 20 bytes - Initialiser state can be found in FIPS stds 180-2 */
138 static const uint8_t sha1_initial_state[QAT_HASH_SHA1_STATE_SIZE] = {
139 0x67, 0x45, 0x23, 0x01,
140 0xef, 0xcd, 0xab, 0x89,
141 0x98, 0xba, 0xdc, 0xfe,
142 0x10, 0x32, 0x54, 0x76,
143 0xc3, 0xd2, 0xe1, 0xf0
144 };
145
146 /* SHA 256 - 32 bytes - Initialiser state can be found in FIPS stds 180-2 */
147 static const uint8_t sha256_initial_state[QAT_HASH_SHA256_STATE_SIZE] = {
148 0x6a, 0x09, 0xe6, 0x67,
149 0xbb, 0x67, 0xae, 0x85,
150 0x3c, 0x6e, 0xf3, 0x72,
151 0xa5, 0x4f, 0xf5, 0x3a,
152 0x51, 0x0e, 0x52, 0x7f,
153 0x9b, 0x05, 0x68, 0x8c,
154 0x1f, 0x83, 0xd9, 0xab,
155 0x5b, 0xe0, 0xcd, 0x19
156 };
157
158 /* SHA 384 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */
159 static const uint8_t sha384_initial_state[QAT_HASH_SHA384_STATE_SIZE] = {
160 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8,
161 0x62, 0x9a, 0x29, 0x2a, 0x36, 0x7c, 0xd5, 0x07,
162 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, 0xdd, 0x17,
163 0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39,
164 0x67, 0x33, 0x26, 0x67, 0xff, 0xc0, 0x0b, 0x31,
165 0x8e, 0xb4, 0x4a, 0x87, 0x68, 0x58, 0x15, 0x11,
166 0xdb, 0x0c, 0x2e, 0x0d, 0x64, 0xf9, 0x8f, 0xa7,
167 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, 0xa4
168 };
169
170 /* SHA 512 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */
171 static const uint8_t sha512_initial_state[QAT_HASH_SHA512_STATE_SIZE] = {
172 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08,
173 0xbb, 0x67, 0xae, 0x85, 0x84, 0xca, 0xa7, 0x3b,
174 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94, 0xf8, 0x2b,
175 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1,
176 0x51, 0x0e, 0x52, 0x7f, 0xad, 0xe6, 0x82, 0xd1,
177 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f,
178 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b,
179 0x5b, 0xe0, 0xcd, 0x19, 0x13, 0x7e, 0x21, 0x79
180 };
181
182 static const struct qat_sym_hash_alg_info sha1_info = {
183 .qshai_digest_len = QAT_HASH_SHA1_DIGEST_SIZE,
184 .qshai_block_len = QAT_HASH_SHA1_BLOCK_SIZE,
185 .qshai_state_size = QAT_HASH_SHA1_STATE_SIZE,
186 .qshai_init_state = sha1_initial_state,
187 .qshai_sah = &auth_hash_hmac_sha1,
188 .qshai_state_offset = 0,
189 .qshai_state_word = 4,
190 };
191
192 static const struct qat_sym_hash_alg_info sha256_info = {
193 .qshai_digest_len = QAT_HASH_SHA256_DIGEST_SIZE,
194 .qshai_block_len = QAT_HASH_SHA256_BLOCK_SIZE,
195 .qshai_state_size = QAT_HASH_SHA256_STATE_SIZE,
196 .qshai_init_state = sha256_initial_state,
197 .qshai_sah = &auth_hash_hmac_sha2_256,
198 .qshai_state_offset = offsetof(SHA256_CTX, state),
199 .qshai_state_word = 4,
200 };
201
202 static const struct qat_sym_hash_alg_info sha384_info = {
203 .qshai_digest_len = QAT_HASH_SHA384_DIGEST_SIZE,
204 .qshai_block_len = QAT_HASH_SHA384_BLOCK_SIZE,
205 .qshai_state_size = QAT_HASH_SHA384_STATE_SIZE,
206 .qshai_init_state = sha384_initial_state,
207 .qshai_sah = &auth_hash_hmac_sha2_384,
208 .qshai_state_offset = offsetof(SHA384_CTX, state),
209 .qshai_state_word = 8,
210 };
211
212 static const struct qat_sym_hash_alg_info sha512_info = {
213 .qshai_digest_len = QAT_HASH_SHA512_DIGEST_SIZE,
214 .qshai_block_len = QAT_HASH_SHA512_BLOCK_SIZE,
215 .qshai_state_size = QAT_HASH_SHA512_STATE_SIZE,
216 .qshai_init_state = sha512_initial_state,
217 .qshai_sah = &auth_hash_hmac_sha2_512,
218 .qshai_state_offset = offsetof(SHA512_CTX, state),
219 .qshai_state_word = 8,
220 };
221
222 static const struct qat_sym_hash_alg_info aes_gcm_info = {
223 .qshai_digest_len = QAT_HASH_AES_GCM_DIGEST_SIZE,
224 .qshai_block_len = QAT_HASH_AES_GCM_BLOCK_SIZE,
225 .qshai_state_size = QAT_HASH_AES_GCM_STATE_SIZE,
226 .qshai_sah = &auth_hash_nist_gmac_aes_128,
227 };
228
229 /* Hash QAT specific structures */
230
231 static const struct qat_sym_hash_qat_info sha1_config = {
232 .qshqi_algo_enc = HW_AUTH_ALGO_SHA1,
233 .qshqi_auth_counter = QAT_HASH_SHA1_BLOCK_SIZE,
234 .qshqi_state1_len = HW_SHA1_STATE1_SZ,
235 .qshqi_state2_len = HW_SHA1_STATE2_SZ,
236 };
237
238 static const struct qat_sym_hash_qat_info sha256_config = {
239 .qshqi_algo_enc = HW_AUTH_ALGO_SHA256,
240 .qshqi_auth_counter = QAT_HASH_SHA256_BLOCK_SIZE,
241 .qshqi_state1_len = HW_SHA256_STATE1_SZ,
242 .qshqi_state2_len = HW_SHA256_STATE2_SZ
243 };
244
245 static const struct qat_sym_hash_qat_info sha384_config = {
246 .qshqi_algo_enc = HW_AUTH_ALGO_SHA384,
247 .qshqi_auth_counter = QAT_HASH_SHA384_BLOCK_SIZE,
248 .qshqi_state1_len = HW_SHA384_STATE1_SZ,
249 .qshqi_state2_len = HW_SHA384_STATE2_SZ
250 };
251
252 static const struct qat_sym_hash_qat_info sha512_config = {
253 .qshqi_algo_enc = HW_AUTH_ALGO_SHA512,
254 .qshqi_auth_counter = QAT_HASH_SHA512_BLOCK_SIZE,
255 .qshqi_state1_len = HW_SHA512_STATE1_SZ,
256 .qshqi_state2_len = HW_SHA512_STATE2_SZ
257 };
258
259 static const struct qat_sym_hash_qat_info aes_gcm_config = {
260 .qshqi_algo_enc = HW_AUTH_ALGO_GALOIS_128,
261 .qshqi_auth_counter = QAT_HASH_AES_GCM_BLOCK_SIZE,
262 .qshqi_state1_len = HW_GALOIS_128_STATE1_SZ,
263 .qshqi_state2_len =
264 HW_GALOIS_H_SZ + HW_GALOIS_LEN_A_SZ + HW_GALOIS_E_CTR0_SZ,
265 };
266
267 static const struct qat_sym_hash_def qat_sym_hash_defs[] = {
268 [QAT_SYM_HASH_SHA1] = { &sha1_info, &sha1_config },
269 [QAT_SYM_HASH_SHA256] = { &sha256_info, &sha256_config },
270 [QAT_SYM_HASH_SHA384] = { &sha384_info, &sha384_config },
271 [QAT_SYM_HASH_SHA512] = { &sha512_info, &sha512_config },
272 [QAT_SYM_HASH_AES_GCM] = { &aes_gcm_info, &aes_gcm_config },
273 };
274
275 static const struct qat_product *qat_lookup(device_t);
276 static int qat_probe(device_t);
277 static int qat_attach(device_t);
278 static int qat_init(device_t);
279 static int qat_start(device_t);
280 static int qat_detach(device_t);
281
282 static int qat_newsession(device_t dev, crypto_session_t cses,
283 const struct crypto_session_params *csp);
284 static void qat_freesession(device_t dev, crypto_session_t cses);
285
286 static int qat_setup_msix_intr(struct qat_softc *);
287
288 static void qat_etr_init(struct qat_softc *);
289 static void qat_etr_deinit(struct qat_softc *);
290 static void qat_etr_bank_init(struct qat_softc *, int);
291 static void qat_etr_bank_deinit(struct qat_softc *sc, int);
292
293 static void qat_etr_ap_bank_init(struct qat_softc *);
294 static void qat_etr_ap_bank_set_ring_mask(uint32_t *, uint32_t, int);
295 static void qat_etr_ap_bank_set_ring_dest(struct qat_softc *, uint32_t *,
296 uint32_t, int);
297 static void qat_etr_ap_bank_setup_ring(struct qat_softc *,
298 struct qat_ring *);
299 static int qat_etr_verify_ring_size(uint32_t, uint32_t);
300
301 static int qat_etr_ring_intr(struct qat_softc *, struct qat_bank *,
302 struct qat_ring *);
303 static void qat_etr_bank_intr(void *);
304
305 static void qat_arb_update(struct qat_softc *, struct qat_bank *);
306
307 static struct qat_sym_cookie *qat_crypto_alloc_sym_cookie(
308 struct qat_crypto_bank *);
309 static void qat_crypto_free_sym_cookie(struct qat_crypto_bank *,
310 struct qat_sym_cookie *);
311 static int qat_crypto_setup_ring(struct qat_softc *,
312 struct qat_crypto_bank *);
313 static int qat_crypto_bank_init(struct qat_softc *,
314 struct qat_crypto_bank *);
315 static int qat_crypto_init(struct qat_softc *);
316 static void qat_crypto_deinit(struct qat_softc *);
317 static int qat_crypto_start(struct qat_softc *);
318 static void qat_crypto_stop(struct qat_softc *);
319 static int qat_crypto_sym_rxintr(struct qat_softc *, void *, void *);
320
321 static MALLOC_DEFINE(M_QAT, "qat", "Intel QAT driver");
322
323 static const struct qat_product *
qat_lookup(device_t dev)324 qat_lookup(device_t dev)
325 {
326 const struct qat_product *qatp;
327
328 for (qatp = qat_products; qatp->qatp_name != NULL; qatp++) {
329 if (pci_get_vendor(dev) == qatp->qatp_vendor &&
330 pci_get_device(dev) == qatp->qatp_product)
331 return qatp;
332 }
333 return NULL;
334 }
335
336 static int
qat_probe(device_t dev)337 qat_probe(device_t dev)
338 {
339 const struct qat_product *prod;
340
341 prod = qat_lookup(dev);
342 if (prod != NULL) {
343 device_set_desc(dev, prod->qatp_name);
344 return BUS_PROBE_DEFAULT;
345 }
346 return ENXIO;
347 }
348
349 static int
qat_attach(device_t dev)350 qat_attach(device_t dev)
351 {
352 struct qat_softc *sc = device_get_softc(dev);
353 const struct qat_product *qatp;
354 int bar, count, error, i;
355
356 sc->sc_dev = dev;
357 sc->sc_rev = pci_get_revid(dev);
358 sc->sc_crypto.qcy_cid = -1;
359
360 qatp = qat_lookup(dev);
361 memcpy(&sc->sc_hw, qatp->qatp_hw, sizeof(struct qat_hw));
362
363 /* Determine active accelerators and engines */
364 sc->sc_accel_mask = sc->sc_hw.qhw_get_accel_mask(sc);
365 sc->sc_ae_mask = sc->sc_hw.qhw_get_ae_mask(sc);
366
367 sc->sc_accel_num = 0;
368 for (i = 0; i < sc->sc_hw.qhw_num_accel; i++) {
369 if (sc->sc_accel_mask & (1 << i))
370 sc->sc_accel_num++;
371 }
372 sc->sc_ae_num = 0;
373 for (i = 0; i < sc->sc_hw.qhw_num_engines; i++) {
374 if (sc->sc_ae_mask & (1 << i))
375 sc->sc_ae_num++;
376 }
377
378 if (!sc->sc_accel_mask || (sc->sc_ae_mask & 0x01) == 0) {
379 device_printf(sc->sc_dev, "couldn't find acceleration");
380 goto fail;
381 }
382
383 MPASS(sc->sc_accel_num <= MAX_NUM_ACCEL);
384 MPASS(sc->sc_ae_num <= MAX_NUM_AE);
385
386 /* Determine SKU and capabilities */
387 sc->sc_sku = sc->sc_hw.qhw_get_sku(sc);
388 sc->sc_accel_cap = sc->sc_hw.qhw_get_accel_cap(sc);
389 sc->sc_fw_uof_name = sc->sc_hw.qhw_get_fw_uof_name(sc);
390
391 i = 0;
392 if (sc->sc_hw.qhw_sram_bar_id != NO_PCI_REG) {
393 MPASS(sc->sc_hw.qhw_sram_bar_id == 0);
394 uint32_t fusectl = pci_read_config(dev, FUSECTL_REG, 4);
395 /* Skip SRAM BAR */
396 i = (fusectl & FUSECTL_MASK) ? 1 : 0;
397 }
398 for (bar = 0; bar < PCIR_MAX_BAR_0; bar++) {
399 uint32_t val = pci_read_config(dev, PCIR_BAR(bar), 4);
400 if (val == 0 || !PCI_BAR_MEM(val))
401 continue;
402
403 sc->sc_rid[i] = PCIR_BAR(bar);
404 sc->sc_res[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
405 &sc->sc_rid[i], RF_ACTIVE);
406 if (sc->sc_res[i] == NULL) {
407 device_printf(dev, "couldn't map BAR %d\n", bar);
408 goto fail;
409 }
410
411 sc->sc_csrt[i] = rman_get_bustag(sc->sc_res[i]);
412 sc->sc_csrh[i] = rman_get_bushandle(sc->sc_res[i]);
413
414 i++;
415 if ((val & PCIM_BAR_MEM_TYPE) == PCIM_BAR_MEM_64)
416 bar++;
417 }
418
419 pci_enable_busmaster(dev);
420
421 count = sc->sc_hw.qhw_num_banks + 1;
422 if (pci_msix_count(dev) < count) {
423 device_printf(dev, "insufficient MSI-X vectors (%d vs. %d)\n",
424 pci_msix_count(dev), count);
425 goto fail;
426 }
427 error = pci_alloc_msix(dev, &count);
428 if (error != 0) {
429 device_printf(dev, "failed to allocate MSI-X vectors\n");
430 goto fail;
431 }
432
433 error = qat_init(dev);
434 if (error == 0)
435 return 0;
436
437 fail:
438 qat_detach(dev);
439 return ENXIO;
440 }
441
442 static int
qat_init(device_t dev)443 qat_init(device_t dev)
444 {
445 struct qat_softc *sc = device_get_softc(dev);
446 int error;
447
448 qat_etr_init(sc);
449
450 if (sc->sc_hw.qhw_init_admin_comms != NULL &&
451 (error = sc->sc_hw.qhw_init_admin_comms(sc)) != 0) {
452 device_printf(sc->sc_dev,
453 "Could not initialize admin comms: %d\n", error);
454 return error;
455 }
456
457 if (sc->sc_hw.qhw_init_arb != NULL &&
458 (error = sc->sc_hw.qhw_init_arb(sc)) != 0) {
459 device_printf(sc->sc_dev,
460 "Could not initialize hw arbiter: %d\n", error);
461 return error;
462 }
463
464 error = qat_ae_init(sc);
465 if (error) {
466 device_printf(sc->sc_dev,
467 "Could not initialize Acceleration Engine: %d\n", error);
468 return error;
469 }
470
471 error = qat_aefw_load(sc);
472 if (error) {
473 device_printf(sc->sc_dev,
474 "Could not load firmware: %d\n", error);
475 return error;
476 }
477
478 error = qat_setup_msix_intr(sc);
479 if (error) {
480 device_printf(sc->sc_dev,
481 "Could not setup interrupts: %d\n", error);
482 return error;
483 }
484
485 sc->sc_hw.qhw_enable_intr(sc);
486
487 error = qat_crypto_init(sc);
488 if (error) {
489 device_printf(sc->sc_dev,
490 "Could not initialize service: %d\n", error);
491 return error;
492 }
493
494 if (sc->sc_hw.qhw_enable_error_correction != NULL)
495 sc->sc_hw.qhw_enable_error_correction(sc);
496
497 if (sc->sc_hw.qhw_set_ssm_wdtimer != NULL &&
498 (error = sc->sc_hw.qhw_set_ssm_wdtimer(sc)) != 0) {
499 device_printf(sc->sc_dev,
500 "Could not initialize watchdog timer: %d\n", error);
501 return error;
502 }
503
504 error = qat_start(dev);
505 if (error) {
506 device_printf(sc->sc_dev,
507 "Could not start: %d\n", error);
508 return error;
509 }
510
511 return 0;
512 }
513
514 static int
qat_start(device_t dev)515 qat_start(device_t dev)
516 {
517 struct qat_softc *sc = device_get_softc(dev);
518 int error;
519
520 error = qat_ae_start(sc);
521 if (error)
522 return error;
523
524 if (sc->sc_hw.qhw_send_admin_init != NULL &&
525 (error = sc->sc_hw.qhw_send_admin_init(sc)) != 0) {
526 return error;
527 }
528
529 error = qat_crypto_start(sc);
530 if (error)
531 return error;
532
533 return 0;
534 }
535
536 static int
qat_detach(device_t dev)537 qat_detach(device_t dev)
538 {
539 struct qat_softc *sc;
540 int bar, i;
541
542 sc = device_get_softc(dev);
543
544 qat_crypto_stop(sc);
545 qat_crypto_deinit(sc);
546 qat_aefw_unload(sc);
547
548 if (sc->sc_etr_banks != NULL) {
549 for (i = 0; i < sc->sc_hw.qhw_num_banks; i++) {
550 struct qat_bank *qb = &sc->sc_etr_banks[i];
551
552 if (qb->qb_ih_cookie != NULL)
553 (void)bus_teardown_intr(dev, qb->qb_ih,
554 qb->qb_ih_cookie);
555 if (qb->qb_ih != NULL)
556 (void)bus_release_resource(dev, SYS_RES_IRQ,
557 i + 1, qb->qb_ih);
558 }
559 }
560 if (sc->sc_ih_cookie != NULL) {
561 (void)bus_teardown_intr(dev, sc->sc_ih, sc->sc_ih_cookie);
562 sc->sc_ih_cookie = NULL;
563 }
564 if (sc->sc_ih != NULL) {
565 (void)bus_release_resource(dev, SYS_RES_IRQ,
566 sc->sc_hw.qhw_num_banks + 1, sc->sc_ih);
567 sc->sc_ih = NULL;
568 }
569 pci_release_msi(dev);
570
571 qat_etr_deinit(sc);
572
573 for (bar = 0; bar < MAX_BARS; bar++) {
574 if (sc->sc_res[bar] != NULL) {
575 (void)bus_release_resource(dev, SYS_RES_MEMORY,
576 sc->sc_rid[bar], sc->sc_res[bar]);
577 sc->sc_res[bar] = NULL;
578 }
579 }
580
581 return 0;
582 }
583
584 void *
qat_alloc_mem(size_t size)585 qat_alloc_mem(size_t size)
586 {
587 return (malloc(size, M_QAT, M_WAITOK | M_ZERO));
588 }
589
590 void
qat_free_mem(void * ptr)591 qat_free_mem(void *ptr)
592 {
593 free(ptr, M_QAT);
594 }
595
596 static void
qat_alloc_dmamem_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)597 qat_alloc_dmamem_cb(void *arg, bus_dma_segment_t *segs, int nseg,
598 int error)
599 {
600 struct qat_dmamem *qdm;
601
602 if (error != 0)
603 return;
604
605 KASSERT(nseg == 1, ("%s: nsegs is %d", __func__, nseg));
606 qdm = arg;
607 qdm->qdm_dma_seg = segs[0];
608 }
609
610 int
qat_alloc_dmamem(struct qat_softc * sc,struct qat_dmamem * qdm,int nseg,bus_size_t size,bus_size_t alignment)611 qat_alloc_dmamem(struct qat_softc *sc, struct qat_dmamem *qdm,
612 int nseg, bus_size_t size, bus_size_t alignment)
613 {
614 int error;
615
616 KASSERT(qdm->qdm_dma_vaddr == NULL,
617 ("%s: DMA memory descriptor in use", __func__));
618
619 error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev),
620 alignment, 0, /* alignment, boundary */
621 BUS_SPACE_MAXADDR, /* lowaddr */
622 BUS_SPACE_MAXADDR, /* highaddr */
623 NULL, NULL, /* filter, filterarg */
624 size, /* maxsize */
625 nseg, /* nsegments */
626 size, /* maxsegsize */
627 BUS_DMA_COHERENT, /* flags */
628 NULL, NULL, /* lockfunc, lockarg */
629 &qdm->qdm_dma_tag);
630 if (error != 0)
631 return error;
632
633 error = bus_dmamem_alloc(qdm->qdm_dma_tag, &qdm->qdm_dma_vaddr,
634 BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT,
635 &qdm->qdm_dma_map);
636 if (error != 0) {
637 device_printf(sc->sc_dev,
638 "couldn't allocate dmamem, error = %d\n", error);
639 goto fail_0;
640 }
641
642 error = bus_dmamap_load(qdm->qdm_dma_tag, qdm->qdm_dma_map,
643 qdm->qdm_dma_vaddr, size, qat_alloc_dmamem_cb, qdm,
644 BUS_DMA_NOWAIT);
645 if (error) {
646 device_printf(sc->sc_dev,
647 "couldn't load dmamem map, error = %d\n", error);
648 goto fail_1;
649 }
650
651 return 0;
652 fail_1:
653 bus_dmamem_free(qdm->qdm_dma_tag, qdm->qdm_dma_vaddr, qdm->qdm_dma_map);
654 fail_0:
655 bus_dma_tag_destroy(qdm->qdm_dma_tag);
656 return error;
657 }
658
659 void
qat_free_dmamem(struct qat_softc * sc,struct qat_dmamem * qdm)660 qat_free_dmamem(struct qat_softc *sc, struct qat_dmamem *qdm)
661 {
662 if (qdm->qdm_dma_tag != NULL) {
663 bus_dmamap_unload(qdm->qdm_dma_tag, qdm->qdm_dma_map);
664 bus_dmamem_free(qdm->qdm_dma_tag, qdm->qdm_dma_vaddr,
665 qdm->qdm_dma_map);
666 bus_dma_tag_destroy(qdm->qdm_dma_tag);
667 explicit_bzero(qdm, sizeof(*qdm));
668 }
669 }
670
671 static int
qat_setup_msix_intr(struct qat_softc * sc)672 qat_setup_msix_intr(struct qat_softc *sc)
673 {
674 device_t dev;
675 int error, i, rid;
676
677 dev = sc->sc_dev;
678
679 for (i = 1; i <= sc->sc_hw.qhw_num_banks; i++) {
680 struct qat_bank *qb = &sc->sc_etr_banks[i - 1];
681
682 rid = i;
683 qb->qb_ih = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
684 RF_ACTIVE);
685 if (qb->qb_ih == NULL) {
686 device_printf(dev,
687 "failed to allocate bank intr resource\n");
688 return ENXIO;
689 }
690 error = bus_setup_intr(dev, qb->qb_ih,
691 INTR_TYPE_NET | INTR_MPSAFE, NULL, qat_etr_bank_intr, qb,
692 &qb->qb_ih_cookie);
693 if (error != 0) {
694 device_printf(dev, "failed to set up bank intr\n");
695 return error;
696 }
697 error = bus_bind_intr(dev, qb->qb_ih, (i - 1) % mp_ncpus);
698 if (error != 0)
699 device_printf(dev, "failed to bind intr %d\n", i);
700 }
701
702 rid = i;
703 sc->sc_ih = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
704 RF_ACTIVE);
705 if (sc->sc_ih == NULL)
706 return ENXIO;
707 error = bus_setup_intr(dev, sc->sc_ih, INTR_TYPE_NET | INTR_MPSAFE,
708 NULL, qat_ae_cluster_intr, sc, &sc->sc_ih_cookie);
709
710 return error;
711 }
712
713 static void
qat_etr_init(struct qat_softc * sc)714 qat_etr_init(struct qat_softc *sc)
715 {
716 int i;
717
718 sc->sc_etr_banks = qat_alloc_mem(
719 sizeof(struct qat_bank) * sc->sc_hw.qhw_num_banks);
720
721 for (i = 0; i < sc->sc_hw.qhw_num_banks; i++)
722 qat_etr_bank_init(sc, i);
723
724 if (sc->sc_hw.qhw_num_ap_banks) {
725 sc->sc_etr_ap_banks = qat_alloc_mem(
726 sizeof(struct qat_ap_bank) * sc->sc_hw.qhw_num_ap_banks);
727 qat_etr_ap_bank_init(sc);
728 }
729 }
730
731 static void
qat_etr_deinit(struct qat_softc * sc)732 qat_etr_deinit(struct qat_softc *sc)
733 {
734 int i;
735
736 if (sc->sc_etr_banks != NULL) {
737 for (i = 0; i < sc->sc_hw.qhw_num_banks; i++)
738 qat_etr_bank_deinit(sc, i);
739 qat_free_mem(sc->sc_etr_banks);
740 sc->sc_etr_banks = NULL;
741 }
742 if (sc->sc_etr_ap_banks != NULL) {
743 qat_free_mem(sc->sc_etr_ap_banks);
744 sc->sc_etr_ap_banks = NULL;
745 }
746 }
747
748 static void
qat_etr_bank_init(struct qat_softc * sc,int bank)749 qat_etr_bank_init(struct qat_softc *sc, int bank)
750 {
751 struct qat_bank *qb = &sc->sc_etr_banks[bank];
752 int i, tx_rx_gap = sc->sc_hw.qhw_tx_rx_gap;
753
754 MPASS(bank < sc->sc_hw.qhw_num_banks);
755
756 mtx_init(&qb->qb_bank_mtx, "qb bank", NULL, MTX_DEF);
757
758 qb->qb_sc = sc;
759 qb->qb_bank = bank;
760 qb->qb_coalescing_time = COALESCING_TIME_INTERVAL_DEFAULT;
761
762 /* Clean CSRs for all rings within the bank */
763 for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank; i++) {
764 struct qat_ring *qr = &qb->qb_et_rings[i];
765
766 qat_etr_bank_ring_write_4(sc, bank, i,
767 ETR_RING_CONFIG, 0);
768 qat_etr_bank_ring_base_write_8(sc, bank, i, 0);
769
770 if (sc->sc_hw.qhw_tx_rings_mask & (1 << i)) {
771 qr->qr_inflight = qat_alloc_mem(sizeof(uint32_t));
772 } else if (sc->sc_hw.qhw_tx_rings_mask &
773 (1 << (i - tx_rx_gap))) {
774 /* Share inflight counter with rx and tx */
775 qr->qr_inflight =
776 qb->qb_et_rings[i - tx_rx_gap].qr_inflight;
777 }
778 }
779
780 if (sc->sc_hw.qhw_init_etr_intr != NULL) {
781 sc->sc_hw.qhw_init_etr_intr(sc, bank);
782 } else {
783 /* common code in qat 1.7 */
784 qat_etr_bank_write_4(sc, bank, ETR_INT_REG,
785 ETR_INT_REG_CLEAR_MASK);
786 for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank /
787 ETR_RINGS_PER_INT_SRCSEL; i++) {
788 qat_etr_bank_write_4(sc, bank, ETR_INT_SRCSEL +
789 (i * ETR_INT_SRCSEL_NEXT_OFFSET),
790 ETR_INT_SRCSEL_MASK);
791 }
792 }
793 }
794
795 static void
qat_etr_bank_deinit(struct qat_softc * sc,int bank)796 qat_etr_bank_deinit(struct qat_softc *sc, int bank)
797 {
798 struct qat_bank *qb;
799 struct qat_ring *qr;
800 int i;
801
802 qb = &sc->sc_etr_banks[bank];
803 for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank; i++) {
804 if (sc->sc_hw.qhw_tx_rings_mask & (1 << i)) {
805 qr = &qb->qb_et_rings[i];
806 qat_free_mem(qr->qr_inflight);
807 }
808 }
809 }
810
811 static void
qat_etr_ap_bank_init(struct qat_softc * sc)812 qat_etr_ap_bank_init(struct qat_softc *sc)
813 {
814 int ap_bank;
815
816 for (ap_bank = 0; ap_bank < sc->sc_hw.qhw_num_ap_banks; ap_bank++) {
817 struct qat_ap_bank *qab = &sc->sc_etr_ap_banks[ap_bank];
818
819 qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_MASK,
820 ETR_AP_NF_MASK_INIT);
821 qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_DEST, 0);
822 qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_MASK,
823 ETR_AP_NE_MASK_INIT);
824 qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_DEST, 0);
825
826 memset(qab, 0, sizeof(*qab));
827 }
828 }
829
830 static void
qat_etr_ap_bank_set_ring_mask(uint32_t * ap_mask,uint32_t ring,int set_mask)831 qat_etr_ap_bank_set_ring_mask(uint32_t *ap_mask, uint32_t ring, int set_mask)
832 {
833 if (set_mask)
834 *ap_mask |= (1 << ETR_RING_NUMBER_IN_AP_BANK(ring));
835 else
836 *ap_mask &= ~(1 << ETR_RING_NUMBER_IN_AP_BANK(ring));
837 }
838
839 static void
qat_etr_ap_bank_set_ring_dest(struct qat_softc * sc,uint32_t * ap_dest,uint32_t ring,int set_dest)840 qat_etr_ap_bank_set_ring_dest(struct qat_softc *sc, uint32_t *ap_dest,
841 uint32_t ring, int set_dest)
842 {
843 uint32_t ae_mask;
844 uint8_t mailbox, ae, nae;
845 uint8_t *dest = (uint8_t *)ap_dest;
846
847 mailbox = ETR_RING_AP_MAILBOX_NUMBER(ring);
848
849 nae = 0;
850 ae_mask = sc->sc_ae_mask;
851 for (ae = 0; ae < sc->sc_hw.qhw_num_engines; ae++) {
852 if ((ae_mask & (1 << ae)) == 0)
853 continue;
854
855 if (set_dest) {
856 dest[nae] = __SHIFTIN(ae, ETR_AP_DEST_AE) |
857 __SHIFTIN(mailbox, ETR_AP_DEST_MAILBOX) |
858 ETR_AP_DEST_ENABLE;
859 } else {
860 dest[nae] = 0;
861 }
862 nae++;
863 if (nae == ETR_MAX_AE_PER_MAILBOX)
864 break;
865 }
866 }
867
868 static void
qat_etr_ap_bank_setup_ring(struct qat_softc * sc,struct qat_ring * qr)869 qat_etr_ap_bank_setup_ring(struct qat_softc *sc, struct qat_ring *qr)
870 {
871 struct qat_ap_bank *qab;
872 int ap_bank;
873
874 if (sc->sc_hw.qhw_num_ap_banks == 0)
875 return;
876
877 ap_bank = ETR_RING_AP_BANK_NUMBER(qr->qr_ring);
878 MPASS(ap_bank < sc->sc_hw.qhw_num_ap_banks);
879 qab = &sc->sc_etr_ap_banks[ap_bank];
880
881 if (qr->qr_cb == NULL) {
882 qat_etr_ap_bank_set_ring_mask(&qab->qab_ne_mask, qr->qr_ring, 1);
883 if (!qab->qab_ne_dest) {
884 qat_etr_ap_bank_set_ring_dest(sc, &qab->qab_ne_dest,
885 qr->qr_ring, 1);
886 qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_DEST,
887 qab->qab_ne_dest);
888 }
889 } else {
890 qat_etr_ap_bank_set_ring_mask(&qab->qab_nf_mask, qr->qr_ring, 1);
891 if (!qab->qab_nf_dest) {
892 qat_etr_ap_bank_set_ring_dest(sc, &qab->qab_nf_dest,
893 qr->qr_ring, 1);
894 qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_DEST,
895 qab->qab_nf_dest);
896 }
897 }
898 }
899
900 static int
qat_etr_verify_ring_size(uint32_t msg_size,uint32_t num_msgs)901 qat_etr_verify_ring_size(uint32_t msg_size, uint32_t num_msgs)
902 {
903 int i = QAT_MIN_RING_SIZE;
904
905 for (; i <= QAT_MAX_RING_SIZE; i++)
906 if ((msg_size * num_msgs) == QAT_SIZE_TO_RING_SIZE_IN_BYTES(i))
907 return i;
908
909 return QAT_DEFAULT_RING_SIZE;
910 }
911
912 int
qat_etr_setup_ring(struct qat_softc * sc,int bank,uint32_t ring,uint32_t num_msgs,uint32_t msg_size,qat_cb_t cb,void * cb_arg,const char * name,struct qat_ring ** rqr)913 qat_etr_setup_ring(struct qat_softc *sc, int bank, uint32_t ring,
914 uint32_t num_msgs, uint32_t msg_size, qat_cb_t cb, void *cb_arg,
915 const char *name, struct qat_ring **rqr)
916 {
917 struct qat_bank *qb;
918 struct qat_ring *qr = NULL;
919 int error;
920 uint32_t ring_size_bytes, ring_config;
921 uint64_t ring_base;
922 uint32_t wm_nf = ETR_RING_CONFIG_NEAR_WM_512;
923 uint32_t wm_ne = ETR_RING_CONFIG_NEAR_WM_0;
924
925 MPASS(bank < sc->sc_hw.qhw_num_banks);
926
927 /* Allocate a ring from specified bank */
928 qb = &sc->sc_etr_banks[bank];
929
930 if (ring >= sc->sc_hw.qhw_num_rings_per_bank)
931 return EINVAL;
932 if (qb->qb_allocated_rings & (1 << ring))
933 return ENOENT;
934 qr = &qb->qb_et_rings[ring];
935 qb->qb_allocated_rings |= 1 << ring;
936
937 /* Initialize allocated ring */
938 qr->qr_ring = ring;
939 qr->qr_bank = bank;
940 qr->qr_name = name;
941 qr->qr_ring_id = qr->qr_bank * sc->sc_hw.qhw_num_rings_per_bank + ring;
942 qr->qr_ring_mask = (1 << ring);
943 qr->qr_cb = cb;
944 qr->qr_cb_arg = cb_arg;
945
946 /* Setup the shadow variables */
947 qr->qr_head = 0;
948 qr->qr_tail = 0;
949 qr->qr_msg_size = QAT_BYTES_TO_MSG_SIZE(msg_size);
950 qr->qr_ring_size = qat_etr_verify_ring_size(msg_size, num_msgs);
951
952 /*
953 * To make sure that ring is alligned to ring size allocate
954 * at least 4k and then tell the user it is smaller.
955 */
956 ring_size_bytes = QAT_SIZE_TO_RING_SIZE_IN_BYTES(qr->qr_ring_size);
957 ring_size_bytes = QAT_RING_SIZE_BYTES_MIN(ring_size_bytes);
958 error = qat_alloc_dmamem(sc, &qr->qr_dma, 1, ring_size_bytes,
959 ring_size_bytes);
960 if (error)
961 return error;
962
963 qr->qr_ring_vaddr = qr->qr_dma.qdm_dma_vaddr;
964 qr->qr_ring_paddr = qr->qr_dma.qdm_dma_seg.ds_addr;
965
966 memset(qr->qr_ring_vaddr, QAT_RING_PATTERN,
967 qr->qr_dma.qdm_dma_seg.ds_len);
968
969 bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map,
970 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
971
972 if (cb == NULL) {
973 ring_config = ETR_RING_CONFIG_BUILD(qr->qr_ring_size);
974 } else {
975 ring_config =
976 ETR_RING_CONFIG_BUILD_RESP(qr->qr_ring_size, wm_nf, wm_ne);
977 }
978 qat_etr_bank_ring_write_4(sc, bank, ring, ETR_RING_CONFIG, ring_config);
979
980 ring_base = ETR_RING_BASE_BUILD(qr->qr_ring_paddr, qr->qr_ring_size);
981 qat_etr_bank_ring_base_write_8(sc, bank, ring, ring_base);
982
983 if (sc->sc_hw.qhw_init_arb != NULL)
984 qat_arb_update(sc, qb);
985
986 mtx_init(&qr->qr_ring_mtx, "qr ring", NULL, MTX_DEF);
987
988 qat_etr_ap_bank_setup_ring(sc, qr);
989
990 if (cb != NULL) {
991 uint32_t intr_mask;
992
993 qb->qb_intr_mask |= qr->qr_ring_mask;
994 intr_mask = qb->qb_intr_mask;
995
996 qat_etr_bank_write_4(sc, bank, ETR_INT_COL_EN, intr_mask);
997 qat_etr_bank_write_4(sc, bank, ETR_INT_COL_CTL,
998 ETR_INT_COL_CTL_ENABLE | qb->qb_coalescing_time);
999 }
1000
1001 *rqr = qr;
1002
1003 return 0;
1004 }
1005
1006 static inline u_int
qat_modulo(u_int data,u_int shift)1007 qat_modulo(u_int data, u_int shift)
1008 {
1009 u_int div = data >> shift;
1010 u_int mult = div << shift;
1011 return data - mult;
1012 }
1013
1014 int
qat_etr_put_msg(struct qat_softc * sc,struct qat_ring * qr,uint32_t * msg)1015 qat_etr_put_msg(struct qat_softc *sc, struct qat_ring *qr, uint32_t *msg)
1016 {
1017 uint32_t inflight;
1018 uint32_t *addr;
1019
1020 mtx_lock(&qr->qr_ring_mtx);
1021
1022 inflight = atomic_fetchadd_32(qr->qr_inflight, 1) + 1;
1023 if (inflight > QAT_MAX_INFLIGHTS(qr->qr_ring_size, qr->qr_msg_size)) {
1024 atomic_subtract_32(qr->qr_inflight, 1);
1025 qr->qr_need_wakeup = true;
1026 mtx_unlock(&qr->qr_ring_mtx);
1027 counter_u64_add(sc->sc_ring_full_restarts, 1);
1028 return ERESTART;
1029 }
1030
1031 addr = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_tail);
1032
1033 memcpy(addr, msg, QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size));
1034
1035 bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map,
1036 BUS_DMASYNC_PREWRITE);
1037
1038 qr->qr_tail = qat_modulo(qr->qr_tail +
1039 QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size),
1040 QAT_RING_SIZE_MODULO(qr->qr_ring_size));
1041
1042 qat_etr_bank_ring_write_4(sc, qr->qr_bank, qr->qr_ring,
1043 ETR_RING_TAIL_OFFSET, qr->qr_tail);
1044
1045 mtx_unlock(&qr->qr_ring_mtx);
1046
1047 return 0;
1048 }
1049
1050 static int
qat_etr_ring_intr(struct qat_softc * sc,struct qat_bank * qb,struct qat_ring * qr)1051 qat_etr_ring_intr(struct qat_softc *sc, struct qat_bank *qb,
1052 struct qat_ring *qr)
1053 {
1054 uint32_t *msg, nmsg = 0;
1055 int handled = 0;
1056 bool blocked = false;
1057
1058 mtx_lock(&qr->qr_ring_mtx);
1059
1060 msg = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_head);
1061
1062 bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map,
1063 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1064
1065 while (atomic_load_32(msg) != ETR_RING_EMPTY_ENTRY_SIG) {
1066 atomic_subtract_32(qr->qr_inflight, 1);
1067
1068 if (qr->qr_cb != NULL) {
1069 mtx_unlock(&qr->qr_ring_mtx);
1070 handled |= qr->qr_cb(sc, qr->qr_cb_arg, msg);
1071 mtx_lock(&qr->qr_ring_mtx);
1072 }
1073
1074 atomic_store_32(msg, ETR_RING_EMPTY_ENTRY_SIG);
1075
1076 qr->qr_head = qat_modulo(qr->qr_head +
1077 QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size),
1078 QAT_RING_SIZE_MODULO(qr->qr_ring_size));
1079 nmsg++;
1080
1081 msg = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_head);
1082 }
1083
1084 bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map,
1085 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1086
1087 if (nmsg > 0) {
1088 qat_etr_bank_ring_write_4(sc, qr->qr_bank, qr->qr_ring,
1089 ETR_RING_HEAD_OFFSET, qr->qr_head);
1090 if (qr->qr_need_wakeup) {
1091 blocked = true;
1092 qr->qr_need_wakeup = false;
1093 }
1094 }
1095
1096 mtx_unlock(&qr->qr_ring_mtx);
1097
1098 if (blocked)
1099 crypto_unblock(sc->sc_crypto.qcy_cid, CRYPTO_SYMQ);
1100
1101 return handled;
1102 }
1103
1104 static void
qat_etr_bank_intr(void * arg)1105 qat_etr_bank_intr(void *arg)
1106 {
1107 struct qat_bank *qb = arg;
1108 struct qat_softc *sc = qb->qb_sc;
1109 uint32_t estat;
1110 int i;
1111
1112 mtx_lock(&qb->qb_bank_mtx);
1113
1114 qat_etr_bank_write_4(sc, qb->qb_bank, ETR_INT_COL_CTL, 0);
1115
1116 /* Now handle all the responses */
1117 estat = ~qat_etr_bank_read_4(sc, qb->qb_bank, ETR_E_STAT);
1118 estat &= qb->qb_intr_mask;
1119
1120 qat_etr_bank_write_4(sc, qb->qb_bank, ETR_INT_COL_CTL,
1121 ETR_INT_COL_CTL_ENABLE | qb->qb_coalescing_time);
1122
1123 mtx_unlock(&qb->qb_bank_mtx);
1124
1125 while ((i = ffs(estat)) != 0) {
1126 struct qat_ring *qr = &qb->qb_et_rings[--i];
1127 estat &= ~(1 << i);
1128 (void)qat_etr_ring_intr(sc, qb, qr);
1129 }
1130 }
1131
1132 void
qat_arb_update(struct qat_softc * sc,struct qat_bank * qb)1133 qat_arb_update(struct qat_softc *sc, struct qat_bank *qb)
1134 {
1135
1136 qat_arb_ringsrvarben_write_4(sc, qb->qb_bank,
1137 qb->qb_allocated_rings & 0xff);
1138 }
1139
1140 static struct qat_sym_cookie *
qat_crypto_alloc_sym_cookie(struct qat_crypto_bank * qcb)1141 qat_crypto_alloc_sym_cookie(struct qat_crypto_bank *qcb)
1142 {
1143 struct qat_sym_cookie *qsc;
1144
1145 mtx_lock(&qcb->qcb_bank_mtx);
1146
1147 if (qcb->qcb_symck_free_count == 0) {
1148 mtx_unlock(&qcb->qcb_bank_mtx);
1149 return NULL;
1150 }
1151
1152 qsc = qcb->qcb_symck_free[--qcb->qcb_symck_free_count];
1153
1154 mtx_unlock(&qcb->qcb_bank_mtx);
1155
1156 return qsc;
1157 }
1158
1159 static void
qat_crypto_free_sym_cookie(struct qat_crypto_bank * qcb,struct qat_sym_cookie * qsc)1160 qat_crypto_free_sym_cookie(struct qat_crypto_bank *qcb,
1161 struct qat_sym_cookie *qsc)
1162 {
1163 explicit_bzero(qsc->qsc_iv_buf, EALG_MAX_BLOCK_LEN);
1164 explicit_bzero(qsc->qsc_auth_res, QAT_SYM_HASH_BUFFER_LEN);
1165
1166 mtx_lock(&qcb->qcb_bank_mtx);
1167 qcb->qcb_symck_free[qcb->qcb_symck_free_count++] = qsc;
1168 mtx_unlock(&qcb->qcb_bank_mtx);
1169 }
1170
1171 void
qat_memcpy_htobe64(void * dst,const void * src,size_t len)1172 qat_memcpy_htobe64(void *dst, const void *src, size_t len)
1173 {
1174 uint64_t *dst0 = dst;
1175 const uint64_t *src0 = src;
1176 size_t i;
1177
1178 MPASS(len % sizeof(*dst0) == 0);
1179
1180 for (i = 0; i < len / sizeof(*dst0); i++)
1181 *(dst0 + i) = htobe64(*(src0 + i));
1182 }
1183
1184 void
qat_memcpy_htobe32(void * dst,const void * src,size_t len)1185 qat_memcpy_htobe32(void *dst, const void *src, size_t len)
1186 {
1187 uint32_t *dst0 = dst;
1188 const uint32_t *src0 = src;
1189 size_t i;
1190
1191 MPASS(len % sizeof(*dst0) == 0);
1192
1193 for (i = 0; i < len / sizeof(*dst0); i++)
1194 *(dst0 + i) = htobe32(*(src0 + i));
1195 }
1196
1197 void
qat_memcpy_htobe(void * dst,const void * src,size_t len,uint32_t wordbyte)1198 qat_memcpy_htobe(void *dst, const void *src, size_t len, uint32_t wordbyte)
1199 {
1200 switch (wordbyte) {
1201 case 4:
1202 qat_memcpy_htobe32(dst, src, len);
1203 break;
1204 case 8:
1205 qat_memcpy_htobe64(dst, src, len);
1206 break;
1207 default:
1208 panic("invalid word size %u", wordbyte);
1209 }
1210 }
1211
1212 void
qat_crypto_gmac_precompute(const struct qat_crypto_desc * desc,const uint8_t * key,int klen,const struct qat_sym_hash_def * hash_def,uint8_t * state)1213 qat_crypto_gmac_precompute(const struct qat_crypto_desc *desc,
1214 const uint8_t *key, int klen, const struct qat_sym_hash_def *hash_def,
1215 uint8_t *state)
1216 {
1217 uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)];
1218 char zeros[AES_BLOCK_LEN];
1219 int rounds;
1220
1221 memset(zeros, 0, sizeof(zeros));
1222 rounds = rijndaelKeySetupEnc(ks, key, klen * NBBY);
1223 rijndaelEncrypt(ks, rounds, zeros, state);
1224 explicit_bzero(ks, sizeof(ks));
1225 }
1226
1227 void
qat_crypto_hmac_precompute(const struct qat_crypto_desc * desc,const uint8_t * key,int klen,const struct qat_sym_hash_def * hash_def,uint8_t * state1,uint8_t * state2)1228 qat_crypto_hmac_precompute(const struct qat_crypto_desc *desc,
1229 const uint8_t *key, int klen, const struct qat_sym_hash_def *hash_def,
1230 uint8_t *state1, uint8_t *state2)
1231 {
1232 union authctx ctx;
1233 const struct auth_hash *sah = hash_def->qshd_alg->qshai_sah;
1234 uint32_t state_offset = hash_def->qshd_alg->qshai_state_offset;
1235 uint32_t state_size = hash_def->qshd_alg->qshai_state_size;
1236 uint32_t state_word = hash_def->qshd_alg->qshai_state_word;
1237
1238 hmac_init_ipad(sah, key, klen, &ctx);
1239 qat_memcpy_htobe(state1, (uint8_t *)&ctx + state_offset, state_size,
1240 state_word);
1241 hmac_init_opad(sah, key, klen, &ctx);
1242 qat_memcpy_htobe(state2, (uint8_t *)&ctx + state_offset, state_size,
1243 state_word);
1244 explicit_bzero(&ctx, sizeof(ctx));
1245 }
1246
1247 static enum hw_cipher_algo
qat_aes_cipher_algo(int klen)1248 qat_aes_cipher_algo(int klen)
1249 {
1250 switch (klen) {
1251 case HW_AES_128_KEY_SZ:
1252 return HW_CIPHER_ALGO_AES128;
1253 case HW_AES_192_KEY_SZ:
1254 return HW_CIPHER_ALGO_AES192;
1255 case HW_AES_256_KEY_SZ:
1256 return HW_CIPHER_ALGO_AES256;
1257 default:
1258 panic("invalid key length %d", klen);
1259 }
1260 }
1261
1262 uint16_t
qat_crypto_load_cipher_session(const struct qat_crypto_desc * desc,const struct qat_session * qs)1263 qat_crypto_load_cipher_session(const struct qat_crypto_desc *desc,
1264 const struct qat_session *qs)
1265 {
1266 enum hw_cipher_algo algo;
1267 enum hw_cipher_dir dir;
1268 enum hw_cipher_convert key_convert;
1269 enum hw_cipher_mode mode;
1270
1271 dir = desc->qcd_cipher_dir;
1272 key_convert = HW_CIPHER_NO_CONVERT;
1273 mode = qs->qs_cipher_mode;
1274 switch (mode) {
1275 case HW_CIPHER_CBC_MODE:
1276 case HW_CIPHER_XTS_MODE:
1277 algo = qs->qs_cipher_algo;
1278
1279 /*
1280 * AES decrypt key needs to be reversed.
1281 * Instead of reversing the key at session registration,
1282 * it is instead reversed on-the-fly by setting the KEY_CONVERT
1283 * bit here.
1284 */
1285 if (desc->qcd_cipher_dir == HW_CIPHER_DECRYPT)
1286 key_convert = HW_CIPHER_KEY_CONVERT;
1287 break;
1288 case HW_CIPHER_CTR_MODE:
1289 algo = qs->qs_cipher_algo;
1290 dir = HW_CIPHER_ENCRYPT;
1291 break;
1292 default:
1293 panic("unhandled cipher mode %d", mode);
1294 break;
1295 }
1296
1297 return HW_CIPHER_CONFIG_BUILD(mode, algo, key_convert, dir);
1298 }
1299
1300 uint16_t
qat_crypto_load_auth_session(const struct qat_crypto_desc * desc,const struct qat_session * qs,const struct qat_sym_hash_def ** hash_def)1301 qat_crypto_load_auth_session(const struct qat_crypto_desc *desc,
1302 const struct qat_session *qs, const struct qat_sym_hash_def **hash_def)
1303 {
1304 enum qat_sym_hash_algorithm algo;
1305
1306 switch (qs->qs_auth_algo) {
1307 case HW_AUTH_ALGO_SHA1:
1308 algo = QAT_SYM_HASH_SHA1;
1309 break;
1310 case HW_AUTH_ALGO_SHA256:
1311 algo = QAT_SYM_HASH_SHA256;
1312 break;
1313 case HW_AUTH_ALGO_SHA384:
1314 algo = QAT_SYM_HASH_SHA384;
1315 break;
1316 case HW_AUTH_ALGO_SHA512:
1317 algo = QAT_SYM_HASH_SHA512;
1318 break;
1319 case HW_AUTH_ALGO_GALOIS_128:
1320 algo = QAT_SYM_HASH_AES_GCM;
1321 break;
1322 default:
1323 panic("unhandled auth algorithm %d", qs->qs_auth_algo);
1324 break;
1325 }
1326 *hash_def = &qat_sym_hash_defs[algo];
1327
1328 return HW_AUTH_CONFIG_BUILD(qs->qs_auth_mode,
1329 (*hash_def)->qshd_qat->qshqi_algo_enc,
1330 (*hash_def)->qshd_alg->qshai_digest_len);
1331 }
1332
1333 struct qat_crypto_load_cb_arg {
1334 struct qat_session *qs;
1335 struct qat_sym_cookie *qsc;
1336 struct cryptop *crp;
1337 int error;
1338 };
1339
1340 static int
qat_crypto_populate_buf_list(struct buffer_list_desc * buffers,bus_dma_segment_t * segs,int niseg,int noseg,int skip)1341 qat_crypto_populate_buf_list(struct buffer_list_desc *buffers,
1342 bus_dma_segment_t *segs, int niseg, int noseg, int skip)
1343 {
1344 struct flat_buffer_desc *flatbuf;
1345 bus_addr_t addr;
1346 bus_size_t len;
1347 int iseg, oseg;
1348
1349 for (iseg = 0, oseg = noseg; iseg < niseg && oseg < QAT_MAXSEG;
1350 iseg++) {
1351 addr = segs[iseg].ds_addr;
1352 len = segs[iseg].ds_len;
1353
1354 if (skip > 0) {
1355 if (skip < len) {
1356 addr += skip;
1357 len -= skip;
1358 skip = 0;
1359 } else {
1360 skip -= len;
1361 continue;
1362 }
1363 }
1364
1365 flatbuf = &buffers->flat_bufs[oseg++];
1366 flatbuf->data_len_in_bytes = (uint32_t)len;
1367 flatbuf->phy_buffer = (uint64_t)addr;
1368 }
1369 buffers->num_buffers = oseg;
1370 return iseg < niseg ? E2BIG : 0;
1371 }
1372
1373 static void
qat_crypto_load_aadbuf_cb(void * _arg,bus_dma_segment_t * segs,int nseg,int error)1374 qat_crypto_load_aadbuf_cb(void *_arg, bus_dma_segment_t *segs, int nseg,
1375 int error)
1376 {
1377 struct qat_crypto_load_cb_arg *arg;
1378 struct qat_sym_cookie *qsc;
1379
1380 arg = _arg;
1381 if (error != 0) {
1382 arg->error = error;
1383 return;
1384 }
1385
1386 qsc = arg->qsc;
1387 arg->error = qat_crypto_populate_buf_list(&qsc->qsc_buf_list, segs,
1388 nseg, 0, 0);
1389 }
1390
1391 static void
qat_crypto_load_buf_cb(void * _arg,bus_dma_segment_t * segs,int nseg,int error)1392 qat_crypto_load_buf_cb(void *_arg, bus_dma_segment_t *segs, int nseg,
1393 int error)
1394 {
1395 struct cryptop *crp;
1396 struct qat_crypto_load_cb_arg *arg;
1397 struct qat_session *qs;
1398 struct qat_sym_cookie *qsc;
1399 int noseg, skip;
1400
1401 arg = _arg;
1402 if (error != 0) {
1403 arg->error = error;
1404 return;
1405 }
1406
1407 crp = arg->crp;
1408 qs = arg->qs;
1409 qsc = arg->qsc;
1410
1411 if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) {
1412 /* AAD was handled in qat_crypto_load(). */
1413 skip = crp->crp_payload_start;
1414 noseg = 0;
1415 } else if (crp->crp_aad == NULL && crp->crp_aad_length > 0) {
1416 skip = crp->crp_aad_start;
1417 noseg = 0;
1418 } else {
1419 skip = crp->crp_payload_start;
1420 noseg = crp->crp_aad == NULL ?
1421 0 : qsc->qsc_buf_list.num_buffers;
1422 }
1423 arg->error = qat_crypto_populate_buf_list(&qsc->qsc_buf_list, segs,
1424 nseg, noseg, skip);
1425 }
1426
1427 static void
qat_crypto_load_obuf_cb(void * _arg,bus_dma_segment_t * segs,int nseg,int error)1428 qat_crypto_load_obuf_cb(void *_arg, bus_dma_segment_t *segs, int nseg,
1429 int error)
1430 {
1431 struct buffer_list_desc *ibufs, *obufs;
1432 struct flat_buffer_desc *ibuf, *obuf;
1433 struct cryptop *crp;
1434 struct qat_crypto_load_cb_arg *arg;
1435 struct qat_session *qs;
1436 struct qat_sym_cookie *qsc;
1437 int buflen, osegs, tocopy;
1438
1439 arg = _arg;
1440 if (error != 0) {
1441 arg->error = error;
1442 return;
1443 }
1444
1445 crp = arg->crp;
1446 qs = arg->qs;
1447 qsc = arg->qsc;
1448
1449 /*
1450 * The payload must start at the same offset in the output SG list as in
1451 * the input SG list. Copy over SG entries from the input corresponding
1452 * to the AAD buffer.
1453 */
1454 osegs = 0;
1455 if (qs->qs_auth_algo != HW_AUTH_ALGO_GALOIS_128 &&
1456 crp->crp_aad_length > 0) {
1457 tocopy = crp->crp_aad == NULL ?
1458 crp->crp_payload_start - crp->crp_aad_start :
1459 crp->crp_aad_length;
1460
1461 ibufs = &qsc->qsc_buf_list;
1462 obufs = &qsc->qsc_obuf_list;
1463 for (; osegs < ibufs->num_buffers && tocopy > 0; osegs++) {
1464 ibuf = &ibufs->flat_bufs[osegs];
1465 obuf = &obufs->flat_bufs[osegs];
1466
1467 obuf->phy_buffer = ibuf->phy_buffer;
1468 buflen = imin(ibuf->data_len_in_bytes, tocopy);
1469 obuf->data_len_in_bytes = buflen;
1470 tocopy -= buflen;
1471 }
1472 }
1473
1474 arg->error = qat_crypto_populate_buf_list(&qsc->qsc_obuf_list, segs,
1475 nseg, osegs, crp->crp_payload_output_start);
1476 }
1477
1478 static int
qat_crypto_load(struct qat_session * qs,struct qat_sym_cookie * qsc,struct qat_crypto_desc const * desc,struct cryptop * crp)1479 qat_crypto_load(struct qat_session *qs, struct qat_sym_cookie *qsc,
1480 struct qat_crypto_desc const *desc, struct cryptop *crp)
1481 {
1482 struct qat_crypto_load_cb_arg arg;
1483 int error;
1484
1485 crypto_read_iv(crp, qsc->qsc_iv_buf);
1486
1487 arg.crp = crp;
1488 arg.qs = qs;
1489 arg.qsc = qsc;
1490 arg.error = 0;
1491
1492 error = 0;
1493 if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128 &&
1494 crp->crp_aad_length > 0) {
1495 /*
1496 * The firmware expects AAD to be in a contiguous buffer and
1497 * padded to a multiple of 16 bytes. To satisfy these
1498 * constraints we bounce the AAD into a per-request buffer.
1499 * There is a small limit on the AAD size so this is not too
1500 * onerous.
1501 */
1502 memset(qsc->qsc_gcm_aad, 0, QAT_GCM_AAD_SIZE_MAX);
1503 if (crp->crp_aad == NULL) {
1504 crypto_copydata(crp, crp->crp_aad_start,
1505 crp->crp_aad_length, qsc->qsc_gcm_aad);
1506 } else {
1507 memcpy(qsc->qsc_gcm_aad, crp->crp_aad,
1508 crp->crp_aad_length);
1509 }
1510 } else if (crp->crp_aad != NULL) {
1511 error = bus_dmamap_load(
1512 qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dma_tag,
1513 qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dmamap,
1514 crp->crp_aad, crp->crp_aad_length,
1515 qat_crypto_load_aadbuf_cb, &arg, BUS_DMA_NOWAIT);
1516 if (error == 0)
1517 error = arg.error;
1518 }
1519 if (error == 0) {
1520 error = bus_dmamap_load_crp_buffer(
1521 qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dma_tag,
1522 qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dmamap,
1523 &crp->crp_buf, qat_crypto_load_buf_cb, &arg,
1524 BUS_DMA_NOWAIT);
1525 if (error == 0)
1526 error = arg.error;
1527 }
1528 if (error == 0 && CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
1529 error = bus_dmamap_load_crp_buffer(
1530 qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dma_tag,
1531 qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dmamap,
1532 &crp->crp_obuf, qat_crypto_load_obuf_cb, &arg,
1533 BUS_DMA_NOWAIT);
1534 if (error == 0)
1535 error = arg.error;
1536 }
1537 return error;
1538 }
1539
1540 static inline struct qat_crypto_bank *
qat_crypto_select_bank(struct qat_crypto * qcy)1541 qat_crypto_select_bank(struct qat_crypto *qcy)
1542 {
1543 u_int cpuid = PCPU_GET(cpuid);
1544
1545 return &qcy->qcy_banks[cpuid % qcy->qcy_num_banks];
1546 }
1547
1548 static int
qat_crypto_setup_ring(struct qat_softc * sc,struct qat_crypto_bank * qcb)1549 qat_crypto_setup_ring(struct qat_softc *sc, struct qat_crypto_bank *qcb)
1550 {
1551 char *name;
1552 int bank, curname, error, i, j;
1553
1554 bank = qcb->qcb_bank;
1555 curname = 0;
1556
1557 name = qcb->qcb_ring_names[curname++];
1558 snprintf(name, QAT_RING_NAME_SIZE, "bank%d sym_tx", bank);
1559 error = qat_etr_setup_ring(sc, qcb->qcb_bank,
1560 sc->sc_hw.qhw_ring_sym_tx, QAT_NSYMREQ, sc->sc_hw.qhw_fw_req_size,
1561 NULL, NULL, name, &qcb->qcb_sym_tx);
1562 if (error)
1563 return error;
1564
1565 name = qcb->qcb_ring_names[curname++];
1566 snprintf(name, QAT_RING_NAME_SIZE, "bank%d sym_rx", bank);
1567 error = qat_etr_setup_ring(sc, qcb->qcb_bank,
1568 sc->sc_hw.qhw_ring_sym_rx, QAT_NSYMREQ, sc->sc_hw.qhw_fw_resp_size,
1569 qat_crypto_sym_rxintr, qcb, name, &qcb->qcb_sym_rx);
1570 if (error)
1571 return error;
1572
1573 for (i = 0; i < QAT_NSYMCOOKIE; i++) {
1574 struct qat_dmamem *qdm = &qcb->qcb_symck_dmamems[i];
1575 struct qat_sym_cookie *qsc;
1576
1577 error = qat_alloc_dmamem(sc, qdm, 1,
1578 sizeof(struct qat_sym_cookie), QAT_OPTIMAL_ALIGN);
1579 if (error)
1580 return error;
1581
1582 qsc = qdm->qdm_dma_vaddr;
1583 qsc->qsc_self_dmamap = qdm->qdm_dma_map;
1584 qsc->qsc_self_dma_tag = qdm->qdm_dma_tag;
1585 qsc->qsc_bulk_req_params_buf_paddr =
1586 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
1587 qsc_bulk_cookie.qsbc_req_params_buf);
1588 qsc->qsc_buffer_list_desc_paddr =
1589 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
1590 qsc_buf_list);
1591 qsc->qsc_obuffer_list_desc_paddr =
1592 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
1593 qsc_obuf_list);
1594 qsc->qsc_obuffer_list_desc_paddr =
1595 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
1596 qsc_obuf_list);
1597 qsc->qsc_iv_buf_paddr =
1598 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
1599 qsc_iv_buf);
1600 qsc->qsc_auth_res_paddr =
1601 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
1602 qsc_auth_res);
1603 qsc->qsc_gcm_aad_paddr =
1604 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
1605 qsc_gcm_aad);
1606 qsc->qsc_content_desc_paddr =
1607 qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie,
1608 qsc_content_desc);
1609 qcb->qcb_symck_free[i] = qsc;
1610 qcb->qcb_symck_free_count++;
1611
1612 for (j = 0; j < QAT_SYM_DMA_COUNT; j++) {
1613 error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev),
1614 1, 0, /* alignment, boundary */
1615 BUS_SPACE_MAXADDR, /* lowaddr */
1616 BUS_SPACE_MAXADDR, /* highaddr */
1617 NULL, NULL, /* filter, filterarg */
1618 QAT_MAXLEN, /* maxsize */
1619 QAT_MAXSEG, /* nsegments */
1620 QAT_MAXLEN, /* maxsegsize */
1621 BUS_DMA_COHERENT, /* flags */
1622 NULL, NULL, /* lockfunc, lockarg */
1623 &qsc->qsc_dma[j].qsd_dma_tag);
1624 if (error != 0)
1625 return error;
1626 error = bus_dmamap_create(qsc->qsc_dma[j].qsd_dma_tag,
1627 BUS_DMA_COHERENT, &qsc->qsc_dma[j].qsd_dmamap);
1628 if (error != 0)
1629 return error;
1630 }
1631 }
1632
1633 return 0;
1634 }
1635
1636 static int
qat_crypto_bank_init(struct qat_softc * sc,struct qat_crypto_bank * qcb)1637 qat_crypto_bank_init(struct qat_softc *sc, struct qat_crypto_bank *qcb)
1638 {
1639 mtx_init(&qcb->qcb_bank_mtx, "qcb bank", NULL, MTX_DEF);
1640
1641 return qat_crypto_setup_ring(sc, qcb);
1642 }
1643
1644 static void
qat_crypto_bank_deinit(struct qat_softc * sc,struct qat_crypto_bank * qcb)1645 qat_crypto_bank_deinit(struct qat_softc *sc, struct qat_crypto_bank *qcb)
1646 {
1647 struct qat_dmamem *qdm;
1648 struct qat_sym_cookie *qsc;
1649 int i, j;
1650
1651 for (i = 0; i < QAT_NSYMCOOKIE; i++) {
1652 qdm = &qcb->qcb_symck_dmamems[i];
1653 qsc = qcb->qcb_symck_free[i];
1654 for (j = 0; j < QAT_SYM_DMA_COUNT; j++) {
1655 bus_dmamap_destroy(qsc->qsc_dma[j].qsd_dma_tag,
1656 qsc->qsc_dma[j].qsd_dmamap);
1657 bus_dma_tag_destroy(qsc->qsc_dma[j].qsd_dma_tag);
1658 }
1659 qat_free_dmamem(sc, qdm);
1660 }
1661 qat_free_dmamem(sc, &qcb->qcb_sym_tx->qr_dma);
1662 qat_free_dmamem(sc, &qcb->qcb_sym_rx->qr_dma);
1663
1664 mtx_destroy(&qcb->qcb_bank_mtx);
1665 }
1666
1667 static int
qat_crypto_init(struct qat_softc * sc)1668 qat_crypto_init(struct qat_softc *sc)
1669 {
1670 struct qat_crypto *qcy = &sc->sc_crypto;
1671 struct sysctl_ctx_list *ctx;
1672 struct sysctl_oid *oid;
1673 struct sysctl_oid_list *children;
1674 int bank, error, num_banks;
1675
1676 qcy->qcy_sc = sc;
1677
1678 if (sc->sc_hw.qhw_init_arb != NULL)
1679 num_banks = imin(mp_ncpus, sc->sc_hw.qhw_num_banks);
1680 else
1681 num_banks = sc->sc_ae_num;
1682
1683 qcy->qcy_num_banks = num_banks;
1684
1685 qcy->qcy_banks =
1686 qat_alloc_mem(sizeof(struct qat_crypto_bank) * num_banks);
1687
1688 for (bank = 0; bank < num_banks; bank++) {
1689 struct qat_crypto_bank *qcb = &qcy->qcy_banks[bank];
1690 qcb->qcb_bank = bank;
1691 error = qat_crypto_bank_init(sc, qcb);
1692 if (error)
1693 return error;
1694 }
1695
1696 mtx_init(&qcy->qcy_crypto_mtx, "qcy crypto", NULL, MTX_DEF);
1697
1698 ctx = device_get_sysctl_ctx(sc->sc_dev);
1699 oid = device_get_sysctl_tree(sc->sc_dev);
1700 children = SYSCTL_CHILDREN(oid);
1701 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats",
1702 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "statistics");
1703 children = SYSCTL_CHILDREN(oid);
1704
1705 sc->sc_gcm_aad_restarts = counter_u64_alloc(M_WAITOK);
1706 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_aad_restarts",
1707 CTLFLAG_RD, &sc->sc_gcm_aad_restarts,
1708 "GCM requests deferred due to AAD size change");
1709 sc->sc_gcm_aad_updates = counter_u64_alloc(M_WAITOK);
1710 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_aad_updates",
1711 CTLFLAG_RD, &sc->sc_gcm_aad_updates,
1712 "GCM requests that required session state update");
1713 sc->sc_ring_full_restarts = counter_u64_alloc(M_WAITOK);
1714 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ring_full",
1715 CTLFLAG_RD, &sc->sc_ring_full_restarts,
1716 "Requests deferred due to in-flight max reached");
1717 sc->sc_sym_alloc_failures = counter_u64_alloc(M_WAITOK);
1718 SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sym_alloc_failures",
1719 CTLFLAG_RD, &sc->sc_sym_alloc_failures,
1720 "Request allocation failures");
1721
1722 return 0;
1723 }
1724
1725 static void
qat_crypto_deinit(struct qat_softc * sc)1726 qat_crypto_deinit(struct qat_softc *sc)
1727 {
1728 struct qat_crypto *qcy = &sc->sc_crypto;
1729 struct qat_crypto_bank *qcb;
1730 int bank;
1731
1732 counter_u64_free(sc->sc_sym_alloc_failures);
1733 counter_u64_free(sc->sc_ring_full_restarts);
1734 counter_u64_free(sc->sc_gcm_aad_updates);
1735 counter_u64_free(sc->sc_gcm_aad_restarts);
1736
1737 if (qcy->qcy_banks != NULL) {
1738 for (bank = 0; bank < qcy->qcy_num_banks; bank++) {
1739 qcb = &qcy->qcy_banks[bank];
1740 qat_crypto_bank_deinit(sc, qcb);
1741 }
1742 qat_free_mem(qcy->qcy_banks);
1743 mtx_destroy(&qcy->qcy_crypto_mtx);
1744 }
1745 }
1746
1747 static int
qat_crypto_start(struct qat_softc * sc)1748 qat_crypto_start(struct qat_softc *sc)
1749 {
1750 struct qat_crypto *qcy;
1751
1752 qcy = &sc->sc_crypto;
1753 qcy->qcy_cid = crypto_get_driverid(sc->sc_dev,
1754 sizeof(struct qat_session), CRYPTOCAP_F_HARDWARE);
1755 if (qcy->qcy_cid < 0) {
1756 device_printf(sc->sc_dev,
1757 "could not get opencrypto driver id\n");
1758 return ENOENT;
1759 }
1760
1761 return 0;
1762 }
1763
1764 static void
qat_crypto_stop(struct qat_softc * sc)1765 qat_crypto_stop(struct qat_softc *sc)
1766 {
1767 struct qat_crypto *qcy;
1768
1769 qcy = &sc->sc_crypto;
1770 if (qcy->qcy_cid >= 0)
1771 (void)crypto_unregister_all(qcy->qcy_cid);
1772 }
1773
1774 static void
qat_crypto_sym_dma_unload(struct qat_sym_cookie * qsc,enum qat_sym_dma i)1775 qat_crypto_sym_dma_unload(struct qat_sym_cookie *qsc, enum qat_sym_dma i)
1776 {
1777 bus_dmamap_sync(qsc->qsc_dma[i].qsd_dma_tag, qsc->qsc_dma[i].qsd_dmamap,
1778 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1779 bus_dmamap_unload(qsc->qsc_dma[i].qsd_dma_tag,
1780 qsc->qsc_dma[i].qsd_dmamap);
1781 }
1782
1783 static int
qat_crypto_sym_rxintr(struct qat_softc * sc,void * arg,void * msg)1784 qat_crypto_sym_rxintr(struct qat_softc *sc, void *arg, void *msg)
1785 {
1786 char icv[QAT_SYM_HASH_BUFFER_LEN];
1787 struct qat_crypto_bank *qcb = arg;
1788 struct qat_crypto *qcy;
1789 struct qat_session *qs;
1790 struct qat_sym_cookie *qsc;
1791 struct qat_sym_bulk_cookie *qsbc;
1792 struct cryptop *crp;
1793 int error;
1794 uint16_t auth_sz;
1795 bool blocked;
1796
1797 qsc = *(void **)((uintptr_t)msg + sc->sc_hw.qhw_crypto_opaque_offset);
1798
1799 qsbc = &qsc->qsc_bulk_cookie;
1800 qcy = qsbc->qsbc_crypto;
1801 qs = qsbc->qsbc_session;
1802 crp = qsbc->qsbc_cb_tag;
1803
1804 bus_dmamap_sync(qsc->qsc_self_dma_tag, qsc->qsc_self_dmamap,
1805 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1806
1807 if (crp->crp_aad != NULL)
1808 qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_AADBUF);
1809 qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_BUF);
1810 if (CRYPTO_HAS_OUTPUT_BUFFER(crp))
1811 qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_OBUF);
1812
1813 error = 0;
1814 if ((auth_sz = qs->qs_auth_mlen) != 0) {
1815 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) {
1816 crypto_copydata(crp, crp->crp_digest_start,
1817 auth_sz, icv);
1818 if (timingsafe_bcmp(icv, qsc->qsc_auth_res,
1819 auth_sz) != 0) {
1820 error = EBADMSG;
1821 }
1822 } else {
1823 crypto_copyback(crp, crp->crp_digest_start,
1824 auth_sz, qsc->qsc_auth_res);
1825 }
1826 }
1827
1828 qat_crypto_free_sym_cookie(qcb, qsc);
1829
1830 blocked = false;
1831 mtx_lock(&qs->qs_session_mtx);
1832 MPASS(qs->qs_status & QAT_SESSION_STATUS_ACTIVE);
1833 qs->qs_inflight--;
1834 if (__predict_false(qs->qs_need_wakeup && qs->qs_inflight == 0)) {
1835 blocked = true;
1836 qs->qs_need_wakeup = false;
1837 }
1838 mtx_unlock(&qs->qs_session_mtx);
1839
1840 crp->crp_etype = error;
1841 crypto_done(crp);
1842
1843 if (blocked)
1844 crypto_unblock(qcy->qcy_cid, CRYPTO_SYMQ);
1845
1846 return 1;
1847 }
1848
1849 static int
qat_probesession(device_t dev,const struct crypto_session_params * csp)1850 qat_probesession(device_t dev, const struct crypto_session_params *csp)
1851 {
1852 if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) !=
1853 0)
1854 return EINVAL;
1855
1856 if (csp->csp_cipher_alg == CRYPTO_AES_XTS &&
1857 qat_lookup(dev)->qatp_chip == QAT_CHIP_C2XXX) {
1858 /*
1859 * AES-XTS is not supported by the NanoQAT.
1860 */
1861 return EINVAL;
1862 }
1863
1864 switch (csp->csp_mode) {
1865 case CSP_MODE_CIPHER:
1866 switch (csp->csp_cipher_alg) {
1867 case CRYPTO_AES_CBC:
1868 case CRYPTO_AES_ICM:
1869 if (csp->csp_ivlen != AES_BLOCK_LEN)
1870 return EINVAL;
1871 break;
1872 case CRYPTO_AES_XTS:
1873 if (csp->csp_ivlen != AES_XTS_IV_LEN)
1874 return EINVAL;
1875 break;
1876 default:
1877 return EINVAL;
1878 }
1879 break;
1880 case CSP_MODE_DIGEST:
1881 switch (csp->csp_auth_alg) {
1882 case CRYPTO_SHA1:
1883 case CRYPTO_SHA1_HMAC:
1884 case CRYPTO_SHA2_256:
1885 case CRYPTO_SHA2_256_HMAC:
1886 case CRYPTO_SHA2_384:
1887 case CRYPTO_SHA2_384_HMAC:
1888 case CRYPTO_SHA2_512:
1889 case CRYPTO_SHA2_512_HMAC:
1890 break;
1891 case CRYPTO_AES_NIST_GMAC:
1892 if (csp->csp_ivlen != AES_GCM_IV_LEN)
1893 return EINVAL;
1894 break;
1895 default:
1896 return EINVAL;
1897 }
1898 break;
1899 case CSP_MODE_AEAD:
1900 switch (csp->csp_cipher_alg) {
1901 case CRYPTO_AES_NIST_GCM_16:
1902 break;
1903 default:
1904 return EINVAL;
1905 }
1906 break;
1907 case CSP_MODE_ETA:
1908 switch (csp->csp_auth_alg) {
1909 case CRYPTO_SHA1_HMAC:
1910 case CRYPTO_SHA2_256_HMAC:
1911 case CRYPTO_SHA2_384_HMAC:
1912 case CRYPTO_SHA2_512_HMAC:
1913 switch (csp->csp_cipher_alg) {
1914 case CRYPTO_AES_CBC:
1915 case CRYPTO_AES_ICM:
1916 if (csp->csp_ivlen != AES_BLOCK_LEN)
1917 return EINVAL;
1918 break;
1919 case CRYPTO_AES_XTS:
1920 if (csp->csp_ivlen != AES_XTS_IV_LEN)
1921 return EINVAL;
1922 break;
1923 default:
1924 return EINVAL;
1925 }
1926 break;
1927 default:
1928 return EINVAL;
1929 }
1930 break;
1931 default:
1932 return EINVAL;
1933 }
1934
1935 return CRYPTODEV_PROBE_HARDWARE;
1936 }
1937
1938 static int
qat_newsession(device_t dev,crypto_session_t cses,const struct crypto_session_params * csp)1939 qat_newsession(device_t dev, crypto_session_t cses,
1940 const struct crypto_session_params *csp)
1941 {
1942 struct qat_crypto *qcy;
1943 struct qat_dmamem *qdm;
1944 struct qat_session *qs;
1945 struct qat_softc *sc;
1946 struct qat_crypto_desc *ddesc, *edesc;
1947 int error, slices;
1948
1949 sc = device_get_softc(dev);
1950 qs = crypto_get_driver_session(cses);
1951 qcy = &sc->sc_crypto;
1952
1953 qdm = &qs->qs_desc_mem;
1954 error = qat_alloc_dmamem(sc, qdm, QAT_MAXSEG,
1955 sizeof(struct qat_crypto_desc) * 2, QAT_OPTIMAL_ALIGN);
1956 if (error != 0)
1957 return error;
1958
1959 mtx_init(&qs->qs_session_mtx, "qs session", NULL, MTX_DEF);
1960 qs->qs_aad_length = -1;
1961
1962 qs->qs_dec_desc = ddesc = qdm->qdm_dma_vaddr;
1963 qs->qs_enc_desc = edesc = ddesc + 1;
1964
1965 ddesc->qcd_desc_paddr = qdm->qdm_dma_seg.ds_addr;
1966 ddesc->qcd_hash_state_paddr = ddesc->qcd_desc_paddr +
1967 offsetof(struct qat_crypto_desc, qcd_hash_state_prefix_buf);
1968 edesc->qcd_desc_paddr = qdm->qdm_dma_seg.ds_addr +
1969 sizeof(struct qat_crypto_desc);
1970 edesc->qcd_hash_state_paddr = edesc->qcd_desc_paddr +
1971 offsetof(struct qat_crypto_desc, qcd_hash_state_prefix_buf);
1972
1973 qs->qs_status = QAT_SESSION_STATUS_ACTIVE;
1974 qs->qs_inflight = 0;
1975
1976 qs->qs_cipher_key = csp->csp_cipher_key;
1977 qs->qs_cipher_klen = csp->csp_cipher_klen;
1978 qs->qs_auth_key = csp->csp_auth_key;
1979 qs->qs_auth_klen = csp->csp_auth_klen;
1980
1981 switch (csp->csp_cipher_alg) {
1982 case CRYPTO_AES_CBC:
1983 qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen);
1984 qs->qs_cipher_mode = HW_CIPHER_CBC_MODE;
1985 break;
1986 case CRYPTO_AES_ICM:
1987 qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen);
1988 qs->qs_cipher_mode = HW_CIPHER_CTR_MODE;
1989 break;
1990 case CRYPTO_AES_XTS:
1991 qs->qs_cipher_algo =
1992 qat_aes_cipher_algo(csp->csp_cipher_klen / 2);
1993 qs->qs_cipher_mode = HW_CIPHER_XTS_MODE;
1994 break;
1995 case CRYPTO_AES_NIST_GCM_16:
1996 qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen);
1997 qs->qs_cipher_mode = HW_CIPHER_CTR_MODE;
1998 qs->qs_auth_algo = HW_AUTH_ALGO_GALOIS_128;
1999 qs->qs_auth_mode = HW_AUTH_MODE1;
2000 break;
2001 case 0:
2002 break;
2003 default:
2004 panic("%s: unhandled cipher algorithm %d", __func__,
2005 csp->csp_cipher_alg);
2006 }
2007
2008 switch (csp->csp_auth_alg) {
2009 case CRYPTO_SHA1_HMAC:
2010 qs->qs_auth_algo = HW_AUTH_ALGO_SHA1;
2011 qs->qs_auth_mode = HW_AUTH_MODE1;
2012 break;
2013 case CRYPTO_SHA1:
2014 qs->qs_auth_algo = HW_AUTH_ALGO_SHA1;
2015 qs->qs_auth_mode = HW_AUTH_MODE0;
2016 break;
2017 case CRYPTO_SHA2_256_HMAC:
2018 qs->qs_auth_algo = HW_AUTH_ALGO_SHA256;
2019 qs->qs_auth_mode = HW_AUTH_MODE1;
2020 break;
2021 case CRYPTO_SHA2_256:
2022 qs->qs_auth_algo = HW_AUTH_ALGO_SHA256;
2023 qs->qs_auth_mode = HW_AUTH_MODE0;
2024 break;
2025 case CRYPTO_SHA2_384_HMAC:
2026 qs->qs_auth_algo = HW_AUTH_ALGO_SHA384;
2027 qs->qs_auth_mode = HW_AUTH_MODE1;
2028 break;
2029 case CRYPTO_SHA2_384:
2030 qs->qs_auth_algo = HW_AUTH_ALGO_SHA384;
2031 qs->qs_auth_mode = HW_AUTH_MODE0;
2032 break;
2033 case CRYPTO_SHA2_512_HMAC:
2034 qs->qs_auth_algo = HW_AUTH_ALGO_SHA512;
2035 qs->qs_auth_mode = HW_AUTH_MODE1;
2036 break;
2037 case CRYPTO_SHA2_512:
2038 qs->qs_auth_algo = HW_AUTH_ALGO_SHA512;
2039 qs->qs_auth_mode = HW_AUTH_MODE0;
2040 break;
2041 case CRYPTO_AES_NIST_GMAC:
2042 qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_auth_klen);
2043 qs->qs_cipher_mode = HW_CIPHER_CTR_MODE;
2044 qs->qs_auth_algo = HW_AUTH_ALGO_GALOIS_128;
2045 qs->qs_auth_mode = HW_AUTH_MODE1;
2046
2047 qs->qs_cipher_key = qs->qs_auth_key;
2048 qs->qs_cipher_klen = qs->qs_auth_klen;
2049 break;
2050 case 0:
2051 break;
2052 default:
2053 panic("%s: unhandled auth algorithm %d", __func__,
2054 csp->csp_auth_alg);
2055 }
2056
2057 slices = 0;
2058 switch (csp->csp_mode) {
2059 case CSP_MODE_AEAD:
2060 case CSP_MODE_ETA:
2061 /* auth then decrypt */
2062 ddesc->qcd_slices[0] = FW_SLICE_AUTH;
2063 ddesc->qcd_slices[1] = FW_SLICE_CIPHER;
2064 ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT;
2065 ddesc->qcd_cmd_id = FW_LA_CMD_HASH_CIPHER;
2066 /* encrypt then auth */
2067 edesc->qcd_slices[0] = FW_SLICE_CIPHER;
2068 edesc->qcd_slices[1] = FW_SLICE_AUTH;
2069 edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT;
2070 edesc->qcd_cmd_id = FW_LA_CMD_CIPHER_HASH;
2071 slices = 2;
2072 break;
2073 case CSP_MODE_CIPHER:
2074 /* decrypt */
2075 ddesc->qcd_slices[0] = FW_SLICE_CIPHER;
2076 ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT;
2077 ddesc->qcd_cmd_id = FW_LA_CMD_CIPHER;
2078 /* encrypt */
2079 edesc->qcd_slices[0] = FW_SLICE_CIPHER;
2080 edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT;
2081 edesc->qcd_cmd_id = FW_LA_CMD_CIPHER;
2082 slices = 1;
2083 break;
2084 case CSP_MODE_DIGEST:
2085 if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) {
2086 /* auth then decrypt */
2087 ddesc->qcd_slices[0] = FW_SLICE_AUTH;
2088 ddesc->qcd_slices[1] = FW_SLICE_CIPHER;
2089 ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT;
2090 ddesc->qcd_cmd_id = FW_LA_CMD_HASH_CIPHER;
2091 /* encrypt then auth */
2092 edesc->qcd_slices[0] = FW_SLICE_CIPHER;
2093 edesc->qcd_slices[1] = FW_SLICE_AUTH;
2094 edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT;
2095 edesc->qcd_cmd_id = FW_LA_CMD_CIPHER_HASH;
2096 slices = 2;
2097 } else {
2098 ddesc->qcd_slices[0] = FW_SLICE_AUTH;
2099 ddesc->qcd_cmd_id = FW_LA_CMD_AUTH;
2100 edesc->qcd_slices[0] = FW_SLICE_AUTH;
2101 edesc->qcd_cmd_id = FW_LA_CMD_AUTH;
2102 slices = 1;
2103 }
2104 break;
2105 default:
2106 panic("%s: unhandled crypto algorithm %d, %d", __func__,
2107 csp->csp_cipher_alg, csp->csp_auth_alg);
2108 }
2109 ddesc->qcd_slices[slices] = FW_SLICE_DRAM_WR;
2110 edesc->qcd_slices[slices] = FW_SLICE_DRAM_WR;
2111
2112 qcy->qcy_sc->sc_hw.qhw_crypto_setup_desc(qcy, qs, ddesc);
2113 qcy->qcy_sc->sc_hw.qhw_crypto_setup_desc(qcy, qs, edesc);
2114
2115 if (csp->csp_auth_mlen != 0)
2116 qs->qs_auth_mlen = csp->csp_auth_mlen;
2117 else
2118 qs->qs_auth_mlen = edesc->qcd_auth_sz;
2119
2120 /* Compute the GMAC by specifying a null cipher payload. */
2121 if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC)
2122 ddesc->qcd_cmd_id = edesc->qcd_cmd_id = FW_LA_CMD_AUTH;
2123
2124 return 0;
2125 }
2126
2127 static void
qat_crypto_clear_desc(struct qat_crypto_desc * desc)2128 qat_crypto_clear_desc(struct qat_crypto_desc *desc)
2129 {
2130 explicit_bzero(desc->qcd_content_desc, sizeof(desc->qcd_content_desc));
2131 explicit_bzero(desc->qcd_hash_state_prefix_buf,
2132 sizeof(desc->qcd_hash_state_prefix_buf));
2133 explicit_bzero(desc->qcd_req_cache, sizeof(desc->qcd_req_cache));
2134 }
2135
2136 static void
qat_freesession(device_t dev,crypto_session_t cses)2137 qat_freesession(device_t dev, crypto_session_t cses)
2138 {
2139 struct qat_session *qs;
2140
2141 qs = crypto_get_driver_session(cses);
2142 KASSERT(qs->qs_inflight == 0,
2143 ("%s: session %p has requests in flight", __func__, qs));
2144
2145 qat_crypto_clear_desc(qs->qs_enc_desc);
2146 qat_crypto_clear_desc(qs->qs_dec_desc);
2147 qat_free_dmamem(device_get_softc(dev), &qs->qs_desc_mem);
2148 mtx_destroy(&qs->qs_session_mtx);
2149 }
2150
2151 static int
qat_process(device_t dev,struct cryptop * crp,int hint)2152 qat_process(device_t dev, struct cryptop *crp, int hint)
2153 {
2154 struct qat_crypto *qcy;
2155 struct qat_crypto_bank *qcb;
2156 struct qat_crypto_desc const *desc;
2157 struct qat_session *qs;
2158 struct qat_softc *sc;
2159 struct qat_sym_cookie *qsc;
2160 struct qat_sym_bulk_cookie *qsbc;
2161 int error;
2162
2163 sc = device_get_softc(dev);
2164 qcy = &sc->sc_crypto;
2165 qs = crypto_get_driver_session(crp->crp_session);
2166 qsc = NULL;
2167
2168 if (__predict_false(crypto_buffer_len(&crp->crp_buf) > QAT_MAXLEN)) {
2169 error = E2BIG;
2170 goto fail1;
2171 }
2172
2173 mtx_lock(&qs->qs_session_mtx);
2174 if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) {
2175 if (crp->crp_aad_length > QAT_GCM_AAD_SIZE_MAX) {
2176 error = E2BIG;
2177 mtx_unlock(&qs->qs_session_mtx);
2178 goto fail1;
2179 }
2180
2181 /*
2182 * The firmware interface for GCM annoyingly requires the AAD
2183 * size to be stored in the session's content descriptor, which
2184 * is not really meant to be updated after session
2185 * initialization. For IPSec the AAD size is fixed so this is
2186 * not much of a problem in practice, but we have to catch AAD
2187 * size updates here so that the device code can safely update
2188 * the session's recorded AAD size.
2189 */
2190 if (__predict_false(crp->crp_aad_length != qs->qs_aad_length)) {
2191 if (qs->qs_inflight == 0) {
2192 if (qs->qs_aad_length != -1) {
2193 counter_u64_add(sc->sc_gcm_aad_updates,
2194 1);
2195 }
2196 qs->qs_aad_length = crp->crp_aad_length;
2197 } else {
2198 qs->qs_need_wakeup = true;
2199 mtx_unlock(&qs->qs_session_mtx);
2200 counter_u64_add(sc->sc_gcm_aad_restarts, 1);
2201 error = ERESTART;
2202 goto fail1;
2203 }
2204 }
2205 }
2206 qs->qs_inflight++;
2207 mtx_unlock(&qs->qs_session_mtx);
2208
2209 qcb = qat_crypto_select_bank(qcy);
2210
2211 qsc = qat_crypto_alloc_sym_cookie(qcb);
2212 if (qsc == NULL) {
2213 counter_u64_add(sc->sc_sym_alloc_failures, 1);
2214 error = ENOBUFS;
2215 goto fail2;
2216 }
2217
2218 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
2219 desc = qs->qs_enc_desc;
2220 else
2221 desc = qs->qs_dec_desc;
2222
2223 error = qat_crypto_load(qs, qsc, desc, crp);
2224 if (error != 0)
2225 goto fail2;
2226
2227 qsbc = &qsc->qsc_bulk_cookie;
2228 qsbc->qsbc_crypto = qcy;
2229 qsbc->qsbc_session = qs;
2230 qsbc->qsbc_cb_tag = crp;
2231
2232 sc->sc_hw.qhw_crypto_setup_req_params(qcb, qs, desc, qsc, crp);
2233
2234 if (crp->crp_aad != NULL) {
2235 bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dma_tag,
2236 qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dmamap,
2237 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2238 }
2239 bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dma_tag,
2240 qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dmamap,
2241 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2242 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
2243 bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dma_tag,
2244 qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dmamap,
2245 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2246 }
2247 bus_dmamap_sync(qsc->qsc_self_dma_tag, qsc->qsc_self_dmamap,
2248 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2249
2250 error = qat_etr_put_msg(sc, qcb->qcb_sym_tx,
2251 (uint32_t *)qsbc->qsbc_msg);
2252 if (error)
2253 goto fail2;
2254
2255 return 0;
2256
2257 fail2:
2258 if (qsc)
2259 qat_crypto_free_sym_cookie(qcb, qsc);
2260 mtx_lock(&qs->qs_session_mtx);
2261 qs->qs_inflight--;
2262 mtx_unlock(&qs->qs_session_mtx);
2263 fail1:
2264 crp->crp_etype = error;
2265 crypto_done(crp);
2266 return 0;
2267 }
2268
2269 static device_method_t qat_methods[] = {
2270 /* Device interface */
2271 DEVMETHOD(device_probe, qat_probe),
2272 DEVMETHOD(device_attach, qat_attach),
2273 DEVMETHOD(device_detach, qat_detach),
2274
2275 /* Cryptodev interface */
2276 DEVMETHOD(cryptodev_probesession, qat_probesession),
2277 DEVMETHOD(cryptodev_newsession, qat_newsession),
2278 DEVMETHOD(cryptodev_freesession, qat_freesession),
2279 DEVMETHOD(cryptodev_process, qat_process),
2280
2281 DEVMETHOD_END
2282 };
2283
2284 static devclass_t qat_devclass;
2285
2286 static driver_t qat_driver = {
2287 .name = "qat",
2288 .methods = qat_methods,
2289 .size = sizeof(struct qat_softc),
2290 };
2291
2292 DRIVER_MODULE(qat, pci, qat_driver, qat_devclass, 0, 0);
2293 MODULE_VERSION(qat, 1);
2294 MODULE_DEPEND(qat, crypto, 1, 1, 1);
2295 MODULE_DEPEND(qat, pci, 1, 1, 1);
2296