1 /* SPDX-License-Identifier: BSD-2-Clause AND BSD-3-Clause */
2 /* $NetBSD: qat_hw17.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */
3
4 /*
5 * Copyright (c) 2019 Internet Initiative Japan, Inc.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 * POSSIBILITY OF SUCH DAMAGE.
28 */
29
30 /*
31 * Copyright(c) 2014 Intel Corporation.
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
34 * are met:
35 *
36 * * Redistributions of source code must retain the above copyright
37 * notice, this list of conditions and the following disclaimer.
38 * * Redistributions in binary form must reproduce the above copyright
39 * notice, this list of conditions and the following disclaimer in
40 * the documentation and/or other materials provided with the
41 * distribution.
42 * * Neither the name of Intel Corporation nor the names of its
43 * contributors may be used to endorse or promote products derived
44 * from this software without specific prior written permission.
45 *
46 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
47 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
48 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
49 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
50 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
51 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
52 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
53 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
54 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
55 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
56 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
57 */
58
59 #include <sys/cdefs.h>
60 #if 0
61 __KERNEL_RCSID(0, "$NetBSD: qat_hw17.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $");
62 #endif
63
64 #include <sys/param.h>
65 #include <sys/systm.h>
66 #include <sys/bus.h>
67 #include <sys/proc.h>
68
69 #include <machine/bus.h>
70
71 #include <opencrypto/xform.h>
72
73 #include <dev/pci/pcireg.h>
74 #include <dev/pci/pcivar.h>
75
76 #include "qatreg.h"
77 #include "qat_hw17reg.h"
78 #include "qatvar.h"
79 #include "qat_hw17var.h"
80
81 int qat_adm_mailbox_put_msg_sync(struct qat_softc *, uint32_t,
82 void *, void *);
83 int qat_adm_mailbox_send(struct qat_softc *,
84 struct fw_init_admin_req *, struct fw_init_admin_resp *);
85 int qat_adm_mailbox_send_init_me(struct qat_softc *);
86 int qat_adm_mailbox_send_hb_timer(struct qat_softc *);
87 int qat_adm_mailbox_send_fw_status(struct qat_softc *);
88 int qat_adm_mailbox_send_constants(struct qat_softc *);
89
90 int
qat_adm_mailbox_init(struct qat_softc * sc)91 qat_adm_mailbox_init(struct qat_softc *sc)
92 {
93 uint64_t addr;
94 int error;
95 struct qat_dmamem *qdm;
96
97 error = qat_alloc_dmamem(sc, &sc->sc_admin_comms.qadc_dma, 1,
98 PAGE_SIZE, PAGE_SIZE);
99 if (error)
100 return error;
101
102 qdm = &sc->sc_admin_comms.qadc_const_tbl_dma;
103 error = qat_alloc_dmamem(sc, qdm, 1, PAGE_SIZE, PAGE_SIZE);
104 if (error)
105 return error;
106
107 memcpy(qdm->qdm_dma_vaddr,
108 mailbox_const_tab, sizeof(mailbox_const_tab));
109
110 bus_dmamap_sync(qdm->qdm_dma_tag, qdm->qdm_dma_map,
111 BUS_DMASYNC_PREWRITE);
112
113 error = qat_alloc_dmamem(sc, &sc->sc_admin_comms.qadc_hb_dma, 1,
114 PAGE_SIZE, PAGE_SIZE);
115 if (error)
116 return error;
117
118 addr = (uint64_t)sc->sc_admin_comms.qadc_dma.qdm_dma_seg.ds_addr;
119 qat_misc_write_4(sc, ADMINMSGUR, addr >> 32);
120 qat_misc_write_4(sc, ADMINMSGLR, addr);
121
122 return 0;
123 }
124
125 int
qat_adm_mailbox_put_msg_sync(struct qat_softc * sc,uint32_t ae,void * in,void * out)126 qat_adm_mailbox_put_msg_sync(struct qat_softc *sc, uint32_t ae,
127 void *in, void *out)
128 {
129 struct qat_dmamem *qdm;
130 uint32_t mailbox;
131 bus_size_t mb_offset = MAILBOX_BASE + (ae * MAILBOX_STRIDE);
132 int offset = ae * ADMINMSG_LEN * 2;
133 int times, received;
134 uint8_t *buf = (uint8_t *)sc->sc_admin_comms.qadc_dma.qdm_dma_vaddr + offset;
135
136 mailbox = qat_misc_read_4(sc, mb_offset);
137 if (mailbox == 1)
138 return EAGAIN;
139
140 qdm = &sc->sc_admin_comms.qadc_dma;
141 memcpy(buf, in, ADMINMSG_LEN);
142 bus_dmamap_sync(qdm->qdm_dma_tag, qdm->qdm_dma_map,
143 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
144 qat_misc_write_4(sc, mb_offset, 1);
145
146 received = 0;
147 for (times = 0; times < 50; times++) {
148 DELAY(20000);
149 if (qat_misc_read_4(sc, mb_offset) == 0) {
150 received = 1;
151 break;
152 }
153 }
154 if (received) {
155 bus_dmamap_sync(qdm->qdm_dma_tag, qdm->qdm_dma_map,
156 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
157 memcpy(out, buf + ADMINMSG_LEN, ADMINMSG_LEN);
158 } else {
159 device_printf(sc->sc_dev,
160 "Failed to send admin msg to accelerator\n");
161 }
162
163 return received ? 0 : EFAULT;
164 }
165
166 int
qat_adm_mailbox_send(struct qat_softc * sc,struct fw_init_admin_req * req,struct fw_init_admin_resp * resp)167 qat_adm_mailbox_send(struct qat_softc *sc,
168 struct fw_init_admin_req *req, struct fw_init_admin_resp *resp)
169 {
170 int error;
171 uint32_t mask;
172 uint8_t ae;
173
174 for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) {
175 if (!(mask & 1))
176 continue;
177
178 error = qat_adm_mailbox_put_msg_sync(sc, ae, req, resp);
179 if (error)
180 return error;
181 if (resp->init_resp_hdr.status) {
182 device_printf(sc->sc_dev,
183 "Failed to send admin msg: cmd %d\n",
184 req->init_admin_cmd_id);
185 return EFAULT;
186 }
187 }
188
189 return 0;
190 }
191
192 int
qat_adm_mailbox_send_init_me(struct qat_softc * sc)193 qat_adm_mailbox_send_init_me(struct qat_softc *sc)
194 {
195 struct fw_init_admin_req req;
196 struct fw_init_admin_resp resp;
197
198 memset(&req, 0, sizeof(req));
199 req.init_admin_cmd_id = FW_INIT_ME;
200
201 return qat_adm_mailbox_send(sc, &req, &resp);
202 }
203
204 int
qat_adm_mailbox_send_hb_timer(struct qat_softc * sc)205 qat_adm_mailbox_send_hb_timer(struct qat_softc *sc)
206 {
207 struct fw_init_admin_req req;
208 struct fw_init_admin_resp resp;
209
210 memset(&req, 0, sizeof(req));
211 req.init_admin_cmd_id = FW_HEARTBEAT_TIMER_SET;
212
213 req.init_cfg_ptr = sc->sc_admin_comms.qadc_hb_dma.qdm_dma_seg.ds_addr;
214 req.heartbeat_ticks =
215 sc->sc_hw.qhw_clock_per_sec / 1000 * QAT_HB_INTERVAL;
216
217 return qat_adm_mailbox_send(sc, &req, &resp);
218 }
219
220 int
qat_adm_mailbox_send_fw_status(struct qat_softc * sc)221 qat_adm_mailbox_send_fw_status(struct qat_softc *sc)
222 {
223 int error;
224 struct fw_init_admin_req req;
225 struct fw_init_admin_resp resp;
226
227 memset(&req, 0, sizeof(req));
228 req.init_admin_cmd_id = FW_STATUS_GET;
229
230 error = qat_adm_mailbox_send(sc, &req, &resp);
231 if (error)
232 return error;
233
234 return 0;
235 }
236
237 int
qat_adm_mailbox_send_constants(struct qat_softc * sc)238 qat_adm_mailbox_send_constants(struct qat_softc *sc)
239 {
240 struct fw_init_admin_req req;
241 struct fw_init_admin_resp resp;
242
243 memset(&req, 0, sizeof(req));
244 req.init_admin_cmd_id = FW_CONSTANTS_CFG;
245
246 req.init_cfg_sz = 1024;
247 req.init_cfg_ptr =
248 sc->sc_admin_comms.qadc_const_tbl_dma.qdm_dma_seg.ds_addr;
249
250 return qat_adm_mailbox_send(sc, &req, &resp);
251 }
252
253 int
qat_adm_mailbox_send_init(struct qat_softc * sc)254 qat_adm_mailbox_send_init(struct qat_softc *sc)
255 {
256 int error;
257
258 error = qat_adm_mailbox_send_init_me(sc);
259 if (error)
260 return error;
261
262 error = qat_adm_mailbox_send_hb_timer(sc);
263 if (error)
264 return error;
265
266 error = qat_adm_mailbox_send_fw_status(sc);
267 if (error)
268 return error;
269
270 return qat_adm_mailbox_send_constants(sc);
271 }
272
273 int
qat_arb_init(struct qat_softc * sc)274 qat_arb_init(struct qat_softc *sc)
275 {
276 uint32_t arb_cfg = 0x1 << 31 | 0x4 << 4 | 0x1;
277 uint32_t arb, i;
278 const uint32_t *thd_2_arb_cfg;
279
280 /* Service arb configured for 32 bytes responses and
281 * ring flow control check enabled. */
282 for (arb = 0; arb < MAX_ARB; arb++)
283 qat_arb_sarconfig_write_4(sc, arb, arb_cfg);
284
285 /* Map worker threads to service arbiters */
286 sc->sc_hw.qhw_get_arb_mapping(sc, &thd_2_arb_cfg);
287
288 if (!thd_2_arb_cfg)
289 return EINVAL;
290
291 for (i = 0; i < sc->sc_hw.qhw_num_engines; i++)
292 qat_arb_wrk_2_ser_map_write_4(sc, i, *(thd_2_arb_cfg + i));
293
294 return 0;
295 }
296
297 int
qat_set_ssm_wdtimer(struct qat_softc * sc)298 qat_set_ssm_wdtimer(struct qat_softc *sc)
299 {
300 uint32_t timer;
301 u_int mask;
302 int i;
303
304 timer = sc->sc_hw.qhw_clock_per_sec / 1000 * QAT_SSM_WDT;
305 for (i = 0, mask = sc->sc_accel_mask; mask; i++, mask >>= 1) {
306 if (!(mask & 1))
307 continue;
308 qat_misc_write_4(sc, SSMWDT(i), timer);
309 qat_misc_write_4(sc, SSMWDTPKE(i), timer);
310 }
311
312 return 0;
313 }
314
315 int
qat_check_slice_hang(struct qat_softc * sc)316 qat_check_slice_hang(struct qat_softc *sc)
317 {
318 int handled = 0;
319
320 return handled;
321 }
322
323 static uint32_t
qat_hw17_crypto_setup_cipher_ctrl(struct qat_crypto_desc * desc,struct qat_session * qs,uint32_t cd_blk_offset,struct fw_la_bulk_req * req_tmpl,enum fw_slice next_slice)324 qat_hw17_crypto_setup_cipher_ctrl(struct qat_crypto_desc *desc,
325 struct qat_session *qs, uint32_t cd_blk_offset,
326 struct fw_la_bulk_req *req_tmpl, enum fw_slice next_slice)
327 {
328 struct fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl =
329 (struct fw_cipher_cd_ctrl_hdr *)&req_tmpl->cd_ctrl;
330
331 desc->qcd_cipher_blk_sz = HW_AES_BLK_SZ;
332 desc->qcd_cipher_offset = cd_blk_offset;
333
334 cipher_cd_ctrl->cipher_state_sz = desc->qcd_cipher_blk_sz >> 3;
335 cipher_cd_ctrl->cipher_key_sz = qs->qs_cipher_klen >> 3;
336 cipher_cd_ctrl->cipher_cfg_offset = cd_blk_offset >> 3;
337 FW_COMN_CURR_ID_SET(cipher_cd_ctrl, FW_SLICE_CIPHER);
338 FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, next_slice);
339
340 return roundup(sizeof(struct hw_cipher_config) + qs->qs_cipher_klen, 8);
341 }
342
343 static void
qat_hw17_crypto_setup_cipher_cdesc(const struct qat_crypto_desc * desc,const struct qat_session * qs,const struct cryptop * crp,union hw_cipher_algo_blk * cipher)344 qat_hw17_crypto_setup_cipher_cdesc(const struct qat_crypto_desc *desc,
345 const struct qat_session *qs, const struct cryptop *crp,
346 union hw_cipher_algo_blk *cipher)
347 {
348 const uint8_t *key;
349
350 cipher->max.cipher_config.val =
351 qat_crypto_load_cipher_session(desc, qs);
352 if (crp != NULL && crp->crp_cipher_key != NULL)
353 key = crp->crp_cipher_key;
354 else
355 key = qs->qs_cipher_key;
356 memcpy(cipher->max.key, key, qs->qs_cipher_klen);
357 }
358
359 static uint32_t
qat_hw17_crypto_setup_auth_ctrl(struct qat_crypto_desc * desc,struct qat_session * qs,uint32_t cd_blk_offset,struct fw_la_bulk_req * req_tmpl,enum fw_slice next_slice)360 qat_hw17_crypto_setup_auth_ctrl(struct qat_crypto_desc *desc,
361 struct qat_session *qs, uint32_t cd_blk_offset,
362 struct fw_la_bulk_req *req_tmpl, enum fw_slice next_slice)
363 {
364 struct fw_auth_cd_ctrl_hdr *auth_cd_ctrl =
365 (struct fw_auth_cd_ctrl_hdr *)&req_tmpl->cd_ctrl;
366 struct qat_sym_hash_def const *hash_def;
367
368 (void)qat_crypto_load_auth_session(desc, qs, &hash_def);
369
370 auth_cd_ctrl->hash_cfg_offset = cd_blk_offset >> 3;
371 auth_cd_ctrl->hash_flags = FW_AUTH_HDR_FLAG_NO_NESTED;
372 auth_cd_ctrl->inner_res_sz = hash_def->qshd_alg->qshai_digest_len;
373 auth_cd_ctrl->final_sz = hash_def->qshd_alg->qshai_sah->hashsize;
374
375 auth_cd_ctrl->inner_state1_sz =
376 roundup(hash_def->qshd_qat->qshqi_state1_len, 8);
377 auth_cd_ctrl->inner_state2_sz =
378 roundup(hash_def->qshd_qat->qshqi_state2_len, 8);
379 auth_cd_ctrl->inner_state2_offset =
380 auth_cd_ctrl->hash_cfg_offset +
381 ((sizeof(struct hw_auth_setup) +
382 auth_cd_ctrl->inner_state1_sz) >> 3);
383
384 FW_COMN_CURR_ID_SET(auth_cd_ctrl, FW_SLICE_AUTH);
385 FW_COMN_NEXT_ID_SET(auth_cd_ctrl, next_slice);
386
387 desc->qcd_auth_sz = auth_cd_ctrl->final_sz;
388 desc->qcd_auth_offset = cd_blk_offset;
389 desc->qcd_gcm_aad_sz_offset1 =
390 cd_blk_offset + offsetof(union hw_auth_algo_blk, max.state1) +
391 auth_cd_ctrl->inner_state1_sz + AES_BLOCK_LEN;
392
393 return roundup(auth_cd_ctrl->inner_state1_sz +
394 auth_cd_ctrl->inner_state2_sz +
395 sizeof(struct hw_auth_setup), 8);
396 }
397
398 static void
qat_hw17_crypto_setup_auth_cdesc(const struct qat_crypto_desc * desc,const struct qat_session * qs,const struct cryptop * crp,union hw_auth_algo_blk * auth)399 qat_hw17_crypto_setup_auth_cdesc(const struct qat_crypto_desc *desc,
400 const struct qat_session *qs, const struct cryptop *crp,
401 union hw_auth_algo_blk *auth)
402 {
403 struct qat_sym_hash_def const *hash_def;
404 uint8_t inner_state1_sz, *state1, *state2;
405 const uint8_t *key;
406
407 auth->max.inner_setup.auth_config.config =
408 qat_crypto_load_auth_session(desc, qs, &hash_def);
409 auth->max.inner_setup.auth_counter.counter =
410 htobe32(hash_def->qshd_qat->qshqi_auth_counter);
411 inner_state1_sz = roundup(hash_def->qshd_qat->qshqi_state1_len, 8);
412
413 state1 = auth->max.state1;
414 state2 = auth->max.state1 + inner_state1_sz;
415 switch (qs->qs_auth_algo) {
416 case HW_AUTH_ALGO_GALOIS_128:
417 key = NULL;
418 if (crp != NULL && crp->crp_cipher_key != NULL)
419 key = crp->crp_cipher_key;
420 else if (qs->qs_cipher_key != NULL)
421 key = qs->qs_cipher_key;
422 if (key != NULL) {
423 qat_crypto_gmac_precompute(desc, key,
424 qs->qs_cipher_klen, hash_def, state2);
425 }
426 break;
427 case HW_AUTH_ALGO_SHA1:
428 case HW_AUTH_ALGO_SHA256:
429 case HW_AUTH_ALGO_SHA384:
430 case HW_AUTH_ALGO_SHA512:
431 switch (qs->qs_auth_mode) {
432 case HW_AUTH_MODE0:
433 memcpy(state1, hash_def->qshd_alg->qshai_init_state,
434 inner_state1_sz);
435 /* Override for mode 0 hashes. */
436 auth->max.inner_setup.auth_counter.counter = 0;
437 break;
438 case HW_AUTH_MODE1:
439 if (crp != NULL && crp->crp_auth_key != NULL)
440 key = crp->crp_auth_key;
441 else
442 key = qs->qs_auth_key;
443 if (key != NULL) {
444 qat_crypto_hmac_precompute(desc, key,
445 qs->qs_auth_klen, hash_def, state1, state2);
446 }
447 break;
448 default:
449 panic("%s: unhandled auth mode %d", __func__,
450 qs->qs_auth_mode);
451 }
452 break;
453 default:
454 panic("%s: unhandled auth algorithm %d", __func__,
455 qs->qs_auth_algo);
456 }
457 }
458
459 static void
qat_hw17_init_comn_req_hdr(struct qat_crypto_desc * desc,struct fw_la_bulk_req * req)460 qat_hw17_init_comn_req_hdr(struct qat_crypto_desc *desc,
461 struct fw_la_bulk_req *req)
462 {
463 union fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
464 struct fw_comn_req_hdr *req_hdr = &req->comn_hdr;
465
466 req_hdr->service_cmd_id = desc->qcd_cmd_id;
467 req_hdr->hdr_flags = FW_COMN_VALID;
468 req_hdr->service_type = FW_COMN_REQ_CPM_FW_LA;
469 req_hdr->comn_req_flags = FW_COMN_FLAGS_BUILD(
470 COMN_CD_FLD_TYPE_64BIT_ADR, COMN_PTR_TYPE_SGL);
471 req_hdr->serv_specif_flags = 0;
472 cd_pars->s.content_desc_addr = desc->qcd_desc_paddr;
473 }
474
475 void
qat_hw17_crypto_setup_desc(struct qat_crypto * qcy,struct qat_session * qs,struct qat_crypto_desc * desc)476 qat_hw17_crypto_setup_desc(struct qat_crypto *qcy, struct qat_session *qs,
477 struct qat_crypto_desc *desc)
478 {
479 union hw_cipher_algo_blk *cipher;
480 union hw_auth_algo_blk *auth;
481 struct fw_la_bulk_req *req_tmpl;
482 struct fw_comn_req_hdr *req_hdr;
483 uint32_t cd_blk_offset = 0;
484 int i;
485 uint8_t *cd_blk_ptr;
486
487 req_tmpl = (struct fw_la_bulk_req *)desc->qcd_req_cache;
488 req_hdr = &req_tmpl->comn_hdr;
489 cd_blk_ptr = desc->qcd_content_desc;
490
491 memset(req_tmpl, 0, sizeof(struct fw_la_bulk_req));
492 qat_hw17_init_comn_req_hdr(desc, req_tmpl);
493
494 for (i = 0; i < MAX_FW_SLICE; i++) {
495 switch (desc->qcd_slices[i]) {
496 case FW_SLICE_CIPHER:
497 cipher = (union hw_cipher_algo_blk *)(cd_blk_ptr +
498 cd_blk_offset);
499 cd_blk_offset += qat_hw17_crypto_setup_cipher_ctrl(desc,
500 qs, cd_blk_offset, req_tmpl,
501 desc->qcd_slices[i + 1]);
502 qat_hw17_crypto_setup_cipher_cdesc(desc, qs, NULL,
503 cipher);
504 break;
505 case FW_SLICE_AUTH:
506 auth = (union hw_auth_algo_blk *)(cd_blk_ptr +
507 cd_blk_offset);
508 cd_blk_offset += qat_hw17_crypto_setup_auth_ctrl(desc,
509 qs, cd_blk_offset, req_tmpl,
510 desc->qcd_slices[i + 1]);
511 qat_hw17_crypto_setup_auth_cdesc(desc, qs, NULL, auth);
512 req_hdr->serv_specif_flags |= FW_LA_RET_AUTH_RES;
513 break;
514 case FW_SLICE_DRAM_WR:
515 i = MAX_FW_SLICE; /* end of chain */
516 break;
517 default:
518 MPASS(0);
519 break;
520 }
521 }
522
523 req_tmpl->cd_pars.s.content_desc_params_sz =
524 roundup(cd_blk_offset, QAT_OPTIMAL_ALIGN) >> 3;
525 if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128)
526 req_hdr->serv_specif_flags |=
527 FW_LA_PROTO_GCM | FW_LA_GCM_IV_LEN_12_OCTETS;
528
529 bus_dmamap_sync(qs->qs_desc_mem.qdm_dma_tag,
530 qs->qs_desc_mem.qdm_dma_map, BUS_DMASYNC_PREWRITE);
531 }
532
533 static void
qat_hw17_crypto_req_setkey(const struct qat_crypto_desc * desc,const struct qat_session * qs,struct qat_sym_cookie * qsc,struct fw_la_bulk_req * bulk_req,const struct cryptop * crp)534 qat_hw17_crypto_req_setkey(const struct qat_crypto_desc *desc,
535 const struct qat_session *qs, struct qat_sym_cookie *qsc,
536 struct fw_la_bulk_req *bulk_req, const struct cryptop *crp)
537 {
538 union hw_auth_algo_blk *auth;
539 union hw_cipher_algo_blk *cipher;
540 uint8_t *cdesc;
541 int i;
542
543 cdesc = qsc->qsc_content_desc;
544 memcpy(cdesc, desc->qcd_content_desc, CONTENT_DESC_MAX_SIZE);
545 for (i = 0; i < MAX_FW_SLICE; i++) {
546 switch (desc->qcd_slices[i]) {
547 case FW_SLICE_CIPHER:
548 cipher = (union hw_cipher_algo_blk *)
549 (cdesc + desc->qcd_cipher_offset);
550 qat_hw17_crypto_setup_cipher_cdesc(desc, qs, crp,
551 cipher);
552 break;
553 case FW_SLICE_AUTH:
554 auth = (union hw_auth_algo_blk *)
555 (cdesc + desc->qcd_auth_offset);
556 qat_hw17_crypto_setup_auth_cdesc(desc, qs, crp, auth);
557 break;
558 case FW_SLICE_DRAM_WR:
559 i = MAX_FW_SLICE; /* end of chain */
560 break;
561 default:
562 MPASS(0);
563 }
564 }
565
566 bulk_req->cd_pars.s.content_desc_addr = qsc->qsc_content_desc_paddr;
567 }
568
569 void
qat_hw17_crypto_setup_req_params(struct qat_crypto_bank * qcb __unused,struct qat_session * qs,const struct qat_crypto_desc * desc,struct qat_sym_cookie * qsc,struct cryptop * crp)570 qat_hw17_crypto_setup_req_params(struct qat_crypto_bank *qcb __unused,
571 struct qat_session *qs, const struct qat_crypto_desc *desc,
572 struct qat_sym_cookie *qsc, struct cryptop *crp)
573 {
574 struct qat_sym_bulk_cookie *qsbc;
575 struct fw_la_bulk_req *bulk_req;
576 struct fw_la_cipher_req_params *cipher_param;
577 struct fw_la_auth_req_params *auth_param;
578 bus_addr_t digest_paddr;
579 uint32_t aad_sz, *aad_szp;
580 uint8_t *req_params_ptr;
581 enum fw_la_cmd_id cmd_id = desc->qcd_cmd_id;
582
583 qsbc = &qsc->qsc_bulk_cookie;
584 bulk_req = (struct fw_la_bulk_req *)qsbc->qsbc_msg;
585
586 memcpy(bulk_req, desc->qcd_req_cache, sizeof(struct fw_la_bulk_req));
587 bulk_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)qsc;
588 bulk_req->comn_mid.src_data_addr = qsc->qsc_buffer_list_desc_paddr;
589 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
590 bulk_req->comn_mid.dest_data_addr =
591 qsc->qsc_obuffer_list_desc_paddr;
592 } else {
593 bulk_req->comn_mid.dest_data_addr =
594 qsc->qsc_buffer_list_desc_paddr;
595 }
596 if (__predict_false(crp->crp_cipher_key != NULL ||
597 crp->crp_auth_key != NULL))
598 qat_hw17_crypto_req_setkey(desc, qs, qsc, bulk_req, crp);
599
600 digest_paddr = 0;
601 if (desc->qcd_auth_sz != 0)
602 digest_paddr = qsc->qsc_auth_res_paddr;
603
604 req_params_ptr = (uint8_t *)&bulk_req->serv_specif_rqpars;
605 cipher_param = (struct fw_la_cipher_req_params *)req_params_ptr;
606 auth_param = (struct fw_la_auth_req_params *)
607 (req_params_ptr + sizeof(struct fw_la_cipher_req_params));
608
609 cipher_param->u.s.cipher_IV_ptr = qsc->qsc_iv_buf_paddr;
610
611 /*
612 * The SG list layout is a bit different for GCM and GMAC, it's simpler
613 * to handle those cases separately.
614 */
615 if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) {
616 if (cmd_id != FW_LA_CMD_AUTH) {
617 /*
618 * Don't fill out the cipher block if we're doing GMAC
619 * only.
620 */
621 cipher_param->cipher_offset = 0;
622 cipher_param->cipher_length = crp->crp_payload_length;
623 }
624 auth_param->auth_off = 0;
625 auth_param->auth_len = crp->crp_payload_length;
626 auth_param->auth_res_addr = digest_paddr;
627 auth_param->auth_res_sz = desc->qcd_auth_sz;
628 auth_param->u1.aad_adr =
629 crp->crp_aad_length > 0 ? qsc->qsc_gcm_aad_paddr : 0;
630 auth_param->u2.aad_sz =
631 roundup2(crp->crp_aad_length, QAT_AES_GCM_AAD_ALIGN);
632 auth_param->hash_state_sz = auth_param->u2.aad_sz >> 3;
633
634 /*
635 * Update the hash state block if necessary. This only occurs
636 * when the AAD length changes between requests in a session and
637 * is synchronized by qat_process().
638 */
639 aad_sz = htobe32(crp->crp_aad_length);
640 aad_szp = (uint32_t *)(
641 __DECONST(uint8_t *, desc->qcd_content_desc) +
642 desc->qcd_gcm_aad_sz_offset1);
643 if (__predict_false(*aad_szp != aad_sz)) {
644 *aad_szp = aad_sz;
645 bus_dmamap_sync(qs->qs_desc_mem.qdm_dma_tag,
646 qs->qs_desc_mem.qdm_dma_map,
647 BUS_DMASYNC_PREWRITE);
648 }
649 } else {
650 if (cmd_id != FW_LA_CMD_AUTH) {
651 if (crp->crp_aad_length == 0) {
652 cipher_param->cipher_offset = 0;
653 } else if (crp->crp_aad == NULL) {
654 cipher_param->cipher_offset =
655 crp->crp_payload_start - crp->crp_aad_start;
656 } else {
657 cipher_param->cipher_offset =
658 crp->crp_aad_length;
659 }
660 cipher_param->cipher_length = crp->crp_payload_length;
661 }
662 if (cmd_id != FW_LA_CMD_CIPHER) {
663 auth_param->auth_off = 0;
664 auth_param->auth_len =
665 crp->crp_payload_length + crp->crp_aad_length;
666 auth_param->auth_res_addr = digest_paddr;
667 auth_param->auth_res_sz = desc->qcd_auth_sz;
668 auth_param->u1.aad_adr = 0;
669 auth_param->u2.aad_sz = 0;
670 auth_param->hash_state_sz = 0;
671 }
672 }
673 }
674