1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause AND BSD-2-Clause
3 *
4 * Copyright © 2014-2016 Freescale Semiconductor, Inc.
5 * Copyright © 2016-2019 NXP
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright notice,
12 * this list of conditions and the following disclaimer.
13 *
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * 3. Neither the name of the copyright holder nor the names of its
19 * contributors may be used to endorse or promote products derived from this
20 * software without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
26 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
33 *
34 * Original source file obtained from:
35 * drivers/soc/fsl/dpio/qbman-portal.c
36 *
37 * Commit: 4c86114194e644b6da9107d75910635c9e87179e
38 * Repository: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
39 */
40
41 /*
42 * Copyright © 2021-2022 Dmitry Salychev
43 *
44 * Redistribution and use in source and binary forms, with or without
45 * modification, are permitted provided that the following conditions
46 * are met:
47 * 1. Redistributions of source code must retain the above copyright
48 * notice, this list of conditions and the following disclaimer.
49 * 2. Redistributions in binary form must reproduce the above copyright
50 * notice, this list of conditions and the following disclaimer in the
51 * documentation and/or other materials provided with the distribution.
52 *
53 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
56 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
63 * SUCH DAMAGE.
64 */
65
66 #include <sys/cdefs.h>
67 /*
68 * DPAA2 QBMan software portal.
69 */
70
71 #include <sys/param.h>
72 #include <sys/kernel.h>
73 #include <sys/bus.h>
74 #include <sys/rman.h>
75 #include <sys/module.h>
76 #include <sys/malloc.h>
77 #include <sys/mutex.h>
78 #include <sys/time.h>
79 #include <sys/types.h>
80 #include <sys/systm.h>
81 #include <sys/condvar.h>
82 #include <sys/lock.h>
83
84 #include <machine/bus.h>
85 #include <machine/resource.h>
86 #include <machine/atomic.h>
87
88 #include "pcib_if.h"
89 #include "pci_if.h"
90
91 #include "dpaa2_swp.h"
92 #include "dpaa2_mc.h"
93 #include "dpaa2_bp.h"
94
95 #define CMD_SPIN_TIMEOUT 100u /* us */
96 #define CMD_SPIN_ATTEMPTS 2000u /* 200 ms max. */
97
98 #define CMD_VERB_MASK 0x7Fu
99
100 /* Shifts in the VERB byte of the enqueue command descriptor. */
101 #define ENQ_CMD_ORP_ENABLE_SHIFT 2
102 #define ENQ_CMD_IRQ_ON_DISPATCH_SHIFT 3
103 #define ENQ_CMD_TARGET_TYPE_SHIFT 4
104 #define ENQ_CMD_DCA_EN_SHIFT 7
105 /* VERB byte options of the enqueue command descriptor. */
106 #define ENQ_CMD_EMPTY 0u
107 #define ENQ_CMD_RESPONSE_ALWAYS 1u
108 #define ENQ_CMD_REJECTS_TO_FQ 2u
109
110 #define ENQ_DESC_FD_OFFSET 32u
111
112 #define ENQ_DCA_IDXMASK 0x0Fu
113 #define ENQ_FLAG_DCA (1ull << 31)
114
115 /* QBMan portal command codes. */
116 #define CMDID_SWP_MC_ACQUIRE 0x30
117 #define CMDID_SWP_BP_QUERY 0x32
118 #define CMDID_SWP_WQCHAN_CONFIGURE 0x46
119
120 /* QBMan portal command result codes. */
121 #define QBMAN_CMD_RC_OK 0xF0
122
123 /* SDQCR attribute codes */
124 #define QB_SDQCR_FC_SHIFT 29u
125 #define QB_SDQCR_FC_MASK 0x1u
126 #define QB_SDQCR_DCT_SHIFT 24u
127 #define QB_SDQCR_DCT_MASK 0x3u
128 #define QB_SDQCR_TOK_SHIFT 16u
129 #define QB_SDQCR_TOK_MASK 0xFFu
130 #define QB_SDQCR_SRC_SHIFT 0u
131 #define QB_SDQCR_SRC_MASK 0xFFFFu
132
133 /* Shifts in the VERB byte of the volatile dequeue command. */
134 #define QB_VDQCR_VERB_DCT0_SHIFT 0
135 #define QB_VDQCR_VERB_DCT1_SHIFT 1
136 #define QB_VDQCR_VERB_DT0_SHIFT 2
137 #define QB_VDQCR_VERB_DT1_SHIFT 3
138 #define QB_VDQCR_VERB_RLS_SHIFT 4
139 #define QB_VDQCR_VERB_WAE_SHIFT 5
140 #define QB_VDQCR_VERB_RAD_SHIFT 6
141
142 /* Maximum timeout period for the DQRR interrupt. */
143 #define DQRR_MAX_ITP 4096u
144 #define DQRR_PI_MASK 0x0Fu
145
146 /* Release Array Allocation register helpers. */
147 #define RAR_IDX(rar) ((rar) & 0x7u)
148 #define RAR_VB(rar) ((rar) & 0x80u)
149 #define RAR_SUCCESS(rar) ((rar) & 0x100u)
150
151 MALLOC_DEFINE(M_DPAA2_SWP, "dpaa2_swp", "DPAA2 QBMan Software Portal");
152
153 enum qbman_sdqcr_dct {
154 qbman_sdqcr_dct_null = 0,
155 qbman_sdqcr_dct_prio_ics,
156 qbman_sdqcr_dct_active_ics,
157 qbman_sdqcr_dct_active
158 };
159
160 enum qbman_sdqcr_fc {
161 qbman_sdqcr_fc_one = 0,
162 qbman_sdqcr_fc_up_to_3 = 1
163 };
164
165 /* Routines to execute software portal commands. */
166 static int dpaa2_swp_exec_mgmt_command(struct dpaa2_swp *,
167 struct dpaa2_swp_cmd *, struct dpaa2_swp_rsp *, uint8_t);
168 static int dpaa2_swp_exec_br_command(struct dpaa2_swp *, struct dpaa2_swp_cmd *,
169 uint32_t);
170 static int dpaa2_swp_exec_vdc_command_locked(struct dpaa2_swp *,
171 struct dpaa2_swp_cmd *);
172
173 /* Management Commands helpers. */
174 static int dpaa2_swp_send_mgmt_command(struct dpaa2_swp *,
175 struct dpaa2_swp_cmd *, uint8_t);
176 static int dpaa2_swp_wait_for_mgmt_response(struct dpaa2_swp *,
177 struct dpaa2_swp_rsp *);
178
179 /* Helper subroutines. */
180 static int dpaa2_swp_cyc_diff(uint8_t, uint8_t, uint8_t);
181
182 int
dpaa2_swp_init_portal(struct dpaa2_swp ** swp,struct dpaa2_swp_desc * desc,uint16_t flags)183 dpaa2_swp_init_portal(struct dpaa2_swp **swp, struct dpaa2_swp_desc *desc,
184 uint16_t flags)
185 {
186 struct dpaa2_swp *p;
187 uint32_t reg, mask_size, eqcr_pi; /* EQCR producer index */
188
189 if (!swp || !desc)
190 return (DPAA2_SWP_STAT_EINVAL);
191
192 p = malloc(sizeof(struct dpaa2_swp), M_DPAA2_SWP,
193 flags & DPAA2_SWP_NOWAIT_ALLOC
194 ? (M_NOWAIT | M_ZERO)
195 : (M_WAITOK | M_ZERO));
196 if (!p)
197 return (DPAA2_SWP_STAT_NO_MEMORY);
198
199 mtx_init(&p->lock, "swp_sleep_lock", NULL, MTX_DEF);
200
201 p->cfg.mem_backed = false;
202 p->cfg.writes_cinh = true;
203
204 p->desc = desc;
205 p->flags = flags;
206 p->mc.valid_bit = DPAA2_SWP_VALID_BIT;
207 p->mr.valid_bit = DPAA2_SWP_VALID_BIT;
208
209 /* FIXME: Memory-backed mode doesn't work now. Why? */
210 p->cena_res = desc->cena_res;
211 p->cena_map = desc->cena_map;
212 p->cinh_res = desc->cinh_res;
213 p->cinh_map = desc->cinh_map;
214
215 /* Static Dequeue Command Register configuration. */
216 p->sdq = 0;
217 p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT;
218 p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT;
219 p->sdq |= DPAA2_SWP_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT;
220
221 /* Volatile Dequeue Command configuration. */
222 p->vdq.valid_bit = DPAA2_SWP_VALID_BIT;
223
224 /* Dequeue Response Ring configuration */
225 p->dqrr.next_idx = 0;
226 p->dqrr.valid_bit = DPAA2_SWP_VALID_BIT;
227 if ((desc->swp_version & DPAA2_SWP_REV_MASK) < DPAA2_SWP_REV_4100) {
228 p->dqrr.ring_size = 4;
229 p->dqrr.reset_bug = 1;
230 } else {
231 p->dqrr.ring_size = 8;
232 p->dqrr.reset_bug = 0;
233 }
234
235 if ((desc->swp_version & DPAA2_SWP_REV_MASK) < DPAA2_SWP_REV_5000) {
236 reg = dpaa2_swp_set_cfg(
237 p->dqrr.ring_size, /* max. entries QMan writes to DQRR */
238 1, /* writes enabled in the CINH memory only */
239 0, /* EQCR_CI stashing threshold */
240 3, /* RPM: RCR in array mode */
241 2, /* DCM: Discrete consumption ack */
242 2, /* EPM: EQCR in ring mode (FIFO) */
243 1, /* mem stashing drop enable enable */
244 1, /* mem stashing priority enable */
245 1, /* mem stashing enable */
246 1, /* dequeue stashing priority enable */
247 0, /* dequeue stashing enable enable */
248 0 /* EQCR_CI stashing priority enable */
249 );
250 reg &= ~(1 << DPAA2_SWP_CFG_CPBS_SHIFT); /* QMan-backed mode */
251 } else {
252 bus_set_region_4(p->cena_map, 0, 0,
253 rman_get_size(p->cena_res) / 4);
254
255 reg = dpaa2_swp_set_cfg(
256 p->dqrr.ring_size, /* max. entries QMan writes to DQRR */ /* DQRR_MF */
257 1, /* writes enabled in the CINH memory only */ /* WN */
258 0, /* EQCR_CI stashing is disabled */ /* EST */
259 3, /* RPM: RCR in array mode */ /* RPM */
260 2, /* DCM: Discrete consumption ack */ /* DCM */
261 2, /* EPM: EQCR in ring mode (FIFO) */ /* EPM */
262 1, /* Dequeued frame data, annotation, and FQ context stashing drop enable */ /* SD */
263 1, /* Dequeued frame data, annotation, and FQ context stashing priority */ /* SP */
264 1, /* Dequeued frame data, annotation, and FQ context stashing enable */ /* SE */
265 1, /* Dequeue response ring (DQRR) entry stashing priority */ /* DP */
266 0, /* Dequeue response ring (DQRR) entry, or cacheable portal area, stashing enable. */ /* DE */
267 0 /* EQCR_CI stashing priority */ /* EP */
268 );
269 /* TODO: Switch to memory-backed mode. */
270 reg &= ~(1 << DPAA2_SWP_CFG_CPBS_SHIFT); /* QMan-backed mode */
271 }
272 dpaa2_swp_write_reg(p, DPAA2_SWP_CINH_CFG, reg);
273 reg = dpaa2_swp_read_reg(p, DPAA2_SWP_CINH_CFG);
274 if (!reg) {
275 free(p, M_DPAA2_SWP);
276 return (DPAA2_SWP_STAT_PORTAL_DISABLED);
277 }
278
279 /*
280 * Static Dequeue Command Register needs to be initialized to 0 when no
281 * channels are being dequeued from or else the QMan HW will indicate an
282 * error. The values that were calculated above will be applied when
283 * dequeues from a specific channel are enabled.
284 */
285 dpaa2_swp_write_reg(p, DPAA2_SWP_CINH_SDQCR, 0);
286
287 p->eqcr.pi_ring_size = 8;
288 /* if ((desc->swp_version & DPAA2_SWP_REV_MASK) >= DPAA2_SWP_REV_5000) */
289 /* p->eqcr.pi_ring_size = 32; */
290
291 for (mask_size = p->eqcr.pi_ring_size; mask_size > 0; mask_size >>= 1)
292 p->eqcr.pi_ci_mask = (p->eqcr.pi_ci_mask << 1) + 1;
293
294 eqcr_pi = dpaa2_swp_read_reg(p, DPAA2_SWP_CINH_EQCR_PI);
295 p->eqcr.pi = eqcr_pi & p->eqcr.pi_ci_mask;
296 p->eqcr.pi_vb = eqcr_pi & DPAA2_SWP_VALID_BIT;
297 p->eqcr.ci = dpaa2_swp_read_reg(p, DPAA2_SWP_CINH_EQCR_CI)
298 & p->eqcr.pi_ci_mask;
299 p->eqcr.available = p->eqcr.pi_ring_size;
300
301 /* Initialize the portal with an IRQ threshold and timeout of 0us. */
302 dpaa2_swp_set_irq_coalescing(p, p->dqrr.ring_size - 1, 0);
303
304 *swp = p;
305
306 return (0);
307 }
308
309 void
dpaa2_swp_free_portal(struct dpaa2_swp * swp)310 dpaa2_swp_free_portal(struct dpaa2_swp *swp)
311 {
312 uint16_t flags;
313
314 KASSERT(swp != NULL, ("%s: swp is NULL", __func__));
315
316 DPAA2_SWP_LOCK(swp, &flags);
317 swp->flags |= DPAA2_SWP_DESTROYED;
318 DPAA2_SWP_UNLOCK(swp);
319
320 /* Let threads stop using this portal. */
321 DELAY(DPAA2_SWP_TIMEOUT);
322
323 mtx_destroy(&swp->lock);
324 free(swp, M_DPAA2_SWP);
325 }
326
327 uint32_t
dpaa2_swp_set_cfg(uint8_t max_fill,uint8_t wn,uint8_t est,uint8_t rpm,uint8_t dcm,uint8_t epm,int sd,int sp,int se,int dp,int de,int ep)328 dpaa2_swp_set_cfg(uint8_t max_fill, uint8_t wn, uint8_t est, uint8_t rpm,
329 uint8_t dcm, uint8_t epm, int sd, int sp, int se, int dp, int de, int ep)
330 {
331 return (
332 max_fill << DPAA2_SWP_CFG_DQRR_MF_SHIFT |
333 est << DPAA2_SWP_CFG_EST_SHIFT |
334 wn << DPAA2_SWP_CFG_WN_SHIFT |
335 rpm << DPAA2_SWP_CFG_RPM_SHIFT |
336 dcm << DPAA2_SWP_CFG_DCM_SHIFT |
337 epm << DPAA2_SWP_CFG_EPM_SHIFT |
338 sd << DPAA2_SWP_CFG_SD_SHIFT |
339 sp << DPAA2_SWP_CFG_SP_SHIFT |
340 se << DPAA2_SWP_CFG_SE_SHIFT |
341 dp << DPAA2_SWP_CFG_DP_SHIFT |
342 de << DPAA2_SWP_CFG_DE_SHIFT |
343 ep << DPAA2_SWP_CFG_EP_SHIFT
344 );
345 }
346
347 /* Read/write registers of a software portal. */
348
349 void
dpaa2_swp_write_reg(struct dpaa2_swp * swp,uint32_t o,uint32_t v)350 dpaa2_swp_write_reg(struct dpaa2_swp *swp, uint32_t o, uint32_t v)
351 {
352 bus_write_4(swp->cinh_map, o, v);
353 }
354
355 uint32_t
dpaa2_swp_read_reg(struct dpaa2_swp * swp,uint32_t o)356 dpaa2_swp_read_reg(struct dpaa2_swp *swp, uint32_t o)
357 {
358 return (bus_read_4(swp->cinh_map, o));
359 }
360
361 /* Helper routines. */
362
363 /**
364 * @brief Set enqueue descriptor without Order Point Record ID.
365 *
366 * ed: Enqueue descriptor.
367 * resp_always: Enqueue with response always (1); FD from a rejected enqueue
368 * will be returned on a FQ (0).
369 */
370 void
dpaa2_swp_set_ed_norp(struct dpaa2_eq_desc * ed,bool resp_always)371 dpaa2_swp_set_ed_norp(struct dpaa2_eq_desc *ed, bool resp_always)
372 {
373 ed->verb &= ~(1 << ENQ_CMD_ORP_ENABLE_SHIFT);
374 if (resp_always)
375 ed->verb |= ENQ_CMD_RESPONSE_ALWAYS;
376 else
377 ed->verb |= ENQ_CMD_REJECTS_TO_FQ;
378 }
379
380 /**
381 * @brief Set FQ of the enqueue descriptor.
382 */
383 void
dpaa2_swp_set_ed_fq(struct dpaa2_eq_desc * ed,uint32_t fqid)384 dpaa2_swp_set_ed_fq(struct dpaa2_eq_desc *ed, uint32_t fqid)
385 {
386 ed->verb &= ~(1 << ENQ_CMD_TARGET_TYPE_SHIFT);
387 ed->tgtid = fqid;
388 }
389
390 /**
391 * @brief Enable interrupts for a software portal.
392 */
393 void
dpaa2_swp_set_intr_trigger(struct dpaa2_swp * swp,uint32_t mask)394 dpaa2_swp_set_intr_trigger(struct dpaa2_swp *swp, uint32_t mask)
395 {
396 if (swp != NULL)
397 dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_IER, mask);
398 }
399
400 /**
401 * @brief Return the value in the SWP_IER register.
402 */
403 uint32_t
dpaa2_swp_get_intr_trigger(struct dpaa2_swp * swp)404 dpaa2_swp_get_intr_trigger(struct dpaa2_swp *swp)
405 {
406 if (swp != NULL)
407 return dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_IER);
408 return (0);
409 }
410
411 /**
412 * @brief Return the value in the SWP_ISR register.
413 */
414 uint32_t
dpaa2_swp_read_intr_status(struct dpaa2_swp * swp)415 dpaa2_swp_read_intr_status(struct dpaa2_swp *swp)
416 {
417 if (swp != NULL)
418 return dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_ISR);
419 return (0);
420 }
421
422 /**
423 * @brief Clear SWP_ISR register according to the given mask.
424 */
425 void
dpaa2_swp_clear_intr_status(struct dpaa2_swp * swp,uint32_t mask)426 dpaa2_swp_clear_intr_status(struct dpaa2_swp *swp, uint32_t mask)
427 {
428 if (swp != NULL)
429 dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_ISR, mask);
430 }
431
432 /**
433 * @brief Enable or disable push dequeue.
434 *
435 * swp: the software portal object
436 * chan_idx: the channel index (0 to 15)
437 * en: enable or disable push dequeue
438 */
439 void
dpaa2_swp_set_push_dequeue(struct dpaa2_swp * swp,uint8_t chan_idx,bool en)440 dpaa2_swp_set_push_dequeue(struct dpaa2_swp *swp, uint8_t chan_idx, bool en)
441 {
442 uint16_t dqsrc;
443
444 if (swp != NULL) {
445 if (chan_idx > 15u) {
446 device_printf(swp->desc->dpio_dev, "channel index "
447 "should be <= 15: chan_idx=%d\n", chan_idx);
448 return;
449 }
450
451 if (en)
452 swp->sdq |= 1 << chan_idx;
453 else
454 swp->sdq &= ~(1 << chan_idx);
455 /*
456 * Read make the complete src map. If no channels are enabled
457 * the SDQCR must be 0 or else QMan will assert errors.
458 */
459 dqsrc = (swp->sdq >> DPAA2_SDQCR_SRC_SHIFT) &
460 DPAA2_SDQCR_SRC_MASK;
461 dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_SDQCR, dqsrc != 0
462 ? swp->sdq : 0);
463 }
464 }
465
466 /**
467 * @brief Set new IRQ coalescing values.
468 *
469 * swp: The software portal object.
470 * threshold: Threshold for DQRR interrupt generation. The DQRR interrupt
471 * asserts when the ring contains greater than "threshold" entries.
472 * holdoff: DQRR interrupt holdoff (timeout) period in us.
473 */
dpaa2_swp_set_irq_coalescing(struct dpaa2_swp * swp,uint32_t threshold,uint32_t holdoff)474 int dpaa2_swp_set_irq_coalescing(struct dpaa2_swp *swp, uint32_t threshold,
475 uint32_t holdoff)
476 {
477 uint32_t itp; /* Interrupt Timeout Period */
478
479 if (swp == NULL)
480 return (EINVAL);
481
482 /*
483 * Convert "holdoff" value from us to 256 QBMAN clock cycles
484 * increments. This depends on the QBMAN internal frequency.
485 */
486 itp = (holdoff * 1000u) / swp->desc->swp_cycles_ratio;
487 if (itp > DQRR_MAX_ITP)
488 itp = DQRR_MAX_ITP;
489 if (threshold >= swp->dqrr.ring_size)
490 threshold = swp->dqrr.ring_size - 1;
491
492 swp->dqrr.irq_threshold = threshold;
493 swp->dqrr.irq_itp = itp;
494
495 dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_DQRR_ITR, threshold);
496 dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_ITPR, itp);
497
498 return (0);
499 }
500
501 /*
502 * Software portal commands.
503 */
504
505 /**
506 * @brief Configure the channel data availability notification (CDAN)
507 * in a particular WQ channel.
508 */
509 int
dpaa2_swp_conf_wq_channel(struct dpaa2_swp * swp,uint16_t chan_id,uint8_t we_mask,bool cdan_en,uint64_t ctx)510 dpaa2_swp_conf_wq_channel(struct dpaa2_swp *swp, uint16_t chan_id,
511 uint8_t we_mask, bool cdan_en, uint64_t ctx)
512 {
513 /* NOTE: 64 bytes command. */
514 struct __packed {
515 uint8_t verb;
516 uint8_t result; /* in response only! */
517 uint16_t chan_id;
518 uint8_t we;
519 uint8_t ctrl;
520 uint16_t _reserved2;
521 uint64_t ctx;
522 uint8_t _reserved3[48];
523 } cmd = {0};
524 struct __packed {
525 uint8_t verb;
526 uint8_t result;
527 uint16_t chan_id;
528 uint8_t _reserved[60];
529 } rsp;
530 int error;
531
532 if (swp == NULL)
533 return (EINVAL);
534
535 cmd.chan_id = chan_id;
536 cmd.we = we_mask;
537 cmd.ctrl = cdan_en ? 1u : 0u;
538 cmd.ctx = ctx;
539
540 error = dpaa2_swp_exec_mgmt_command(swp, (struct dpaa2_swp_cmd *) &cmd,
541 (struct dpaa2_swp_rsp *) &rsp, CMDID_SWP_WQCHAN_CONFIGURE);
542 if (error)
543 return (error);
544
545 if (rsp.result != QBMAN_CMD_RC_OK) {
546 device_printf(swp->desc->dpio_dev, "WQ channel configuration "
547 "error: channel_id=%d, result=0x%02x\n", chan_id,
548 rsp.result);
549 return (EIO);
550 }
551
552 return (0);
553 }
554
555 /**
556 * @brief Query current configuration/state of the buffer pool.
557 */
558 int
dpaa2_swp_query_bp(struct dpaa2_swp * swp,uint16_t bpid,struct dpaa2_bp_conf * conf)559 dpaa2_swp_query_bp(struct dpaa2_swp *swp, uint16_t bpid,
560 struct dpaa2_bp_conf *conf)
561 {
562 /* NOTE: 64 bytes command. */
563 struct __packed {
564 uint8_t verb;
565 uint8_t _reserved1;
566 uint16_t bpid;
567 uint8_t _reserved2[60];
568 } cmd = {0};
569 struct __packed {
570 uint8_t verb;
571 uint8_t result;
572 uint32_t _reserved1;
573 uint8_t bdi;
574 uint8_t state;
575 uint32_t fill;
576 /* TODO: Support the other fields as well. */
577 uint8_t _reserved2[52];
578 } rsp;
579 int error;
580
581 if (swp == NULL || conf == NULL)
582 return (EINVAL);
583
584 cmd.bpid = bpid;
585
586 error = dpaa2_swp_exec_mgmt_command(swp, (struct dpaa2_swp_cmd *) &cmd,
587 (struct dpaa2_swp_rsp *) &rsp, CMDID_SWP_BP_QUERY);
588 if (error)
589 return (error);
590
591 if (rsp.result != QBMAN_CMD_RC_OK) {
592 device_printf(swp->desc->dpio_dev, "BP query error: bpid=%d, "
593 "result=0x%02x\n", bpid, rsp.result);
594 return (EIO);
595 }
596
597 conf->bdi = rsp.bdi;
598 conf->state = rsp.state;
599 conf->free_bufn = rsp.fill;
600
601 return (0);
602 }
603
604 int
dpaa2_swp_release_bufs(struct dpaa2_swp * swp,uint16_t bpid,bus_addr_t * buf,uint32_t buf_num)605 dpaa2_swp_release_bufs(struct dpaa2_swp *swp, uint16_t bpid, bus_addr_t *buf,
606 uint32_t buf_num)
607 {
608 /* NOTE: 64 bytes command. */
609 struct __packed {
610 uint8_t verb;
611 uint8_t _reserved1;
612 uint16_t bpid;
613 uint32_t _reserved2;
614 uint64_t buf[DPAA2_SWP_BUFS_PER_CMD];
615 } cmd = {0};
616 int error;
617
618 if (swp == NULL || buf == NULL || buf_num == 0u ||
619 buf_num > DPAA2_SWP_BUFS_PER_CMD)
620 return (EINVAL);
621
622 for (uint32_t i = 0; i < buf_num; i++)
623 cmd.buf[i] = buf[i];
624 cmd.bpid = bpid;
625 cmd.verb |= 1 << 5; /* Switch release buffer command to valid. */
626
627 error = dpaa2_swp_exec_br_command(swp, (struct dpaa2_swp_cmd *) &cmd,
628 buf_num);
629 if (error) {
630 device_printf(swp->desc->dpio_dev, "buffers release command "
631 "failed\n");
632 return (error);
633 }
634
635 return (0);
636 }
637
638 int
dpaa2_swp_dqrr_next_locked(struct dpaa2_swp * swp,struct dpaa2_dq * dq,uint32_t * idx)639 dpaa2_swp_dqrr_next_locked(struct dpaa2_swp *swp, struct dpaa2_dq *dq,
640 uint32_t *idx)
641 {
642 struct resource_map *map = swp->cinh_map;
643 struct dpaa2_swp_rsp *rsp = (struct dpaa2_swp_rsp *) dq;
644 uint32_t verb, pi; /* producer index */
645 uint32_t offset = swp->cfg.mem_backed
646 ? DPAA2_SWP_CENA_DQRR_MEM(swp->dqrr.next_idx)
647 : DPAA2_SWP_CENA_DQRR(swp->dqrr.next_idx);
648
649 if (swp == NULL || dq == NULL)
650 return (EINVAL);
651
652 /*
653 * Before using valid-bit to detect if something is there, we have to
654 * handle the case of the DQRR reset bug...
655 */
656 if (swp->dqrr.reset_bug) {
657 /*
658 * We pick up new entries by cache-inhibited producer index,
659 * which means that a non-coherent mapping would require us to
660 * invalidate and read *only* once that PI has indicated that
661 * there's an entry here. The first trip around the DQRR ring
662 * will be much less efficient than all subsequent trips around
663 * it...
664 */
665 pi = dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_DQPI) & DQRR_PI_MASK;
666
667 /* There are new entries if pi != next_idx */
668 if (pi == swp->dqrr.next_idx)
669 return (ENOENT);
670
671 /*
672 * If next_idx is/was the last ring index, and 'pi' is
673 * different, we can disable the workaround as all the ring
674 * entries have now been DMA'd to so valid-bit checking is
675 * repaired.
676 *
677 * NOTE: This logic needs to be based on next_idx (which
678 * increments one at a time), rather than on pi (which
679 * can burst and wrap-around between our snapshots of it).
680 */
681 if (swp->dqrr.next_idx == (swp->dqrr.ring_size - 1))
682 swp->dqrr.reset_bug = 0;
683 }
684
685 verb = bus_read_4(map, offset);
686 if ((verb & DPAA2_SWP_VALID_BIT) != swp->dqrr.valid_bit)
687 return (ENOENT);
688
689 /* Read dequeue response message. */
690 for (int i = 0; i < DPAA2_SWP_RSP_PARAMS_N; i++)
691 rsp->params[i] = bus_read_8(map, offset + i * sizeof(uint64_t));
692
693 /* Return index of the current entry (if requested). */
694 if (idx != NULL)
695 *idx = swp->dqrr.next_idx;
696
697 /*
698 * There's something there. Move "next_idx" attention to the next ring
699 * entry before returning what we found.
700 */
701 swp->dqrr.next_idx++;
702 swp->dqrr.next_idx &= swp->dqrr.ring_size - 1; /* wrap around */
703 if (swp->dqrr.next_idx == 0u)
704 swp->dqrr.valid_bit ^= DPAA2_SWP_VALID_BIT;
705
706 return (0);
707 }
708
709 int
dpaa2_swp_pull(struct dpaa2_swp * swp,uint16_t chan_id,struct dpaa2_buf * buf,uint32_t frames_n)710 dpaa2_swp_pull(struct dpaa2_swp *swp, uint16_t chan_id, struct dpaa2_buf *buf,
711 uint32_t frames_n)
712 {
713 /* NOTE: 64 bytes command. */
714 struct __packed {
715 uint8_t verb;
716 uint8_t numf;
717 uint8_t tok;
718 uint8_t _reserved;
719 uint32_t dq_src;
720 uint64_t rsp_addr;
721 uint64_t _reserved1[6];
722 } cmd = {0};
723 struct dpaa2_dq *msg;
724 uint16_t flags;
725 int i, error;
726
727 KASSERT(swp != NULL, ("%s: swp is NULL", __func__));
728 KASSERT(frames_n != 0u, ("%s: cannot pull zero frames", __func__));
729 KASSERT(frames_n <= 16u, ("%s: too much frames to pull", __func__));
730 KASSERT(buf->type == DPAA2_BUF_STORE, ("%s: not channel storage "
731 "buffer", __func__));
732
733 cmd.numf = frames_n - 1;
734 cmd.tok = DPAA2_SWP_VDQCR_TOKEN;
735 cmd.dq_src = chan_id;
736 cmd.rsp_addr = (uint64_t) buf->store.paddr;
737
738 /* Dequeue command type */
739 cmd.verb &= ~(1 << QB_VDQCR_VERB_DCT0_SHIFT);
740 cmd.verb |= (1 << QB_VDQCR_VERB_DCT1_SHIFT);
741 /* Dequeue from a specific software portal channel (ID's in DQ_SRC). */
742 cmd.verb &= ~(1 << QB_VDQCR_VERB_DT0_SHIFT);
743 cmd.verb &= ~(1 << QB_VDQCR_VERB_DT1_SHIFT);
744 /* Write the response to this command into memory (at the RSP_ADDR). */
745 cmd.verb |= (1 << QB_VDQCR_VERB_RLS_SHIFT);
746 /* Response writes won't attempt to allocate into a cache. */
747 cmd.verb &= ~(1 << QB_VDQCR_VERB_WAE_SHIFT);
748 /* Allow the FQ to remain active in the portal after dequeue. */
749 cmd.verb &= ~(1 << QB_VDQCR_VERB_RAD_SHIFT);
750
751 DPAA2_SWP_LOCK(swp, &flags);
752 if (flags & DPAA2_SWP_DESTROYED) {
753 /* Terminate operation if portal is destroyed. */
754 DPAA2_SWP_UNLOCK(swp);
755 return (ENOENT);
756 }
757
758 error = dpaa2_swp_exec_vdc_command_locked(swp,
759 (struct dpaa2_swp_cmd *) &cmd);
760 if (error != 0) {
761 DPAA2_SWP_UNLOCK(swp);
762 return (error);
763 }
764
765 /* Let's sync before reading VDQ response from QBMan. */
766 bus_dmamap_sync(buf->store.dmat, buf->store.dmap, BUS_DMASYNC_POSTREAD);
767
768 /* Read VDQ response from QBMan. */
769 msg = (struct dpaa2_dq *) buf->store.vaddr;
770 for (i = 1; i <= CMD_SPIN_ATTEMPTS; i++) {
771 if ((msg->fdr.desc.stat & DPAA2_DQ_STAT_VOLATILE) &&
772 (msg->fdr.desc.tok == DPAA2_SWP_VDQCR_TOKEN)) {
773 /* Reset token. */
774 msg->fdr.desc.tok = 0;
775 break;
776 }
777 DELAY(CMD_SPIN_TIMEOUT);
778 }
779 DPAA2_SWP_UNLOCK(swp);
780
781 /* Return an error on expired timeout. */
782 return (i > CMD_SPIN_ATTEMPTS ? ETIMEDOUT : 0);
783 }
784
785 /**
786 * @brief Issue a command to enqueue a frame using one enqueue descriptor.
787 *
788 * swp: Software portal used to send this command to.
789 * ed: Enqueue command descriptor.
790 * fd: Frame descriptor to enqueue.
791 */
792 int
dpaa2_swp_enq(struct dpaa2_swp * swp,struct dpaa2_eq_desc * ed,struct dpaa2_fd * fd)793 dpaa2_swp_enq(struct dpaa2_swp *swp, struct dpaa2_eq_desc *ed,
794 struct dpaa2_fd *fd)
795 {
796 uint32_t flags = 0;
797 int rc = dpaa2_swp_enq_mult(swp, ed, fd, &flags, 1);
798
799 return (rc >= 0 ? 0 : EBUSY);
800 }
801
802 /**
803 * @brief Issue a command to enqueue frames using one enqueue descriptor.
804 *
805 * swp: Software portal used to send this command to.
806 * ed: Enqueue command descriptor.
807 * fd: Frame descriptor to enqueue.
808 * flags: Table pointer of QBMAN_ENQUEUE_FLAG_DCA flags, not used if NULL.
809 * frames_n: Number of FDs to enqueue.
810 *
811 * NOTE: Enqueue command (64 bytes): 32 (eq. descriptor) + 32 (frame descriptor).
812 */
813 int
dpaa2_swp_enq_mult(struct dpaa2_swp * swp,struct dpaa2_eq_desc * ed,struct dpaa2_fd * fd,uint32_t * flags,int frames_n)814 dpaa2_swp_enq_mult(struct dpaa2_swp *swp, struct dpaa2_eq_desc *ed,
815 struct dpaa2_fd *fd, uint32_t *flags, int frames_n)
816 {
817 const uint8_t *ed_pdat8 = (const uint8_t *) ed;
818 const uint32_t *ed_pdat32 = (const uint32_t *) ed;
819 const uint64_t *ed_pdat64 = (const uint64_t *) ed;
820 const uint64_t *fd_pdat64 = (const uint64_t *) fd;
821 struct resource_map *map;
822 uint32_t eqcr_ci, eqcr_pi; /* EQCR consumer/producer index */
823 uint32_t half_mask, full_mask, val, ci_offset;
824 uint16_t swp_flags;
825 int num_enq = 0;
826
827 if (swp == NULL || ed == NULL || fd == NULL || flags == NULL ||
828 frames_n == 0)
829 return (EINVAL);
830
831 DPAA2_SWP_LOCK(swp, &swp_flags);
832 if (swp_flags & DPAA2_SWP_DESTROYED) {
833 /* Terminate operation if portal is destroyed. */
834 DPAA2_SWP_UNLOCK(swp);
835 return (ENOENT);
836 }
837
838 map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map;
839 ci_offset = swp->cfg.mem_backed
840 ? DPAA2_SWP_CENA_EQCR_CI_MEMBACK
841 : DPAA2_SWP_CENA_EQCR_CI;
842
843 half_mask = swp->eqcr.pi_ci_mask >> 1;
844 full_mask = swp->eqcr.pi_ci_mask;
845
846 if (swp->eqcr.available == 0) {
847 val = dpaa2_swp_read_reg(swp, ci_offset);
848 eqcr_ci = swp->eqcr.ci;
849 swp->eqcr.ci = val & full_mask;
850
851 swp->eqcr.available = dpaa2_swp_cyc_diff(swp->eqcr.pi_ring_size,
852 eqcr_ci, swp->eqcr.ci);
853
854 if (swp->eqcr.available == 0) {
855 DPAA2_SWP_UNLOCK(swp);
856 return (0);
857 }
858 }
859
860 eqcr_pi = swp->eqcr.pi;
861 num_enq = swp->eqcr.available < frames_n
862 ? swp->eqcr.available : frames_n;
863 swp->eqcr.available -= num_enq;
864
865 KASSERT(num_enq >= 0 && num_enq <= swp->eqcr.pi_ring_size,
866 ("%s: unexpected num_enq=%d", __func__, num_enq));
867 KASSERT(swp->eqcr.available >= 0 &&
868 swp->eqcr.available <= swp->eqcr.pi_ring_size,
869 ("%s: unexpected eqcr.available=%d", __func__, swp->eqcr.available));
870
871 /* Fill in the EQCR ring. */
872 for (int i = 0; i < num_enq; i++) {
873 /* Write enq. desc. without the VERB, DCA, SEQNUM and OPRID. */
874 for (int j = 1; j <= 3; j++)
875 bus_write_8(map,
876 DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) +
877 sizeof(uint64_t) * j, ed_pdat64[j]);
878 /* Write OPRID. */
879 bus_write_4(map,
880 DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) + sizeof(uint32_t),
881 ed_pdat32[1]);
882 /* Write DCA and SEQNUM without VERB byte. */
883 for (int j = 1; j <= 3; j++)
884 bus_write_1(map,
885 DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) +
886 sizeof(uint8_t) * j, ed_pdat8[j]);
887
888 /* Write frame descriptor. */
889 for (int j = 0; j <= 3; j++)
890 bus_write_8(map,
891 DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) +
892 ENQ_DESC_FD_OFFSET +
893 sizeof(uint64_t) * j, fd_pdat64[j]);
894 eqcr_pi++;
895 }
896
897 wmb();
898
899 /* Write the VERB byte of enqueue descriptor. */
900 eqcr_pi = swp->eqcr.pi;
901 for (int i = 0; i < num_enq; i++) {
902 bus_write_1(map,
903 DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask),
904 ed_pdat8[0] | swp->eqcr.pi_vb);
905
906 if (flags && (flags[i] & ENQ_FLAG_DCA)) {
907 /* Update DCA byte. */
908 bus_write_1(map,
909 DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) + 1,
910 (1 << ENQ_CMD_DCA_EN_SHIFT) |
911 (flags[i] & ENQ_DCA_IDXMASK));
912 }
913 eqcr_pi++;
914 if (!(eqcr_pi & half_mask))
915 swp->eqcr.pi_vb ^= DPAA2_SWP_VALID_BIT;
916 }
917 swp->eqcr.pi = eqcr_pi & full_mask;
918
919 DPAA2_SWP_UNLOCK(swp);
920
921 return (num_enq);
922 }
923
924 static int
dpaa2_swp_cyc_diff(uint8_t ringsize,uint8_t first,uint8_t last)925 dpaa2_swp_cyc_diff(uint8_t ringsize, uint8_t first, uint8_t last)
926 {
927 /* 'first' is included, 'last' is excluded */
928 return ((first <= last)
929 ? (last - first) : ((2 * ringsize) - (first - last)));
930 }
931
932 /**
933 * @brief Execute Buffer Release Command (BRC).
934 */
935 static int
dpaa2_swp_exec_br_command(struct dpaa2_swp * swp,struct dpaa2_swp_cmd * cmd,uint32_t buf_num)936 dpaa2_swp_exec_br_command(struct dpaa2_swp *swp, struct dpaa2_swp_cmd *cmd,
937 uint32_t buf_num)
938 {
939 struct __packed with_verb {
940 uint8_t verb;
941 uint8_t _reserved[63];
942 } *c;
943 const uint8_t *cmd_pdat8 = (const uint8_t *) cmd->params;
944 const uint32_t *cmd_pdat32 = (const uint32_t *) cmd->params;
945 struct resource_map *map;
946 uint32_t offset, rar; /* Release Array Allocation register */
947 uint16_t flags;
948
949 if (!swp || !cmd)
950 return (EINVAL);
951
952 DPAA2_SWP_LOCK(swp, &flags);
953 if (flags & DPAA2_SWP_DESTROYED) {
954 /* Terminate operation if portal is destroyed. */
955 DPAA2_SWP_UNLOCK(swp);
956 return (ENOENT);
957 }
958
959 rar = dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_RAR);
960 if (!RAR_SUCCESS(rar)) {
961 DPAA2_SWP_UNLOCK(swp);
962 return (EBUSY);
963 }
964
965 map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map;
966 offset = swp->cfg.mem_backed
967 ? DPAA2_SWP_CENA_RCR_MEM(RAR_IDX(rar))
968 : DPAA2_SWP_CENA_RCR(RAR_IDX(rar));
969 c = (struct with_verb *) cmd;
970
971 /* Write command bytes (without VERB byte). */
972 for (uint32_t i = 1; i < DPAA2_SWP_CMD_PARAMS_N; i++)
973 bus_write_8(map, offset + sizeof(uint64_t) * i, cmd->params[i]);
974 bus_write_4(map, offset + 4, cmd_pdat32[1]);
975 for (uint32_t i = 1; i <= 3; i++)
976 bus_write_1(map, offset + i, cmd_pdat8[i]);
977
978 /* Write VERB byte and trigger command execution. */
979 if (swp->cfg.mem_backed) {
980 bus_write_1(map, offset, c->verb | RAR_VB(rar) | buf_num);
981 wmb();
982 dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_RCR_AM_RT +
983 RAR_IDX(rar) * 4, DPAA2_SWP_RT_MODE);
984 } else {
985 wmb();
986 bus_write_1(map, offset, c->verb | RAR_VB(rar) | buf_num);
987 }
988
989 DPAA2_SWP_UNLOCK(swp);
990
991 return (0);
992 }
993
994 /**
995 * @brief Execute Volatile Dequeue Command (VDC).
996 *
997 * This command will be executed by QBMan only once in order to deliver requested
998 * number of frames (1-16 or 1-32 depending on QBMan version) to the driver via
999 * DQRR or arbitrary DMA-mapped memory.
1000 *
1001 * NOTE: There is a counterpart to the volatile dequeue command called static
1002 * dequeue command (SDQC) which is executed periodically all the time the
1003 * command is present in the SDQCR register.
1004 */
1005 static int
dpaa2_swp_exec_vdc_command_locked(struct dpaa2_swp * swp,struct dpaa2_swp_cmd * cmd)1006 dpaa2_swp_exec_vdc_command_locked(struct dpaa2_swp *swp,
1007 struct dpaa2_swp_cmd *cmd)
1008 {
1009 struct __packed with_verb {
1010 uint8_t verb;
1011 uint8_t _reserved[63];
1012 } *c;
1013 const uint8_t *p8 = (const uint8_t *) cmd->params;
1014 const uint32_t *p32 = (const uint32_t *) cmd->params;
1015 struct resource_map *map;
1016 uint32_t offset;
1017
1018 map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map;
1019 offset = swp->cfg.mem_backed
1020 ? DPAA2_SWP_CENA_VDQCR_MEM : DPAA2_SWP_CENA_VDQCR;
1021 c = (struct with_verb *) cmd;
1022
1023 /* Write command bytes (without VERB byte). */
1024 for (uint32_t i = 1; i < DPAA2_SWP_CMD_PARAMS_N; i++)
1025 bus_write_8(map, offset + sizeof(uint64_t) * i, cmd->params[i]);
1026 bus_write_4(map, offset + 4, p32[1]);
1027 for (uint32_t i = 1; i <= 3; i++)
1028 bus_write_1(map, offset + i, p8[i]);
1029
1030 /* Write VERB byte and trigger command execution. */
1031 if (swp->cfg.mem_backed) {
1032 bus_write_1(map, offset, c->verb | swp->vdq.valid_bit);
1033 swp->vdq.valid_bit ^= DPAA2_SWP_VALID_BIT;
1034 wmb();
1035 dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_VDQCR_RT,
1036 DPAA2_SWP_RT_MODE);
1037 } else {
1038 wmb();
1039 bus_write_1(map, offset, c->verb | swp->vdq.valid_bit);
1040 swp->vdq.valid_bit ^= DPAA2_SWP_VALID_BIT;
1041 }
1042
1043 return (0);
1044 }
1045
1046 /**
1047 * @brief Execute a QBMan management command.
1048 */
1049 static int
dpaa2_swp_exec_mgmt_command(struct dpaa2_swp * swp,struct dpaa2_swp_cmd * cmd,struct dpaa2_swp_rsp * rsp,uint8_t cmdid)1050 dpaa2_swp_exec_mgmt_command(struct dpaa2_swp *swp, struct dpaa2_swp_cmd *cmd,
1051 struct dpaa2_swp_rsp *rsp, uint8_t cmdid)
1052 {
1053 #if (defined(_KERNEL) && defined(INVARIANTS))
1054 struct __packed with_verb {
1055 uint8_t verb;
1056 uint8_t _reserved[63];
1057 } *r;
1058 #endif
1059 uint16_t flags;
1060 int error;
1061
1062 if (swp == NULL || cmd == NULL || rsp == NULL)
1063 return (EINVAL);
1064
1065 DPAA2_SWP_LOCK(swp, &flags);
1066 if (flags & DPAA2_SWP_DESTROYED) {
1067 /* Terminate operation if portal is destroyed. */
1068 DPAA2_SWP_UNLOCK(swp);
1069 return (ENOENT);
1070 }
1071
1072 /*
1073 * Send a command to QBMan using Management Command register and wait
1074 * for response from the Management Response registers.
1075 */
1076 dpaa2_swp_send_mgmt_command(swp, cmd, cmdid);
1077 error = dpaa2_swp_wait_for_mgmt_response(swp, rsp);
1078 if (error) {
1079 DPAA2_SWP_UNLOCK(swp);
1080 return (error);
1081 }
1082 DPAA2_SWP_UNLOCK(swp);
1083
1084 #if (defined(_KERNEL) && defined(INVARIANTS))
1085 r = (struct with_verb *) rsp;
1086 KASSERT((r->verb & CMD_VERB_MASK) == cmdid,
1087 ("wrong VERB byte in response: resp=0x%02x, expected=0x%02x",
1088 r->verb, cmdid));
1089 #endif
1090
1091 return (0);
1092 }
1093
1094 static int
dpaa2_swp_send_mgmt_command(struct dpaa2_swp * swp,struct dpaa2_swp_cmd * cmd,uint8_t cmdid)1095 dpaa2_swp_send_mgmt_command(struct dpaa2_swp *swp, struct dpaa2_swp_cmd *cmd,
1096 uint8_t cmdid)
1097 {
1098 const uint8_t *cmd_pdat8 = (const uint8_t *) cmd->params;
1099 const uint32_t *cmd_pdat32 = (const uint32_t *) cmd->params;
1100 struct resource_map *map;
1101 uint32_t offset;
1102
1103 map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map;
1104 offset = swp->cfg.mem_backed ? DPAA2_SWP_CENA_CR_MEM : DPAA2_SWP_CENA_CR;
1105
1106 /* Write command bytes (without VERB byte). */
1107 for (uint32_t i = 1; i < DPAA2_SWP_CMD_PARAMS_N; i++)
1108 bus_write_8(map, offset + sizeof(uint64_t) * i, cmd->params[i]);
1109 bus_write_4(map, offset + 4, cmd_pdat32[1]);
1110 for (uint32_t i = 1; i <= 3; i++)
1111 bus_write_1(map, offset + i, cmd_pdat8[i]);
1112
1113 /* Write VERB byte and trigger command execution. */
1114 if (swp->cfg.mem_backed) {
1115 bus_write_1(map, offset, cmdid | swp->mr.valid_bit);
1116 wmb();
1117 dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_CR_RT,
1118 DPAA2_SWP_RT_MODE);
1119 } else {
1120 wmb();
1121 bus_write_1(map, offset, cmdid | swp->mc.valid_bit);
1122 }
1123
1124 return (0);
1125 }
1126
1127 static int
dpaa2_swp_wait_for_mgmt_response(struct dpaa2_swp * swp,struct dpaa2_swp_rsp * rsp)1128 dpaa2_swp_wait_for_mgmt_response(struct dpaa2_swp *swp, struct dpaa2_swp_rsp *rsp)
1129 {
1130 struct resource_map *map = swp->cfg.mem_backed
1131 ? swp->cena_map : swp->cinh_map;
1132 /* Management command response to be read from the only RR or RR0/RR1. */
1133 const uint32_t offset = swp->cfg.mem_backed
1134 ? DPAA2_SWP_CENA_RR_MEM
1135 : DPAA2_SWP_CENA_RR(swp->mc.valid_bit);
1136 uint32_t i, verb, ret;
1137 int rc;
1138
1139 /* Wait for a command response from QBMan. */
1140 for (i = 1; i <= CMD_SPIN_ATTEMPTS; i++) {
1141 if (swp->cfg.mem_backed) {
1142 verb = (uint32_t) (bus_read_4(map, offset) & 0xFFu);
1143 if (swp->mr.valid_bit != (verb & DPAA2_SWP_VALID_BIT))
1144 goto wait;
1145 if (!(verb & ~DPAA2_SWP_VALID_BIT))
1146 goto wait;
1147 swp->mr.valid_bit ^= DPAA2_SWP_VALID_BIT;
1148 } else {
1149 ret = bus_read_4(map, offset);
1150 verb = ret & ~DPAA2_SWP_VALID_BIT; /* remove valid bit */
1151 if (verb == 0u)
1152 goto wait;
1153 swp->mc.valid_bit ^= DPAA2_SWP_VALID_BIT;
1154 }
1155 break;
1156 wait:
1157 DELAY(CMD_SPIN_TIMEOUT);
1158 }
1159 /* Return an error on expired timeout. */
1160 rc = i > CMD_SPIN_ATTEMPTS ? ETIMEDOUT : 0;
1161
1162 /* Read command response. */
1163 for (i = 0; i < DPAA2_SWP_RSP_PARAMS_N; i++)
1164 rsp->params[i] = bus_read_8(map, offset + i * sizeof(uint64_t));
1165
1166 return (rc);
1167 }
1168