1 /*-
2 * Copyright (c) 2007-2015 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
29 */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33
34 #include "efsys.h"
35 #include "efx.h"
36 #include "efx_types.h"
37 #include "efx_regs.h"
38 #include "efx_impl.h"
39
40 #if EFSYS_OPT_QSTATS
41 #define EFX_TX_QSTAT_INCR(_etp, _stat) \
42 do { \
43 (_etp)->et_stat[_stat]++; \
44 _NOTE(CONSTANTCONDITION) \
45 } while (B_FALSE)
46 #else
47 #define EFX_TX_QSTAT_INCR(_etp, _stat)
48 #endif
49
50 #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
51
52 static __checkReturn efx_rc_t
53 falconsiena_tx_init(
54 __in efx_nic_t *enp);
55
56 static void
57 falconsiena_tx_fini(
58 __in efx_nic_t *enp);
59
60 static __checkReturn efx_rc_t
61 falconsiena_tx_qcreate(
62 __in efx_nic_t *enp,
63 __in unsigned int index,
64 __in unsigned int label,
65 __in efsys_mem_t *esmp,
66 __in size_t n,
67 __in uint32_t id,
68 __in uint16_t flags,
69 __in efx_evq_t *eep,
70 __in efx_txq_t *etp,
71 __out unsigned int *addedp);
72
73 static void
74 falconsiena_tx_qdestroy(
75 __in efx_txq_t *etp);
76
77 static __checkReturn efx_rc_t
78 falconsiena_tx_qpost(
79 __in efx_txq_t *etp,
80 __in_ecount(n) efx_buffer_t *eb,
81 __in unsigned int n,
82 __in unsigned int completed,
83 __inout unsigned int *addedp);
84
85 static void
86 falconsiena_tx_qpush(
87 __in efx_txq_t *etp,
88 __in unsigned int added,
89 __in unsigned int pushed);
90
91 static __checkReturn efx_rc_t
92 falconsiena_tx_qpace(
93 __in efx_txq_t *etp,
94 __in unsigned int ns);
95
96 static __checkReturn efx_rc_t
97 falconsiena_tx_qflush(
98 __in efx_txq_t *etp);
99
100 static void
101 falconsiena_tx_qenable(
102 __in efx_txq_t *etp);
103
104 __checkReturn efx_rc_t
105 falconsiena_tx_qdesc_post(
106 __in efx_txq_t *etp,
107 __in_ecount(n) efx_desc_t *ed,
108 __in unsigned int n,
109 __in unsigned int completed,
110 __inout unsigned int *addedp);
111
112 void
113 falconsiena_tx_qdesc_dma_create(
114 __in efx_txq_t *etp,
115 __in efsys_dma_addr_t addr,
116 __in size_t size,
117 __in boolean_t eop,
118 __out efx_desc_t *edp);
119
120 #if EFSYS_OPT_QSTATS
121 static void
122 falconsiena_tx_qstats_update(
123 __in efx_txq_t *etp,
124 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
125 #endif
126
127 #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
128
129
130 #if EFSYS_OPT_FALCON
131 static efx_tx_ops_t __efx_tx_falcon_ops = {
132 falconsiena_tx_init, /* etxo_init */
133 falconsiena_tx_fini, /* etxo_fini */
134 falconsiena_tx_qcreate, /* etxo_qcreate */
135 falconsiena_tx_qdestroy, /* etxo_qdestroy */
136 falconsiena_tx_qpost, /* etxo_qpost */
137 falconsiena_tx_qpush, /* etxo_qpush */
138 falconsiena_tx_qpace, /* etxo_qpace */
139 falconsiena_tx_qflush, /* etxo_qflush */
140 falconsiena_tx_qenable, /* etxo_qenable */
141 NULL, /* etxo_qpio_enable */
142 NULL, /* etxo_qpio_disable */
143 NULL, /* etxo_qpio_write */
144 NULL, /* etxo_qpio_post */
145 falconsiena_tx_qdesc_post, /* etxo_qdesc_post */
146 falconsiena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
147 NULL, /* etxo_qdesc_tso_create */
148 NULL, /* etxo_qdesc_vlantci_create */
149 #if EFSYS_OPT_QSTATS
150 falconsiena_tx_qstats_update, /* etxo_qstats_update */
151 #endif
152 };
153 #endif /* EFSYS_OPT_FALCON */
154
155 #if EFSYS_OPT_SIENA
156 static efx_tx_ops_t __efx_tx_siena_ops = {
157 falconsiena_tx_init, /* etxo_init */
158 falconsiena_tx_fini, /* etxo_fini */
159 falconsiena_tx_qcreate, /* etxo_qcreate */
160 falconsiena_tx_qdestroy, /* etxo_qdestroy */
161 falconsiena_tx_qpost, /* etxo_qpost */
162 falconsiena_tx_qpush, /* etxo_qpush */
163 falconsiena_tx_qpace, /* etxo_qpace */
164 falconsiena_tx_qflush, /* etxo_qflush */
165 falconsiena_tx_qenable, /* etxo_qenable */
166 NULL, /* etxo_qpio_enable */
167 NULL, /* etxo_qpio_disable */
168 NULL, /* etxo_qpio_write */
169 NULL, /* etxo_qpio_post */
170 falconsiena_tx_qdesc_post, /* etxo_qdesc_post */
171 falconsiena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
172 NULL, /* etxo_qdesc_tso_create */
173 NULL, /* etxo_qdesc_vlantci_create */
174 #if EFSYS_OPT_QSTATS
175 falconsiena_tx_qstats_update, /* etxo_qstats_update */
176 #endif
177 };
178 #endif /* EFSYS_OPT_SIENA */
179
180 #if EFSYS_OPT_HUNTINGTON
181 static efx_tx_ops_t __efx_tx_hunt_ops = {
182 hunt_tx_init, /* etxo_init */
183 hunt_tx_fini, /* etxo_fini */
184 hunt_tx_qcreate, /* etxo_qcreate */
185 hunt_tx_qdestroy, /* etxo_qdestroy */
186 hunt_tx_qpost, /* etxo_qpost */
187 hunt_tx_qpush, /* etxo_qpush */
188 hunt_tx_qpace, /* etxo_qpace */
189 hunt_tx_qflush, /* etxo_qflush */
190 hunt_tx_qenable, /* etxo_qenable */
191 hunt_tx_qpio_enable, /* etxo_qpio_enable */
192 hunt_tx_qpio_disable, /* etxo_qpio_disable */
193 hunt_tx_qpio_write, /* etxo_qpio_write */
194 hunt_tx_qpio_post, /* etxo_qpio_post */
195 hunt_tx_qdesc_post, /* etxo_qdesc_post */
196 hunt_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
197 hunt_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
198 hunt_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
199 #if EFSYS_OPT_QSTATS
200 hunt_tx_qstats_update, /* etxo_qstats_update */
201 #endif
202 };
203 #endif /* EFSYS_OPT_HUNTINGTON */
204
205 __checkReturn efx_rc_t
efx_tx_init(__in efx_nic_t * enp)206 efx_tx_init(
207 __in efx_nic_t *enp)
208 {
209 efx_tx_ops_t *etxop;
210 efx_rc_t rc;
211
212 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
213 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
214
215 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
216 rc = EINVAL;
217 goto fail1;
218 }
219
220 if (enp->en_mod_flags & EFX_MOD_TX) {
221 rc = EINVAL;
222 goto fail2;
223 }
224
225 switch (enp->en_family) {
226 #if EFSYS_OPT_FALCON
227 case EFX_FAMILY_FALCON:
228 etxop = (efx_tx_ops_t *)&__efx_tx_falcon_ops;
229 break;
230 #endif /* EFSYS_OPT_FALCON */
231
232 #if EFSYS_OPT_SIENA
233 case EFX_FAMILY_SIENA:
234 etxop = (efx_tx_ops_t *)&__efx_tx_siena_ops;
235 break;
236 #endif /* EFSYS_OPT_SIENA */
237
238 #if EFSYS_OPT_HUNTINGTON
239 case EFX_FAMILY_HUNTINGTON:
240 etxop = (efx_tx_ops_t *)&__efx_tx_hunt_ops;
241 break;
242 #endif /* EFSYS_OPT_HUNTINGTON */
243
244 default:
245 EFSYS_ASSERT(0);
246 rc = ENOTSUP;
247 goto fail3;
248 }
249
250 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
251
252 if ((rc = etxop->etxo_init(enp)) != 0)
253 goto fail4;
254
255 enp->en_etxop = etxop;
256 enp->en_mod_flags |= EFX_MOD_TX;
257 return (0);
258
259 fail4:
260 EFSYS_PROBE(fail4);
261 fail3:
262 EFSYS_PROBE(fail3);
263 fail2:
264 EFSYS_PROBE(fail2);
265 fail1:
266 EFSYS_PROBE1(fail1, efx_rc_t, rc);
267
268 enp->en_etxop = NULL;
269 enp->en_mod_flags &= ~EFX_MOD_TX;
270 return (rc);
271 }
272
273 void
efx_tx_fini(__in efx_nic_t * enp)274 efx_tx_fini(
275 __in efx_nic_t *enp)
276 {
277 efx_tx_ops_t *etxop = enp->en_etxop;
278
279 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
280 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
281 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
282 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
283
284 etxop->etxo_fini(enp);
285
286 enp->en_etxop = NULL;
287 enp->en_mod_flags &= ~EFX_MOD_TX;
288 }
289
290 __checkReturn efx_rc_t
efx_tx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efsys_mem_t * esmp,__in size_t n,__in uint32_t id,__in uint16_t flags,__in efx_evq_t * eep,__deref_out efx_txq_t ** etpp,__out unsigned int * addedp)291 efx_tx_qcreate(
292 __in efx_nic_t *enp,
293 __in unsigned int index,
294 __in unsigned int label,
295 __in efsys_mem_t *esmp,
296 __in size_t n,
297 __in uint32_t id,
298 __in uint16_t flags,
299 __in efx_evq_t *eep,
300 __deref_out efx_txq_t **etpp,
301 __out unsigned int *addedp)
302 {
303 efx_tx_ops_t *etxop = enp->en_etxop;
304 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
305 efx_txq_t *etp;
306 efx_rc_t rc;
307
308 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
309 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
310
311 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit);
312
313 /* Allocate an TXQ object */
314 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
315
316 if (etp == NULL) {
317 rc = ENOMEM;
318 goto fail1;
319 }
320
321 etp->et_magic = EFX_TXQ_MAGIC;
322 etp->et_enp = enp;
323 etp->et_index = index;
324 etp->et_mask = n - 1;
325 etp->et_esmp = esmp;
326
327 /* Initial descriptor index may be modified by etxo_qcreate */
328 *addedp = 0;
329
330 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
331 n, id, flags, eep, etp, addedp)) != 0)
332 goto fail2;
333
334 enp->en_tx_qcount++;
335 *etpp = etp;
336
337 return (0);
338
339 fail2:
340 EFSYS_PROBE(fail2);
341 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
342 fail1:
343 EFSYS_PROBE1(fail1, efx_rc_t, rc);
344 return (rc);
345 }
346
347 void
efx_tx_qdestroy(__in efx_txq_t * etp)348 efx_tx_qdestroy(
349 __in efx_txq_t *etp)
350 {
351 efx_nic_t *enp = etp->et_enp;
352 efx_tx_ops_t *etxop = enp->en_etxop;
353
354 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
355
356 EFSYS_ASSERT(enp->en_tx_qcount != 0);
357 --enp->en_tx_qcount;
358
359 etxop->etxo_qdestroy(etp);
360
361 /* Free the TXQ object */
362 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
363 }
364
365 __checkReturn efx_rc_t
efx_tx_qpost(__in efx_txq_t * etp,__in_ecount (n)efx_buffer_t * eb,__in unsigned int n,__in unsigned int completed,__inout unsigned int * addedp)366 efx_tx_qpost(
367 __in efx_txq_t *etp,
368 __in_ecount(n) efx_buffer_t *eb,
369 __in unsigned int n,
370 __in unsigned int completed,
371 __inout unsigned int *addedp)
372 {
373 efx_nic_t *enp = etp->et_enp;
374 efx_tx_ops_t *etxop = enp->en_etxop;
375 efx_rc_t rc;
376
377 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
378
379 if ((rc = etxop->etxo_qpost(etp, eb,
380 n, completed, addedp)) != 0)
381 goto fail1;
382
383 return (0);
384
385 fail1:
386 EFSYS_PROBE1(fail1, efx_rc_t, rc);
387 return (rc);
388 }
389
390 void
efx_tx_qpush(__in efx_txq_t * etp,__in unsigned int added,__in unsigned int pushed)391 efx_tx_qpush(
392 __in efx_txq_t *etp,
393 __in unsigned int added,
394 __in unsigned int pushed)
395 {
396 efx_nic_t *enp = etp->et_enp;
397 efx_tx_ops_t *etxop = enp->en_etxop;
398
399 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
400
401 etxop->etxo_qpush(etp, added, pushed);
402 }
403
404 __checkReturn efx_rc_t
efx_tx_qpace(__in efx_txq_t * etp,__in unsigned int ns)405 efx_tx_qpace(
406 __in efx_txq_t *etp,
407 __in unsigned int ns)
408 {
409 efx_nic_t *enp = etp->et_enp;
410 efx_tx_ops_t *etxop = enp->en_etxop;
411 efx_rc_t rc;
412
413 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
414
415 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
416 goto fail1;
417
418 return (0);
419
420 fail1:
421 EFSYS_PROBE1(fail1, efx_rc_t, rc);
422 return (rc);
423 }
424
425 __checkReturn efx_rc_t
efx_tx_qflush(__in efx_txq_t * etp)426 efx_tx_qflush(
427 __in efx_txq_t *etp)
428 {
429 efx_nic_t *enp = etp->et_enp;
430 efx_tx_ops_t *etxop = enp->en_etxop;
431 efx_rc_t rc;
432
433 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
434
435 if ((rc = etxop->etxo_qflush(etp)) != 0)
436 goto fail1;
437
438 return (0);
439
440 fail1:
441 EFSYS_PROBE1(fail1, efx_rc_t, rc);
442 return (rc);
443 }
444
445 void
efx_tx_qenable(__in efx_txq_t * etp)446 efx_tx_qenable(
447 __in efx_txq_t *etp)
448 {
449 efx_nic_t *enp = etp->et_enp;
450 efx_tx_ops_t *etxop = enp->en_etxop;
451
452 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
453
454 etxop->etxo_qenable(etp);
455 }
456
457 __checkReturn efx_rc_t
efx_tx_qpio_enable(__in efx_txq_t * etp)458 efx_tx_qpio_enable(
459 __in efx_txq_t *etp)
460 {
461 efx_nic_t *enp = etp->et_enp;
462 efx_tx_ops_t *etxop = enp->en_etxop;
463 efx_rc_t rc;
464
465 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
466
467 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
468 rc = ENOTSUP;
469 goto fail1;
470 }
471 if (etxop->etxo_qpio_enable == NULL) {
472 rc = ENOTSUP;
473 goto fail2;
474 }
475 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
476 goto fail3;
477
478 return (0);
479
480 fail3:
481 EFSYS_PROBE(fail3);
482 fail2:
483 EFSYS_PROBE(fail2);
484 fail1:
485 EFSYS_PROBE1(fail1, efx_rc_t, rc);
486 return (rc);
487 }
488
489 void
efx_tx_qpio_disable(__in efx_txq_t * etp)490 efx_tx_qpio_disable(
491 __in efx_txq_t *etp)
492 {
493 efx_nic_t *enp = etp->et_enp;
494 efx_tx_ops_t *etxop = enp->en_etxop;
495
496 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
497
498 if (etxop->etxo_qpio_disable != NULL)
499 etxop->etxo_qpio_disable(etp);
500 }
501
502 __checkReturn efx_rc_t
efx_tx_qpio_write(__in efx_txq_t * etp,__in_ecount (buf_length)uint8_t * buffer,__in size_t buf_length,__in size_t pio_buf_offset)503 efx_tx_qpio_write(
504 __in efx_txq_t *etp,
505 __in_ecount(buf_length) uint8_t *buffer,
506 __in size_t buf_length,
507 __in size_t pio_buf_offset)
508 {
509 efx_nic_t *enp = etp->et_enp;
510 efx_tx_ops_t *etxop = enp->en_etxop;
511 efx_rc_t rc;
512
513 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
514
515 if (etxop->etxo_qpio_write != NULL) {
516 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
517 pio_buf_offset)) != 0)
518 goto fail1;
519 return (0);
520 }
521
522 return (ENOTSUP);
523
524 fail1:
525 EFSYS_PROBE1(fail1, efx_rc_t, rc);
526 return (rc);
527 }
528
529 __checkReturn efx_rc_t
efx_tx_qpio_post(__in efx_txq_t * etp,__in size_t pkt_length,__in unsigned int completed,__inout unsigned int * addedp)530 efx_tx_qpio_post(
531 __in efx_txq_t *etp,
532 __in size_t pkt_length,
533 __in unsigned int completed,
534 __inout unsigned int *addedp)
535 {
536 efx_nic_t *enp = etp->et_enp;
537 efx_tx_ops_t *etxop = enp->en_etxop;
538 efx_rc_t rc;
539
540 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
541
542 if (etxop->etxo_qpio_post != NULL) {
543 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
544 addedp)) != 0)
545 goto fail1;
546 return (0);
547 }
548
549 return (ENOTSUP);
550
551 fail1:
552 EFSYS_PROBE1(fail1, efx_rc_t, rc);
553 return (rc);
554 }
555
556 __checkReturn efx_rc_t
efx_tx_qdesc_post(__in efx_txq_t * etp,__in_ecount (n)efx_desc_t * ed,__in unsigned int n,__in unsigned int completed,__inout unsigned int * addedp)557 efx_tx_qdesc_post(
558 __in efx_txq_t *etp,
559 __in_ecount(n) efx_desc_t *ed,
560 __in unsigned int n,
561 __in unsigned int completed,
562 __inout unsigned int *addedp)
563 {
564 efx_nic_t *enp = etp->et_enp;
565 efx_tx_ops_t *etxop = enp->en_etxop;
566 efx_rc_t rc;
567
568 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
569
570 if ((rc = etxop->etxo_qdesc_post(etp, ed,
571 n, completed, addedp)) != 0)
572 goto fail1;
573
574 return (0);
575
576 fail1:
577 EFSYS_PROBE1(fail1, efx_rc_t, rc);
578 return (rc);
579 }
580
581 void
efx_tx_qdesc_dma_create(__in efx_txq_t * etp,__in efsys_dma_addr_t addr,__in size_t size,__in boolean_t eop,__out efx_desc_t * edp)582 efx_tx_qdesc_dma_create(
583 __in efx_txq_t *etp,
584 __in efsys_dma_addr_t addr,
585 __in size_t size,
586 __in boolean_t eop,
587 __out efx_desc_t *edp)
588 {
589 efx_nic_t *enp = etp->et_enp;
590 efx_tx_ops_t *etxop = enp->en_etxop;
591
592 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
593 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
594
595 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
596 }
597
598 void
efx_tx_qdesc_tso_create(__in efx_txq_t * etp,__in uint16_t ipv4_id,__in uint32_t tcp_seq,__in uint8_t tcp_flags,__out efx_desc_t * edp)599 efx_tx_qdesc_tso_create(
600 __in efx_txq_t *etp,
601 __in uint16_t ipv4_id,
602 __in uint32_t tcp_seq,
603 __in uint8_t tcp_flags,
604 __out efx_desc_t *edp)
605 {
606 efx_nic_t *enp = etp->et_enp;
607 efx_tx_ops_t *etxop = enp->en_etxop;
608
609 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
610 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
611
612 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
613 }
614
615 void
efx_tx_qdesc_vlantci_create(__in efx_txq_t * etp,__in uint16_t tci,__out efx_desc_t * edp)616 efx_tx_qdesc_vlantci_create(
617 __in efx_txq_t *etp,
618 __in uint16_t tci,
619 __out efx_desc_t *edp)
620 {
621 efx_nic_t *enp = etp->et_enp;
622 efx_tx_ops_t *etxop = enp->en_etxop;
623
624 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
625 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
626
627 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
628 }
629
630
631 #if EFSYS_OPT_QSTATS
632 void
efx_tx_qstats_update(__in efx_txq_t * etp,__inout_ecount (TX_NQSTATS)efsys_stat_t * stat)633 efx_tx_qstats_update(
634 __in efx_txq_t *etp,
635 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
636 {
637 efx_nic_t *enp = etp->et_enp;
638 efx_tx_ops_t *etxop = enp->en_etxop;
639
640 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
641
642 etxop->etxo_qstats_update(etp, stat);
643 }
644 #endif
645
646
647 #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
648
649 static __checkReturn efx_rc_t
falconsiena_tx_init(__in efx_nic_t * enp)650 falconsiena_tx_init(
651 __in efx_nic_t *enp)
652 {
653 efx_oword_t oword;
654
655 /*
656 * Disable the timer-based TX DMA backoff and allow TX DMA to be
657 * controlled by the RX FIFO fill level (although always allow a
658 * minimal trickle).
659 */
660 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
661 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
662 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
663 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
664 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
665 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
666 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
667 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
668
669 /*
670 * Filter all packets less than 14 bytes to avoid parsing
671 * errors.
672 */
673 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
674 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
675
676 /*
677 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
678 * descriptors (which is bad).
679 */
680 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
681 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
682 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
683
684 return (0);
685 }
686
687 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
688 do { \
689 unsigned int id; \
690 size_t offset; \
691 efx_qword_t qword; \
692 \
693 id = (_added)++ & (_etp)->et_mask; \
694 offset = id * sizeof (efx_qword_t); \
695 \
696 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
697 unsigned int, id, efsys_dma_addr_t, (_addr), \
698 size_t, (_size), boolean_t, (_eop)); \
699 \
700 EFX_POPULATE_QWORD_4(qword, \
701 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
702 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
703 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
704 (uint32_t)((_addr) & 0xffffffff), \
705 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
706 (uint32_t)((_addr) >> 32)); \
707 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
708 \
709 _NOTE(CONSTANTCONDITION) \
710 } while (B_FALSE)
711
712 static __checkReturn efx_rc_t
falconsiena_tx_qpost(__in efx_txq_t * etp,__in_ecount (n)efx_buffer_t * eb,__in unsigned int n,__in unsigned int completed,__inout unsigned int * addedp)713 falconsiena_tx_qpost(
714 __in efx_txq_t *etp,
715 __in_ecount(n) efx_buffer_t *eb,
716 __in unsigned int n,
717 __in unsigned int completed,
718 __inout unsigned int *addedp)
719 {
720 unsigned int added = *addedp;
721 unsigned int i;
722 int rc = ENOSPC;
723
724 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
725 goto fail1;
726
727 for (i = 0; i < n; i++) {
728 efx_buffer_t *ebp = &eb[i];
729 efsys_dma_addr_t start = ebp->eb_addr;
730 size_t size = ebp->eb_size;
731 efsys_dma_addr_t end = start + size;
732
733 /* Fragments must not span 4k boundaries. */
734 EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end);
735
736 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
737 }
738
739 EFX_TX_QSTAT_INCR(etp, TX_POST);
740
741 *addedp = added;
742 return (0);
743
744 fail1:
745 EFSYS_PROBE1(fail1, efx_rc_t, rc);
746
747 return (rc);
748 }
749
750 static void
falconsiena_tx_qpush(__in efx_txq_t * etp,__in unsigned int added,__in unsigned int pushed)751 falconsiena_tx_qpush(
752 __in efx_txq_t *etp,
753 __in unsigned int added,
754 __in unsigned int pushed)
755 {
756 efx_nic_t *enp = etp->et_enp;
757 uint32_t wptr;
758 efx_dword_t dword;
759 efx_oword_t oword;
760
761 /* Push the populated descriptors out */
762 wptr = added & etp->et_mask;
763
764 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
765
766 /* Only write the third DWORD */
767 EFX_POPULATE_DWORD_1(dword,
768 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
769
770 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
771 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
772 wptr, pushed & etp->et_mask);
773 EFSYS_PIO_WRITE_BARRIER();
774 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
775 etp->et_index, &dword, B_FALSE);
776 }
777
778 #define EFX_MAX_PACE_VALUE 20
779 #define EFX_TX_PACE_CLOCK_BASE 104
780
781 static __checkReturn efx_rc_t
falconsiena_tx_qpace(__in efx_txq_t * etp,__in unsigned int ns)782 falconsiena_tx_qpace(
783 __in efx_txq_t *etp,
784 __in unsigned int ns)
785 {
786 efx_nic_t *enp = etp->et_enp;
787 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
788 efx_oword_t oword;
789 unsigned int pace_val;
790 unsigned int timer_period;
791 efx_rc_t rc;
792
793 if (ns == 0) {
794 pace_val = 0;
795 } else {
796 /*
797 * The pace_val to write into the table is s.t
798 * ns <= timer_period * (2 ^ pace_val)
799 */
800 timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult;
801 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
802 if ((timer_period << pace_val) >= ns)
803 break;
804 }
805 }
806 if (pace_val > EFX_MAX_PACE_VALUE) {
807 rc = EINVAL;
808 goto fail1;
809 }
810
811 /* Update the pacing table */
812 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
813 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
814 &oword, B_TRUE);
815
816 return (0);
817
818 fail1:
819 EFSYS_PROBE1(fail1, efx_rc_t, rc);
820
821 return (rc);
822 }
823
824 static __checkReturn efx_rc_t
falconsiena_tx_qflush(__in efx_txq_t * etp)825 falconsiena_tx_qflush(
826 __in efx_txq_t *etp)
827 {
828 efx_nic_t *enp = etp->et_enp;
829 efx_oword_t oword;
830 uint32_t label;
831
832 efx_tx_qpace(etp, 0);
833
834 label = etp->et_index;
835
836 /* Flush the queue */
837 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
838 FRF_AZ_TX_FLUSH_DESCQ, label);
839 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
840
841 return (0);
842 }
843
844 static void
falconsiena_tx_qenable(__in efx_txq_t * etp)845 falconsiena_tx_qenable(
846 __in efx_txq_t *etp)
847 {
848 efx_nic_t *enp = etp->et_enp;
849 efx_oword_t oword;
850
851 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
852 etp->et_index, &oword, B_TRUE);
853
854 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
855 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
856 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
857 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
858 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
859
860 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
861 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
862 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
863
864 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
865 etp->et_index, &oword, B_TRUE);
866 }
867
868 static __checkReturn efx_rc_t
falconsiena_tx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efsys_mem_t * esmp,__in size_t n,__in uint32_t id,__in uint16_t flags,__in efx_evq_t * eep,__in efx_txq_t * etp,__out unsigned int * addedp)869 falconsiena_tx_qcreate(
870 __in efx_nic_t *enp,
871 __in unsigned int index,
872 __in unsigned int label,
873 __in efsys_mem_t *esmp,
874 __in size_t n,
875 __in uint32_t id,
876 __in uint16_t flags,
877 __in efx_evq_t *eep,
878 __in efx_txq_t *etp,
879 __out unsigned int *addedp)
880 {
881 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
882 efx_oword_t oword;
883 uint32_t size;
884 efx_rc_t rc;
885
886 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
887 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
888 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
889
890 EFSYS_ASSERT(ISP2(EFX_TXQ_MAXNDESCS(encp)));
891 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
892
893 if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) {
894 rc = EINVAL;
895 goto fail1;
896 }
897 if (index >= encp->enc_txq_limit) {
898 rc = EINVAL;
899 goto fail2;
900 }
901 for (size = 0;
902 (1 << size) <= (EFX_TXQ_MAXNDESCS(encp) / EFX_TXQ_MINNDESCS);
903 size++)
904 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
905 break;
906 if (id + (1 << size) >= encp->enc_buftbl_limit) {
907 rc = EINVAL;
908 goto fail3;
909 }
910
911 /* Set up the new descriptor queue */
912 *addedp = 0;
913
914 EFX_POPULATE_OWORD_6(oword,
915 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
916 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
917 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
918 FRF_AZ_TX_DESCQ_LABEL, label,
919 FRF_AZ_TX_DESCQ_SIZE, size,
920 FRF_AZ_TX_DESCQ_TYPE, 0);
921
922 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
923 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
924 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
925 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
926 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
927
928 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
929 etp->et_index, &oword, B_TRUE);
930
931 return (0);
932
933 fail3:
934 EFSYS_PROBE(fail3);
935 fail2:
936 EFSYS_PROBE(fail2);
937 fail1:
938 EFSYS_PROBE1(fail1, efx_rc_t, rc);
939
940 return (rc);
941 }
942
943 __checkReturn efx_rc_t
falconsiena_tx_qdesc_post(__in efx_txq_t * etp,__in_ecount (n)efx_desc_t * ed,__in unsigned int n,__in unsigned int completed,__inout unsigned int * addedp)944 falconsiena_tx_qdesc_post(
945 __in efx_txq_t *etp,
946 __in_ecount(n) efx_desc_t *ed,
947 __in unsigned int n,
948 __in unsigned int completed,
949 __inout unsigned int *addedp)
950 {
951 unsigned int added = *addedp;
952 unsigned int i;
953 efx_rc_t rc;
954
955 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
956 rc = ENOSPC;
957 goto fail1;
958 }
959
960 for (i = 0; i < n; i++) {
961 efx_desc_t *edp = &ed[i];
962 unsigned int id;
963 size_t offset;
964
965 id = added++ & etp->et_mask;
966 offset = id * sizeof (efx_desc_t);
967
968 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
969 }
970
971 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
972 unsigned int, added, unsigned int, n);
973
974 EFX_TX_QSTAT_INCR(etp, TX_POST);
975
976 *addedp = added;
977 return (0);
978
979 fail1:
980 EFSYS_PROBE1(fail1, efx_rc_t, rc);
981 return (rc);
982 }
983
984 void
falconsiena_tx_qdesc_dma_create(__in efx_txq_t * etp,__in efsys_dma_addr_t addr,__in size_t size,__in boolean_t eop,__out efx_desc_t * edp)985 falconsiena_tx_qdesc_dma_create(
986 __in efx_txq_t *etp,
987 __in efsys_dma_addr_t addr,
988 __in size_t size,
989 __in boolean_t eop,
990 __out efx_desc_t *edp)
991 {
992 /* Fragments must not span 4k boundaries. */
993 EFSYS_ASSERT(P2ROUNDUP(addr + 1, 4096) >= addr + size);
994
995 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
996 efsys_dma_addr_t, addr,
997 size_t, size, boolean_t, eop);
998
999 EFX_POPULATE_QWORD_4(edp->ed_eq,
1000 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1001 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1002 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1003 (uint32_t)(addr & 0xffffffff),
1004 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1005 (uint32_t)(addr >> 32));
1006 }
1007
1008 #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
1009
1010 #if EFSYS_OPT_QSTATS
1011 #if EFSYS_OPT_NAMES
1012 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 9d8d26a0a5e2c453 */
1013 static const char *__efx_tx_qstat_name[] = {
1014 "post",
1015 "post_pio",
1016 };
1017 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1018
1019 const char *
efx_tx_qstat_name(__in efx_nic_t * enp,__in unsigned int id)1020 efx_tx_qstat_name(
1021 __in efx_nic_t *enp,
1022 __in unsigned int id)
1023 {
1024 _NOTE(ARGUNUSED(enp))
1025 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1026 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1027
1028 return (__efx_tx_qstat_name[id]);
1029 }
1030 #endif /* EFSYS_OPT_NAMES */
1031 #endif /* EFSYS_OPT_QSTATS */
1032
1033 #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
1034
1035 #if EFSYS_OPT_QSTATS
1036 static void
falconsiena_tx_qstats_update(__in efx_txq_t * etp,__inout_ecount (TX_NQSTATS)efsys_stat_t * stat)1037 falconsiena_tx_qstats_update(
1038 __in efx_txq_t *etp,
1039 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1040 {
1041 unsigned int id;
1042
1043 for (id = 0; id < TX_NQSTATS; id++) {
1044 efsys_stat_t *essp = &stat[id];
1045
1046 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1047 etp->et_stat[id] = 0;
1048 }
1049 }
1050 #endif /* EFSYS_OPT_QSTATS */
1051
1052 static void
falconsiena_tx_qdestroy(__in efx_txq_t * etp)1053 falconsiena_tx_qdestroy(
1054 __in efx_txq_t *etp)
1055 {
1056 efx_nic_t *enp = etp->et_enp;
1057 efx_oword_t oword;
1058
1059 /* Purge descriptor queue */
1060 EFX_ZERO_OWORD(oword);
1061
1062 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1063 etp->et_index, &oword, B_TRUE);
1064 }
1065
1066 static void
falconsiena_tx_fini(__in efx_nic_t * enp)1067 falconsiena_tx_fini(
1068 __in efx_nic_t *enp)
1069 {
1070 _NOTE(ARGUNUSED(enp))
1071 }
1072
1073 #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
1074