1 /*-
2 * Copyright (c) 2007-2016 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
29 */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33
34 #include "efx.h"
35 #include "efx_impl.h"
36
37 #if EFSYS_OPT_QSTATS
38 #define EFX_TX_QSTAT_INCR(_etp, _stat) \
39 do { \
40 (_etp)->et_stat[_stat]++; \
41 _NOTE(CONSTANTCONDITION) \
42 } while (B_FALSE)
43 #else
44 #define EFX_TX_QSTAT_INCR(_etp, _stat)
45 #endif
46
47 #if EFSYS_OPT_SIENA
48
49 static __checkReturn efx_rc_t
50 siena_tx_init(
51 __in efx_nic_t *enp);
52
53 static void
54 siena_tx_fini(
55 __in efx_nic_t *enp);
56
57 static __checkReturn efx_rc_t
58 siena_tx_qcreate(
59 __in efx_nic_t *enp,
60 __in unsigned int index,
61 __in unsigned int label,
62 __in efsys_mem_t *esmp,
63 __in size_t n,
64 __in uint32_t id,
65 __in uint16_t flags,
66 __in efx_evq_t *eep,
67 __in efx_txq_t *etp,
68 __out unsigned int *addedp);
69
70 static void
71 siena_tx_qdestroy(
72 __in efx_txq_t *etp);
73
74 static __checkReturn efx_rc_t
75 siena_tx_qpost(
76 __in efx_txq_t *etp,
77 __in_ecount(n) efx_buffer_t *eb,
78 __in unsigned int n,
79 __in unsigned int completed,
80 __inout unsigned int *addedp);
81
82 static void
83 siena_tx_qpush(
84 __in efx_txq_t *etp,
85 __in unsigned int added,
86 __in unsigned int pushed);
87
88 static __checkReturn efx_rc_t
89 siena_tx_qpace(
90 __in efx_txq_t *etp,
91 __in unsigned int ns);
92
93 static __checkReturn efx_rc_t
94 siena_tx_qflush(
95 __in efx_txq_t *etp);
96
97 static void
98 siena_tx_qenable(
99 __in efx_txq_t *etp);
100
101 __checkReturn efx_rc_t
102 siena_tx_qdesc_post(
103 __in efx_txq_t *etp,
104 __in_ecount(n) efx_desc_t *ed,
105 __in unsigned int n,
106 __in unsigned int completed,
107 __inout unsigned int *addedp);
108
109 void
110 siena_tx_qdesc_dma_create(
111 __in efx_txq_t *etp,
112 __in efsys_dma_addr_t addr,
113 __in size_t size,
114 __in boolean_t eop,
115 __out efx_desc_t *edp);
116
117 #if EFSYS_OPT_QSTATS
118 static void
119 siena_tx_qstats_update(
120 __in efx_txq_t *etp,
121 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
122 #endif
123
124 #endif /* EFSYS_OPT_SIENA */
125
126
127 #if EFSYS_OPT_SIENA
128 static const efx_tx_ops_t __efx_tx_siena_ops = {
129 siena_tx_init, /* etxo_init */
130 siena_tx_fini, /* etxo_fini */
131 siena_tx_qcreate, /* etxo_qcreate */
132 siena_tx_qdestroy, /* etxo_qdestroy */
133 siena_tx_qpost, /* etxo_qpost */
134 siena_tx_qpush, /* etxo_qpush */
135 siena_tx_qpace, /* etxo_qpace */
136 siena_tx_qflush, /* etxo_qflush */
137 siena_tx_qenable, /* etxo_qenable */
138 NULL, /* etxo_qpio_enable */
139 NULL, /* etxo_qpio_disable */
140 NULL, /* etxo_qpio_write */
141 NULL, /* etxo_qpio_post */
142 siena_tx_qdesc_post, /* etxo_qdesc_post */
143 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
144 NULL, /* etxo_qdesc_tso_create */
145 NULL, /* etxo_qdesc_tso2_create */
146 NULL, /* etxo_qdesc_vlantci_create */
147 NULL, /* etxo_qdesc_checksum_create */
148 #if EFSYS_OPT_QSTATS
149 siena_tx_qstats_update, /* etxo_qstats_update */
150 #endif
151 };
152 #endif /* EFSYS_OPT_SIENA */
153
154 #if EFSYS_OPT_HUNTINGTON
155 static const efx_tx_ops_t __efx_tx_hunt_ops = {
156 ef10_tx_init, /* etxo_init */
157 ef10_tx_fini, /* etxo_fini */
158 ef10_tx_qcreate, /* etxo_qcreate */
159 ef10_tx_qdestroy, /* etxo_qdestroy */
160 ef10_tx_qpost, /* etxo_qpost */
161 ef10_tx_qpush, /* etxo_qpush */
162 ef10_tx_qpace, /* etxo_qpace */
163 ef10_tx_qflush, /* etxo_qflush */
164 ef10_tx_qenable, /* etxo_qenable */
165 ef10_tx_qpio_enable, /* etxo_qpio_enable */
166 ef10_tx_qpio_disable, /* etxo_qpio_disable */
167 ef10_tx_qpio_write, /* etxo_qpio_write */
168 ef10_tx_qpio_post, /* etxo_qpio_post */
169 ef10_tx_qdesc_post, /* etxo_qdesc_post */
170 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
171 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
172 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
173 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
174 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
175 #if EFSYS_OPT_QSTATS
176 ef10_tx_qstats_update, /* etxo_qstats_update */
177 #endif
178 };
179 #endif /* EFSYS_OPT_HUNTINGTON */
180
181 #if EFSYS_OPT_MEDFORD
182 static const efx_tx_ops_t __efx_tx_medford_ops = {
183 ef10_tx_init, /* etxo_init */
184 ef10_tx_fini, /* etxo_fini */
185 ef10_tx_qcreate, /* etxo_qcreate */
186 ef10_tx_qdestroy, /* etxo_qdestroy */
187 ef10_tx_qpost, /* etxo_qpost */
188 ef10_tx_qpush, /* etxo_qpush */
189 ef10_tx_qpace, /* etxo_qpace */
190 ef10_tx_qflush, /* etxo_qflush */
191 ef10_tx_qenable, /* etxo_qenable */
192 ef10_tx_qpio_enable, /* etxo_qpio_enable */
193 ef10_tx_qpio_disable, /* etxo_qpio_disable */
194 ef10_tx_qpio_write, /* etxo_qpio_write */
195 ef10_tx_qpio_post, /* etxo_qpio_post */
196 ef10_tx_qdesc_post, /* etxo_qdesc_post */
197 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
198 NULL, /* etxo_qdesc_tso_create */
199 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
200 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
201 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
202 #if EFSYS_OPT_QSTATS
203 ef10_tx_qstats_update, /* etxo_qstats_update */
204 #endif
205 };
206 #endif /* EFSYS_OPT_MEDFORD */
207
208 __checkReturn efx_rc_t
efx_tx_init(__in efx_nic_t * enp)209 efx_tx_init(
210 __in efx_nic_t *enp)
211 {
212 const efx_tx_ops_t *etxop;
213 efx_rc_t rc;
214
215 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
216 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
217
218 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
219 rc = EINVAL;
220 goto fail1;
221 }
222
223 if (enp->en_mod_flags & EFX_MOD_TX) {
224 rc = EINVAL;
225 goto fail2;
226 }
227
228 switch (enp->en_family) {
229 #if EFSYS_OPT_SIENA
230 case EFX_FAMILY_SIENA:
231 etxop = &__efx_tx_siena_ops;
232 break;
233 #endif /* EFSYS_OPT_SIENA */
234
235 #if EFSYS_OPT_HUNTINGTON
236 case EFX_FAMILY_HUNTINGTON:
237 etxop = &__efx_tx_hunt_ops;
238 break;
239 #endif /* EFSYS_OPT_HUNTINGTON */
240
241 #if EFSYS_OPT_MEDFORD
242 case EFX_FAMILY_MEDFORD:
243 etxop = &__efx_tx_medford_ops;
244 break;
245 #endif /* EFSYS_OPT_MEDFORD */
246
247 default:
248 EFSYS_ASSERT(0);
249 rc = ENOTSUP;
250 goto fail3;
251 }
252
253 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
254
255 if ((rc = etxop->etxo_init(enp)) != 0)
256 goto fail4;
257
258 enp->en_etxop = etxop;
259 enp->en_mod_flags |= EFX_MOD_TX;
260 return (0);
261
262 fail4:
263 EFSYS_PROBE(fail4);
264 fail3:
265 EFSYS_PROBE(fail3);
266 fail2:
267 EFSYS_PROBE(fail2);
268 fail1:
269 EFSYS_PROBE1(fail1, efx_rc_t, rc);
270
271 enp->en_etxop = NULL;
272 enp->en_mod_flags &= ~EFX_MOD_TX;
273 return (rc);
274 }
275
276 void
efx_tx_fini(__in efx_nic_t * enp)277 efx_tx_fini(
278 __in efx_nic_t *enp)
279 {
280 const efx_tx_ops_t *etxop = enp->en_etxop;
281
282 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
283 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
284 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
285 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
286
287 etxop->etxo_fini(enp);
288
289 enp->en_etxop = NULL;
290 enp->en_mod_flags &= ~EFX_MOD_TX;
291 }
292
293 __checkReturn efx_rc_t
efx_tx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efsys_mem_t * esmp,__in size_t n,__in uint32_t id,__in uint16_t flags,__in efx_evq_t * eep,__deref_out efx_txq_t ** etpp,__out unsigned int * addedp)294 efx_tx_qcreate(
295 __in efx_nic_t *enp,
296 __in unsigned int index,
297 __in unsigned int label,
298 __in efsys_mem_t *esmp,
299 __in size_t n,
300 __in uint32_t id,
301 __in uint16_t flags,
302 __in efx_evq_t *eep,
303 __deref_out efx_txq_t **etpp,
304 __out unsigned int *addedp)
305 {
306 const efx_tx_ops_t *etxop = enp->en_etxop;
307 efx_txq_t *etp;
308 efx_rc_t rc;
309
310 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
311 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
312
313 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <,
314 enp->en_nic_cfg.enc_txq_limit);
315
316 /* Allocate an TXQ object */
317 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
318
319 if (etp == NULL) {
320 rc = ENOMEM;
321 goto fail1;
322 }
323
324 etp->et_magic = EFX_TXQ_MAGIC;
325 etp->et_enp = enp;
326 etp->et_index = index;
327 etp->et_mask = n - 1;
328 etp->et_esmp = esmp;
329
330 /* Initial descriptor index may be modified by etxo_qcreate */
331 *addedp = 0;
332
333 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
334 n, id, flags, eep, etp, addedp)) != 0)
335 goto fail2;
336
337 enp->en_tx_qcount++;
338 *etpp = etp;
339
340 return (0);
341
342 fail2:
343 EFSYS_PROBE(fail2);
344 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
345 fail1:
346 EFSYS_PROBE1(fail1, efx_rc_t, rc);
347 return (rc);
348 }
349
350 void
efx_tx_qdestroy(__in efx_txq_t * etp)351 efx_tx_qdestroy(
352 __in efx_txq_t *etp)
353 {
354 efx_nic_t *enp = etp->et_enp;
355 const efx_tx_ops_t *etxop = enp->en_etxop;
356
357 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
358
359 EFSYS_ASSERT(enp->en_tx_qcount != 0);
360 --enp->en_tx_qcount;
361
362 etxop->etxo_qdestroy(etp);
363
364 /* Free the TXQ object */
365 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
366 }
367
368 __checkReturn efx_rc_t
efx_tx_qpost(__in efx_txq_t * etp,__in_ecount (n)efx_buffer_t * eb,__in unsigned int n,__in unsigned int completed,__inout unsigned int * addedp)369 efx_tx_qpost(
370 __in efx_txq_t *etp,
371 __in_ecount(n) efx_buffer_t *eb,
372 __in unsigned int n,
373 __in unsigned int completed,
374 __inout unsigned int *addedp)
375 {
376 efx_nic_t *enp = etp->et_enp;
377 const efx_tx_ops_t *etxop = enp->en_etxop;
378 efx_rc_t rc;
379
380 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
381
382 if ((rc = etxop->etxo_qpost(etp, eb,
383 n, completed, addedp)) != 0)
384 goto fail1;
385
386 return (0);
387
388 fail1:
389 EFSYS_PROBE1(fail1, efx_rc_t, rc);
390 return (rc);
391 }
392
393 void
efx_tx_qpush(__in efx_txq_t * etp,__in unsigned int added,__in unsigned int pushed)394 efx_tx_qpush(
395 __in efx_txq_t *etp,
396 __in unsigned int added,
397 __in unsigned int pushed)
398 {
399 efx_nic_t *enp = etp->et_enp;
400 const efx_tx_ops_t *etxop = enp->en_etxop;
401
402 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
403
404 etxop->etxo_qpush(etp, added, pushed);
405 }
406
407 __checkReturn efx_rc_t
efx_tx_qpace(__in efx_txq_t * etp,__in unsigned int ns)408 efx_tx_qpace(
409 __in efx_txq_t *etp,
410 __in unsigned int ns)
411 {
412 efx_nic_t *enp = etp->et_enp;
413 const efx_tx_ops_t *etxop = enp->en_etxop;
414 efx_rc_t rc;
415
416 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
417
418 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
419 goto fail1;
420
421 return (0);
422
423 fail1:
424 EFSYS_PROBE1(fail1, efx_rc_t, rc);
425 return (rc);
426 }
427
428 __checkReturn efx_rc_t
efx_tx_qflush(__in efx_txq_t * etp)429 efx_tx_qflush(
430 __in efx_txq_t *etp)
431 {
432 efx_nic_t *enp = etp->et_enp;
433 const efx_tx_ops_t *etxop = enp->en_etxop;
434 efx_rc_t rc;
435
436 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
437
438 if ((rc = etxop->etxo_qflush(etp)) != 0)
439 goto fail1;
440
441 return (0);
442
443 fail1:
444 EFSYS_PROBE1(fail1, efx_rc_t, rc);
445 return (rc);
446 }
447
448 void
efx_tx_qenable(__in efx_txq_t * etp)449 efx_tx_qenable(
450 __in efx_txq_t *etp)
451 {
452 efx_nic_t *enp = etp->et_enp;
453 const efx_tx_ops_t *etxop = enp->en_etxop;
454
455 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
456
457 etxop->etxo_qenable(etp);
458 }
459
460 __checkReturn efx_rc_t
efx_tx_qpio_enable(__in efx_txq_t * etp)461 efx_tx_qpio_enable(
462 __in efx_txq_t *etp)
463 {
464 efx_nic_t *enp = etp->et_enp;
465 const efx_tx_ops_t *etxop = enp->en_etxop;
466 efx_rc_t rc;
467
468 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
469
470 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
471 rc = ENOTSUP;
472 goto fail1;
473 }
474 if (etxop->etxo_qpio_enable == NULL) {
475 rc = ENOTSUP;
476 goto fail2;
477 }
478 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
479 goto fail3;
480
481 return (0);
482
483 fail3:
484 EFSYS_PROBE(fail3);
485 fail2:
486 EFSYS_PROBE(fail2);
487 fail1:
488 EFSYS_PROBE1(fail1, efx_rc_t, rc);
489 return (rc);
490 }
491
492 void
efx_tx_qpio_disable(__in efx_txq_t * etp)493 efx_tx_qpio_disable(
494 __in efx_txq_t *etp)
495 {
496 efx_nic_t *enp = etp->et_enp;
497 const efx_tx_ops_t *etxop = enp->en_etxop;
498
499 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
500
501 if (etxop->etxo_qpio_disable != NULL)
502 etxop->etxo_qpio_disable(etp);
503 }
504
505 __checkReturn efx_rc_t
efx_tx_qpio_write(__in efx_txq_t * etp,__in_ecount (buf_length)uint8_t * buffer,__in size_t buf_length,__in size_t pio_buf_offset)506 efx_tx_qpio_write(
507 __in efx_txq_t *etp,
508 __in_ecount(buf_length) uint8_t *buffer,
509 __in size_t buf_length,
510 __in size_t pio_buf_offset)
511 {
512 efx_nic_t *enp = etp->et_enp;
513 const efx_tx_ops_t *etxop = enp->en_etxop;
514 efx_rc_t rc;
515
516 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
517
518 if (etxop->etxo_qpio_write != NULL) {
519 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
520 pio_buf_offset)) != 0)
521 goto fail1;
522 return (0);
523 }
524
525 return (ENOTSUP);
526
527 fail1:
528 EFSYS_PROBE1(fail1, efx_rc_t, rc);
529 return (rc);
530 }
531
532 __checkReturn efx_rc_t
efx_tx_qpio_post(__in efx_txq_t * etp,__in size_t pkt_length,__in unsigned int completed,__inout unsigned int * addedp)533 efx_tx_qpio_post(
534 __in efx_txq_t *etp,
535 __in size_t pkt_length,
536 __in unsigned int completed,
537 __inout unsigned int *addedp)
538 {
539 efx_nic_t *enp = etp->et_enp;
540 const efx_tx_ops_t *etxop = enp->en_etxop;
541 efx_rc_t rc;
542
543 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
544
545 if (etxop->etxo_qpio_post != NULL) {
546 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
547 addedp)) != 0)
548 goto fail1;
549 return (0);
550 }
551
552 return (ENOTSUP);
553
554 fail1:
555 EFSYS_PROBE1(fail1, efx_rc_t, rc);
556 return (rc);
557 }
558
559 __checkReturn efx_rc_t
efx_tx_qdesc_post(__in efx_txq_t * etp,__in_ecount (n)efx_desc_t * ed,__in unsigned int n,__in unsigned int completed,__inout unsigned int * addedp)560 efx_tx_qdesc_post(
561 __in efx_txq_t *etp,
562 __in_ecount(n) efx_desc_t *ed,
563 __in unsigned int n,
564 __in unsigned int completed,
565 __inout unsigned int *addedp)
566 {
567 efx_nic_t *enp = etp->et_enp;
568 const efx_tx_ops_t *etxop = enp->en_etxop;
569 efx_rc_t rc;
570
571 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
572
573 if ((rc = etxop->etxo_qdesc_post(etp, ed,
574 n, completed, addedp)) != 0)
575 goto fail1;
576
577 return (0);
578
579 fail1:
580 EFSYS_PROBE1(fail1, efx_rc_t, rc);
581 return (rc);
582 }
583
584 void
efx_tx_qdesc_dma_create(__in efx_txq_t * etp,__in efsys_dma_addr_t addr,__in size_t size,__in boolean_t eop,__out efx_desc_t * edp)585 efx_tx_qdesc_dma_create(
586 __in efx_txq_t *etp,
587 __in efsys_dma_addr_t addr,
588 __in size_t size,
589 __in boolean_t eop,
590 __out efx_desc_t *edp)
591 {
592 efx_nic_t *enp = etp->et_enp;
593 const efx_tx_ops_t *etxop = enp->en_etxop;
594
595 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
596 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
597
598 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
599 }
600
601 void
efx_tx_qdesc_tso_create(__in efx_txq_t * etp,__in uint16_t ipv4_id,__in uint32_t tcp_seq,__in uint8_t tcp_flags,__out efx_desc_t * edp)602 efx_tx_qdesc_tso_create(
603 __in efx_txq_t *etp,
604 __in uint16_t ipv4_id,
605 __in uint32_t tcp_seq,
606 __in uint8_t tcp_flags,
607 __out efx_desc_t *edp)
608 {
609 efx_nic_t *enp = etp->et_enp;
610 const efx_tx_ops_t *etxop = enp->en_etxop;
611
612 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
613 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
614
615 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
616 }
617
618 void
efx_tx_qdesc_tso2_create(__in efx_txq_t * etp,__in uint16_t ipv4_id,__in uint32_t tcp_seq,__in uint16_t mss,__out_ecount (count)efx_desc_t * edp,__in int count)619 efx_tx_qdesc_tso2_create(
620 __in efx_txq_t *etp,
621 __in uint16_t ipv4_id,
622 __in uint32_t tcp_seq,
623 __in uint16_t mss,
624 __out_ecount(count) efx_desc_t *edp,
625 __in int count)
626 {
627 efx_nic_t *enp = etp->et_enp;
628 const efx_tx_ops_t *etxop = enp->en_etxop;
629
630 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
631 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
632
633 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count);
634 }
635
636 void
efx_tx_qdesc_vlantci_create(__in efx_txq_t * etp,__in uint16_t tci,__out efx_desc_t * edp)637 efx_tx_qdesc_vlantci_create(
638 __in efx_txq_t *etp,
639 __in uint16_t tci,
640 __out efx_desc_t *edp)
641 {
642 efx_nic_t *enp = etp->et_enp;
643 const efx_tx_ops_t *etxop = enp->en_etxop;
644
645 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
646 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
647
648 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
649 }
650
651 void
efx_tx_qdesc_checksum_create(__in efx_txq_t * etp,__in uint16_t flags,__out efx_desc_t * edp)652 efx_tx_qdesc_checksum_create(
653 __in efx_txq_t *etp,
654 __in uint16_t flags,
655 __out efx_desc_t *edp)
656 {
657 efx_nic_t *enp = etp->et_enp;
658 const efx_tx_ops_t *etxop = enp->en_etxop;
659
660 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
661 EFSYS_ASSERT(etxop->etxo_qdesc_checksum_create != NULL);
662
663 etxop->etxo_qdesc_checksum_create(etp, flags, edp);
664 }
665
666
667 #if EFSYS_OPT_QSTATS
668 void
efx_tx_qstats_update(__in efx_txq_t * etp,__inout_ecount (TX_NQSTATS)efsys_stat_t * stat)669 efx_tx_qstats_update(
670 __in efx_txq_t *etp,
671 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
672 {
673 efx_nic_t *enp = etp->et_enp;
674 const efx_tx_ops_t *etxop = enp->en_etxop;
675
676 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
677
678 etxop->etxo_qstats_update(etp, stat);
679 }
680 #endif
681
682
683 #if EFSYS_OPT_SIENA
684
685 static __checkReturn efx_rc_t
siena_tx_init(__in efx_nic_t * enp)686 siena_tx_init(
687 __in efx_nic_t *enp)
688 {
689 efx_oword_t oword;
690
691 /*
692 * Disable the timer-based TX DMA backoff and allow TX DMA to be
693 * controlled by the RX FIFO fill level (although always allow a
694 * minimal trickle).
695 */
696 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
697 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
698 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
699 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
700 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
701 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
702 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
703 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
704
705 /*
706 * Filter all packets less than 14 bytes to avoid parsing
707 * errors.
708 */
709 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
710 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
711
712 /*
713 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
714 * descriptors (which is bad).
715 */
716 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
717 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
718 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
719
720 return (0);
721 }
722
723 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
724 do { \
725 unsigned int id; \
726 size_t offset; \
727 efx_qword_t qword; \
728 \
729 id = (_added)++ & (_etp)->et_mask; \
730 offset = id * sizeof (efx_qword_t); \
731 \
732 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
733 unsigned int, id, efsys_dma_addr_t, (_addr), \
734 size_t, (_size), boolean_t, (_eop)); \
735 \
736 EFX_POPULATE_QWORD_4(qword, \
737 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
738 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
739 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
740 (uint32_t)((_addr) & 0xffffffff), \
741 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
742 (uint32_t)((_addr) >> 32)); \
743 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
744 \
745 _NOTE(CONSTANTCONDITION) \
746 } while (B_FALSE)
747
748 static __checkReturn efx_rc_t
siena_tx_qpost(__in efx_txq_t * etp,__in_ecount (n)efx_buffer_t * eb,__in unsigned int n,__in unsigned int completed,__inout unsigned int * addedp)749 siena_tx_qpost(
750 __in efx_txq_t *etp,
751 __in_ecount(n) efx_buffer_t *eb,
752 __in unsigned int n,
753 __in unsigned int completed,
754 __inout unsigned int *addedp)
755 {
756 unsigned int added = *addedp;
757 unsigned int i;
758 int rc = ENOSPC;
759
760 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
761 goto fail1;
762
763 for (i = 0; i < n; i++) {
764 efx_buffer_t *ebp = &eb[i];
765 efsys_dma_addr_t start = ebp->eb_addr;
766 size_t size = ebp->eb_size;
767 efsys_dma_addr_t end = start + size;
768
769 /*
770 * Fragments must not span 4k boundaries.
771 * Here it is a stricter requirement than the maximum length.
772 */
773 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, start + 1,
774 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end);
775
776 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
777 }
778
779 EFX_TX_QSTAT_INCR(etp, TX_POST);
780
781 *addedp = added;
782 return (0);
783
784 fail1:
785 EFSYS_PROBE1(fail1, efx_rc_t, rc);
786
787 return (rc);
788 }
789
790 static void
siena_tx_qpush(__in efx_txq_t * etp,__in unsigned int added,__in unsigned int pushed)791 siena_tx_qpush(
792 __in efx_txq_t *etp,
793 __in unsigned int added,
794 __in unsigned int pushed)
795 {
796 efx_nic_t *enp = etp->et_enp;
797 uint32_t wptr;
798 efx_dword_t dword;
799 efx_oword_t oword;
800
801 /* Push the populated descriptors out */
802 wptr = added & etp->et_mask;
803
804 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
805
806 /* Only write the third DWORD */
807 EFX_POPULATE_DWORD_1(dword,
808 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
809
810 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
811 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
812 wptr, pushed & etp->et_mask);
813 EFSYS_PIO_WRITE_BARRIER();
814 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
815 etp->et_index, &dword, B_FALSE);
816 }
817
818 #define EFX_MAX_PACE_VALUE 20
819 #define EFX_TX_PACE_CLOCK_BASE 104
820
821 static __checkReturn efx_rc_t
siena_tx_qpace(__in efx_txq_t * etp,__in unsigned int ns)822 siena_tx_qpace(
823 __in efx_txq_t *etp,
824 __in unsigned int ns)
825 {
826 efx_nic_t *enp = etp->et_enp;
827 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
828 efx_oword_t oword;
829 unsigned int pace_val;
830 unsigned int timer_period;
831 efx_rc_t rc;
832
833 if (ns == 0) {
834 pace_val = 0;
835 } else {
836 /*
837 * The pace_val to write into the table is s.t
838 * ns <= timer_period * (2 ^ pace_val)
839 */
840 timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult;
841 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
842 if ((timer_period << pace_val) >= ns)
843 break;
844 }
845 }
846 if (pace_val > EFX_MAX_PACE_VALUE) {
847 rc = EINVAL;
848 goto fail1;
849 }
850
851 /* Update the pacing table */
852 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
853 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
854 &oword, B_TRUE);
855
856 return (0);
857
858 fail1:
859 EFSYS_PROBE1(fail1, efx_rc_t, rc);
860
861 return (rc);
862 }
863
864 static __checkReturn efx_rc_t
siena_tx_qflush(__in efx_txq_t * etp)865 siena_tx_qflush(
866 __in efx_txq_t *etp)
867 {
868 efx_nic_t *enp = etp->et_enp;
869 efx_oword_t oword;
870 uint32_t label;
871
872 efx_tx_qpace(etp, 0);
873
874 label = etp->et_index;
875
876 /* Flush the queue */
877 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
878 FRF_AZ_TX_FLUSH_DESCQ, label);
879 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
880
881 return (0);
882 }
883
884 static void
siena_tx_qenable(__in efx_txq_t * etp)885 siena_tx_qenable(
886 __in efx_txq_t *etp)
887 {
888 efx_nic_t *enp = etp->et_enp;
889 efx_oword_t oword;
890
891 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
892 etp->et_index, &oword, B_TRUE);
893
894 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
895 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
896 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
897 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
898 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
899
900 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
901 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
902 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
903
904 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
905 etp->et_index, &oword, B_TRUE);
906 }
907
908 static __checkReturn efx_rc_t
siena_tx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efsys_mem_t * esmp,__in size_t n,__in uint32_t id,__in uint16_t flags,__in efx_evq_t * eep,__in efx_txq_t * etp,__out unsigned int * addedp)909 siena_tx_qcreate(
910 __in efx_nic_t *enp,
911 __in unsigned int index,
912 __in unsigned int label,
913 __in efsys_mem_t *esmp,
914 __in size_t n,
915 __in uint32_t id,
916 __in uint16_t flags,
917 __in efx_evq_t *eep,
918 __in efx_txq_t *etp,
919 __out unsigned int *addedp)
920 {
921 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
922 efx_oword_t oword;
923 uint32_t size;
924 uint16_t inner_csum;
925 efx_rc_t rc;
926
927 _NOTE(ARGUNUSED(esmp))
928
929 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
930 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
931 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
932
933 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs));
934 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
935
936 if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) {
937 rc = EINVAL;
938 goto fail1;
939 }
940 if (index >= encp->enc_txq_limit) {
941 rc = EINVAL;
942 goto fail2;
943 }
944 for (size = 0;
945 (1 << size) <= (int)(encp->enc_txq_max_ndescs / EFX_TXQ_MINNDESCS);
946 size++)
947 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
948 break;
949 if (id + (1 << size) >= encp->enc_buftbl_limit) {
950 rc = EINVAL;
951 goto fail3;
952 }
953
954 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP;
955 if ((flags & inner_csum) != 0) {
956 rc = EINVAL;
957 goto fail4;
958 }
959
960 /* Set up the new descriptor queue */
961 *addedp = 0;
962
963 EFX_POPULATE_OWORD_6(oword,
964 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
965 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
966 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
967 FRF_AZ_TX_DESCQ_LABEL, label,
968 FRF_AZ_TX_DESCQ_SIZE, size,
969 FRF_AZ_TX_DESCQ_TYPE, 0);
970
971 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
972 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
973 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
974 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
975 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
976
977 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
978 etp->et_index, &oword, B_TRUE);
979
980 return (0);
981
982 fail4:
983 EFSYS_PROBE(fail4);
984 fail3:
985 EFSYS_PROBE(fail3);
986 fail2:
987 EFSYS_PROBE(fail2);
988 fail1:
989 EFSYS_PROBE1(fail1, efx_rc_t, rc);
990
991 return (rc);
992 }
993
994 __checkReturn efx_rc_t
siena_tx_qdesc_post(__in efx_txq_t * etp,__in_ecount (n)efx_desc_t * ed,__in unsigned int n,__in unsigned int completed,__inout unsigned int * addedp)995 siena_tx_qdesc_post(
996 __in efx_txq_t *etp,
997 __in_ecount(n) efx_desc_t *ed,
998 __in unsigned int n,
999 __in unsigned int completed,
1000 __inout unsigned int *addedp)
1001 {
1002 unsigned int added = *addedp;
1003 unsigned int i;
1004 efx_rc_t rc;
1005
1006 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
1007 rc = ENOSPC;
1008 goto fail1;
1009 }
1010
1011 for (i = 0; i < n; i++) {
1012 efx_desc_t *edp = &ed[i];
1013 unsigned int id;
1014 size_t offset;
1015
1016 id = added++ & etp->et_mask;
1017 offset = id * sizeof (efx_desc_t);
1018
1019 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
1020 }
1021
1022 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
1023 unsigned int, added, unsigned int, n);
1024
1025 EFX_TX_QSTAT_INCR(etp, TX_POST);
1026
1027 *addedp = added;
1028 return (0);
1029
1030 fail1:
1031 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1032 return (rc);
1033 }
1034
1035 void
siena_tx_qdesc_dma_create(__in efx_txq_t * etp,__in efsys_dma_addr_t addr,__in size_t size,__in boolean_t eop,__out efx_desc_t * edp)1036 siena_tx_qdesc_dma_create(
1037 __in efx_txq_t *etp,
1038 __in efsys_dma_addr_t addr,
1039 __in size_t size,
1040 __in boolean_t eop,
1041 __out efx_desc_t *edp)
1042 {
1043 /*
1044 * Fragments must not span 4k boundaries.
1045 * Here it is a stricter requirement than the maximum length.
1046 */
1047 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, addr + 1,
1048 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size);
1049
1050 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1051 efsys_dma_addr_t, addr,
1052 size_t, size, boolean_t, eop);
1053
1054 EFX_POPULATE_QWORD_4(edp->ed_eq,
1055 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1056 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1057 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1058 (uint32_t)(addr & 0xffffffff),
1059 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1060 (uint32_t)(addr >> 32));
1061 }
1062
1063 #endif /* EFSYS_OPT_SIENA */
1064
1065 #if EFSYS_OPT_QSTATS
1066 #if EFSYS_OPT_NAMES
1067 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */
1068 static const char * const __efx_tx_qstat_name[] = {
1069 "post",
1070 "post_pio",
1071 };
1072 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1073
1074 const char *
efx_tx_qstat_name(__in efx_nic_t * enp,__in unsigned int id)1075 efx_tx_qstat_name(
1076 __in efx_nic_t *enp,
1077 __in unsigned int id)
1078 {
1079 _NOTE(ARGUNUSED(enp))
1080 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1081 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1082
1083 return (__efx_tx_qstat_name[id]);
1084 }
1085 #endif /* EFSYS_OPT_NAMES */
1086 #endif /* EFSYS_OPT_QSTATS */
1087
1088 #if EFSYS_OPT_SIENA
1089
1090 #if EFSYS_OPT_QSTATS
1091 static void
siena_tx_qstats_update(__in efx_txq_t * etp,__inout_ecount (TX_NQSTATS)efsys_stat_t * stat)1092 siena_tx_qstats_update(
1093 __in efx_txq_t *etp,
1094 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1095 {
1096 unsigned int id;
1097
1098 for (id = 0; id < TX_NQSTATS; id++) {
1099 efsys_stat_t *essp = &stat[id];
1100
1101 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1102 etp->et_stat[id] = 0;
1103 }
1104 }
1105 #endif /* EFSYS_OPT_QSTATS */
1106
1107 static void
siena_tx_qdestroy(__in efx_txq_t * etp)1108 siena_tx_qdestroy(
1109 __in efx_txq_t *etp)
1110 {
1111 efx_nic_t *enp = etp->et_enp;
1112 efx_oword_t oword;
1113
1114 /* Purge descriptor queue */
1115 EFX_ZERO_OWORD(oword);
1116
1117 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1118 etp->et_index, &oword, B_TRUE);
1119 }
1120
1121 static void
siena_tx_fini(__in efx_nic_t * enp)1122 siena_tx_fini(
1123 __in efx_nic_t *enp)
1124 {
1125 _NOTE(ARGUNUSED(enp))
1126 }
1127
1128 #endif /* EFSYS_OPT_SIENA */
1129