1 /*-
2 * Copyright (c) 2007-2015 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
29 */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33
34 #include "efsys.h"
35 #include "efx.h"
36 #include "efx_types.h"
37 #include "efx_regs.h"
38 #include "efx_impl.h"
39
40
41 #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
42
43 static __checkReturn efx_rc_t
44 falconsiena_rx_init(
45 __in efx_nic_t *enp);
46
47 static void
48 falconsiena_rx_fini(
49 __in efx_nic_t *enp);
50
51 #if EFSYS_OPT_RX_HDR_SPLIT
52 static __checkReturn efx_rc_t
53 falconsiena_rx_hdr_split_enable(
54 __in efx_nic_t *enp,
55 __in unsigned int hdr_buf_size,
56 __in unsigned int pld_buf_size);
57 #endif /* EFSYS_OPT_RX_HDR_SPLIT */
58
59 #if EFSYS_OPT_RX_SCATTER
60 static __checkReturn efx_rc_t
61 falconsiena_rx_scatter_enable(
62 __in efx_nic_t *enp,
63 __in unsigned int buf_size);
64 #endif /* EFSYS_OPT_RX_SCATTER */
65
66 #if EFSYS_OPT_RX_SCALE
67 static __checkReturn efx_rc_t
68 falconsiena_rx_scale_mode_set(
69 __in efx_nic_t *enp,
70 __in efx_rx_hash_alg_t alg,
71 __in efx_rx_hash_type_t type,
72 __in boolean_t insert);
73
74 static __checkReturn efx_rc_t
75 falconsiena_rx_scale_key_set(
76 __in efx_nic_t *enp,
77 __in_ecount(n) uint8_t *key,
78 __in size_t n);
79
80 static __checkReturn efx_rc_t
81 falconsiena_rx_scale_tbl_set(
82 __in efx_nic_t *enp,
83 __in_ecount(n) unsigned int *table,
84 __in size_t n);
85
86 #endif /* EFSYS_OPT_RX_SCALE */
87
88 static void
89 falconsiena_rx_qpost(
90 __in efx_rxq_t *erp,
91 __in_ecount(n) efsys_dma_addr_t *addrp,
92 __in size_t size,
93 __in unsigned int n,
94 __in unsigned int completed,
95 __in unsigned int added);
96
97 static void
98 falconsiena_rx_qpush(
99 __in efx_rxq_t *erp,
100 __in unsigned int added,
101 __inout unsigned int *pushedp);
102
103 static __checkReturn efx_rc_t
104 falconsiena_rx_qflush(
105 __in efx_rxq_t *erp);
106
107 static void
108 falconsiena_rx_qenable(
109 __in efx_rxq_t *erp);
110
111 static __checkReturn efx_rc_t
112 falconsiena_rx_qcreate(
113 __in efx_nic_t *enp,
114 __in unsigned int index,
115 __in unsigned int label,
116 __in efx_rxq_type_t type,
117 __in efsys_mem_t *esmp,
118 __in size_t n,
119 __in uint32_t id,
120 __in efx_evq_t *eep,
121 __in efx_rxq_t *erp);
122
123 static void
124 falconsiena_rx_qdestroy(
125 __in efx_rxq_t *erp);
126
127 #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
128
129
130 #if EFSYS_OPT_FALCON
131 static efx_rx_ops_t __efx_rx_falcon_ops = {
132 falconsiena_rx_init, /* erxo_init */
133 falconsiena_rx_fini, /* erxo_fini */
134 #if EFSYS_OPT_RX_HDR_SPLIT
135 falconsiena_rx_hdr_split_enable, /* erxo_hdr_split_enable */
136 #endif
137 #if EFSYS_OPT_RX_SCATTER
138 falconsiena_rx_scatter_enable, /* erxo_scatter_enable */
139 #endif
140 #if EFSYS_OPT_RX_SCALE
141 falconsiena_rx_scale_mode_set, /* erxo_scale_mode_set */
142 falconsiena_rx_scale_key_set, /* erxo_scale_key_set */
143 falconsiena_rx_scale_tbl_set, /* erxo_scale_tbl_set */
144 #endif
145 falconsiena_rx_qpost, /* erxo_qpost */
146 falconsiena_rx_qpush, /* erxo_qpush */
147 falconsiena_rx_qflush, /* erxo_qflush */
148 falconsiena_rx_qenable, /* erxo_qenable */
149 falconsiena_rx_qcreate, /* erxo_qcreate */
150 falconsiena_rx_qdestroy, /* erxo_qdestroy */
151 };
152 #endif /* EFSYS_OPT_FALCON */
153
154 #if EFSYS_OPT_SIENA
155 static efx_rx_ops_t __efx_rx_siena_ops = {
156 falconsiena_rx_init, /* erxo_init */
157 falconsiena_rx_fini, /* erxo_fini */
158 #if EFSYS_OPT_RX_HDR_SPLIT
159 falconsiena_rx_hdr_split_enable, /* erxo_hdr_split_enable */
160 #endif
161 #if EFSYS_OPT_RX_SCATTER
162 falconsiena_rx_scatter_enable, /* erxo_scatter_enable */
163 #endif
164 #if EFSYS_OPT_RX_SCALE
165 falconsiena_rx_scale_mode_set, /* erxo_scale_mode_set */
166 falconsiena_rx_scale_key_set, /* erxo_scale_key_set */
167 falconsiena_rx_scale_tbl_set, /* erxo_scale_tbl_set */
168 #endif
169 falconsiena_rx_qpost, /* erxo_qpost */
170 falconsiena_rx_qpush, /* erxo_qpush */
171 falconsiena_rx_qflush, /* erxo_qflush */
172 falconsiena_rx_qenable, /* erxo_qenable */
173 falconsiena_rx_qcreate, /* erxo_qcreate */
174 falconsiena_rx_qdestroy, /* erxo_qdestroy */
175 };
176 #endif /* EFSYS_OPT_SIENA */
177
178 #if EFSYS_OPT_HUNTINGTON
179 static efx_rx_ops_t __efx_rx_hunt_ops = {
180 hunt_rx_init, /* erxo_init */
181 hunt_rx_fini, /* erxo_fini */
182 #if EFSYS_OPT_RX_HDR_SPLIT
183 hunt_rx_hdr_split_enable, /* erxo_hdr_split_enable */
184 #endif
185 #if EFSYS_OPT_RX_SCATTER
186 hunt_rx_scatter_enable, /* erxo_scatter_enable */
187 #endif
188 #if EFSYS_OPT_RX_SCALE
189 hunt_rx_scale_mode_set, /* erxo_scale_mode_set */
190 hunt_rx_scale_key_set, /* erxo_scale_key_set */
191 hunt_rx_scale_tbl_set, /* erxo_scale_tbl_set */
192 #endif
193 hunt_rx_qpost, /* erxo_qpost */
194 hunt_rx_qpush, /* erxo_qpush */
195 hunt_rx_qflush, /* erxo_qflush */
196 hunt_rx_qenable, /* erxo_qenable */
197 hunt_rx_qcreate, /* erxo_qcreate */
198 hunt_rx_qdestroy, /* erxo_qdestroy */
199 };
200 #endif /* EFSYS_OPT_HUNTINGTON */
201
202
203 __checkReturn efx_rc_t
efx_rx_init(__inout efx_nic_t * enp)204 efx_rx_init(
205 __inout efx_nic_t *enp)
206 {
207 efx_rx_ops_t *erxop;
208 efx_rc_t rc;
209
210 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
211 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
212
213 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
214 rc = EINVAL;
215 goto fail1;
216 }
217
218 if (enp->en_mod_flags & EFX_MOD_RX) {
219 rc = EINVAL;
220 goto fail2;
221 }
222
223 switch (enp->en_family) {
224 #if EFSYS_OPT_FALCON
225 case EFX_FAMILY_FALCON:
226 erxop = (efx_rx_ops_t *)&__efx_rx_falcon_ops;
227 break;
228 #endif /* EFSYS_OPT_FALCON */
229
230 #if EFSYS_OPT_SIENA
231 case EFX_FAMILY_SIENA:
232 erxop = (efx_rx_ops_t *)&__efx_rx_siena_ops;
233 break;
234 #endif /* EFSYS_OPT_SIENA */
235
236 #if EFSYS_OPT_HUNTINGTON
237 case EFX_FAMILY_HUNTINGTON:
238 erxop = (efx_rx_ops_t *)&__efx_rx_hunt_ops;
239 break;
240 #endif /* EFSYS_OPT_HUNTINGTON */
241
242 default:
243 EFSYS_ASSERT(0);
244 rc = ENOTSUP;
245 goto fail3;
246 }
247
248 if ((rc = erxop->erxo_init(enp)) != 0)
249 goto fail4;
250
251 enp->en_erxop = erxop;
252 enp->en_mod_flags |= EFX_MOD_RX;
253 return (0);
254
255 fail4:
256 EFSYS_PROBE(fail4);
257 fail3:
258 EFSYS_PROBE(fail3);
259 fail2:
260 EFSYS_PROBE(fail2);
261 fail1:
262 EFSYS_PROBE1(fail1, efx_rc_t, rc);
263
264 enp->en_erxop = NULL;
265 enp->en_mod_flags &= ~EFX_MOD_RX;
266 return (rc);
267 }
268
269 void
efx_rx_fini(__in efx_nic_t * enp)270 efx_rx_fini(
271 __in efx_nic_t *enp)
272 {
273 efx_rx_ops_t *erxop = enp->en_erxop;
274
275 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
276 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
277 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
278 EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
279
280 erxop->erxo_fini(enp);
281
282 enp->en_erxop = NULL;
283 enp->en_mod_flags &= ~EFX_MOD_RX;
284 }
285
286 #if EFSYS_OPT_RX_HDR_SPLIT
287 __checkReturn efx_rc_t
efx_rx_hdr_split_enable(__in efx_nic_t * enp,__in unsigned int hdr_buf_size,__in unsigned int pld_buf_size)288 efx_rx_hdr_split_enable(
289 __in efx_nic_t *enp,
290 __in unsigned int hdr_buf_size,
291 __in unsigned int pld_buf_size)
292 {
293 efx_rx_ops_t *erxop = enp->en_erxop;
294 efx_rc_t rc;
295
296 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
297 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
298 EFSYS_ASSERT3U(enp->en_family, >=, EFX_FAMILY_SIENA);
299
300 if ((rc = erxop->erxo_hdr_split_enable(enp, hdr_buf_size,
301 pld_buf_size)) != 0)
302 goto fail1;
303
304 return (0);
305
306 fail1:
307 EFSYS_PROBE1(fail1, efx_rc_t, rc);
308 return (rc);
309 }
310 #endif /* EFSYS_OPT_RX_HDR_SPLIT */
311
312 #if EFSYS_OPT_RX_SCATTER
313 __checkReturn efx_rc_t
efx_rx_scatter_enable(__in efx_nic_t * enp,__in unsigned int buf_size)314 efx_rx_scatter_enable(
315 __in efx_nic_t *enp,
316 __in unsigned int buf_size)
317 {
318 efx_rx_ops_t *erxop = enp->en_erxop;
319 efx_rc_t rc;
320
321 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
322 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
323
324 if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0)
325 goto fail1;
326
327 return (0);
328
329 fail1:
330 EFSYS_PROBE1(fail1, efx_rc_t, rc);
331 return (rc);
332 }
333 #endif /* EFSYS_OPT_RX_SCATTER */
334
335 #if EFSYS_OPT_RX_SCALE
336 __checkReturn efx_rc_t
efx_rx_hash_support_get(__in efx_nic_t * enp,__out efx_rx_hash_support_t * supportp)337 efx_rx_hash_support_get(
338 __in efx_nic_t *enp,
339 __out efx_rx_hash_support_t *supportp)
340 {
341 efx_rc_t rc;
342
343 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
344 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
345
346 if (supportp == NULL) {
347 rc = EINVAL;
348 goto fail1;
349 }
350
351 /* Report if resources are available to insert RX hash value */
352 *supportp = enp->en_hash_support;
353
354 return (0);
355
356 fail1:
357 EFSYS_PROBE1(fail1, efx_rc_t, rc);
358
359 return (rc);
360 }
361
362 __checkReturn efx_rc_t
efx_rx_scale_support_get(__in efx_nic_t * enp,__out efx_rx_scale_support_t * supportp)363 efx_rx_scale_support_get(
364 __in efx_nic_t *enp,
365 __out efx_rx_scale_support_t *supportp)
366 {
367 efx_rc_t rc;
368
369 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
370 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
371
372 if (supportp == NULL) {
373 rc = EINVAL;
374 goto fail1;
375 }
376
377 /* Report if resources are available to support RSS */
378 *supportp = enp->en_rss_support;
379
380 return (0);
381
382 fail1:
383 EFSYS_PROBE1(fail1, efx_rc_t, rc);
384
385 return (rc);
386 }
387
388 __checkReturn efx_rc_t
efx_rx_scale_mode_set(__in efx_nic_t * enp,__in efx_rx_hash_alg_t alg,__in efx_rx_hash_type_t type,__in boolean_t insert)389 efx_rx_scale_mode_set(
390 __in efx_nic_t *enp,
391 __in efx_rx_hash_alg_t alg,
392 __in efx_rx_hash_type_t type,
393 __in boolean_t insert)
394 {
395 efx_rx_ops_t *erxop = enp->en_erxop;
396 efx_rc_t rc;
397
398 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
399 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
400
401 if (erxop->erxo_scale_mode_set != NULL) {
402 if ((rc = erxop->erxo_scale_mode_set(enp, alg,
403 type, insert)) != 0)
404 goto fail1;
405 }
406
407 return (0);
408
409 fail1:
410 EFSYS_PROBE1(fail1, efx_rc_t, rc);
411 return (rc);
412 }
413 #endif /* EFSYS_OPT_RX_SCALE */
414
415 #if EFSYS_OPT_RX_SCALE
416 __checkReturn efx_rc_t
efx_rx_scale_key_set(__in efx_nic_t * enp,__in_ecount (n)uint8_t * key,__in size_t n)417 efx_rx_scale_key_set(
418 __in efx_nic_t *enp,
419 __in_ecount(n) uint8_t *key,
420 __in size_t n)
421 {
422 efx_rx_ops_t *erxop = enp->en_erxop;
423 efx_rc_t rc;
424
425 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
426 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
427
428 if ((rc = erxop->erxo_scale_key_set(enp, key, n)) != 0)
429 goto fail1;
430
431 return (0);
432
433 fail1:
434 EFSYS_PROBE1(fail1, efx_rc_t, rc);
435
436 return (rc);
437 }
438 #endif /* EFSYS_OPT_RX_SCALE */
439
440 #if EFSYS_OPT_RX_SCALE
441 __checkReturn efx_rc_t
efx_rx_scale_tbl_set(__in efx_nic_t * enp,__in_ecount (n)unsigned int * table,__in size_t n)442 efx_rx_scale_tbl_set(
443 __in efx_nic_t *enp,
444 __in_ecount(n) unsigned int *table,
445 __in size_t n)
446 {
447 efx_rx_ops_t *erxop = enp->en_erxop;
448 efx_rc_t rc;
449
450 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
451 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
452
453 if ((rc = erxop->erxo_scale_tbl_set(enp, table, n)) != 0)
454 goto fail1;
455
456 return (0);
457
458 fail1:
459 EFSYS_PROBE1(fail1, efx_rc_t, rc);
460
461 return (rc);
462 }
463 #endif /* EFSYS_OPT_RX_SCALE */
464
465 void
efx_rx_qpost(__in efx_rxq_t * erp,__in_ecount (n)efsys_dma_addr_t * addrp,__in size_t size,__in unsigned int n,__in unsigned int completed,__in unsigned int added)466 efx_rx_qpost(
467 __in efx_rxq_t *erp,
468 __in_ecount(n) efsys_dma_addr_t *addrp,
469 __in size_t size,
470 __in unsigned int n,
471 __in unsigned int completed,
472 __in unsigned int added)
473 {
474 efx_nic_t *enp = erp->er_enp;
475 efx_rx_ops_t *erxop = enp->en_erxop;
476
477 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
478
479 erxop->erxo_qpost(erp, addrp, size, n, completed, added);
480 }
481
482 void
efx_rx_qpush(__in efx_rxq_t * erp,__in unsigned int added,__inout unsigned int * pushedp)483 efx_rx_qpush(
484 __in efx_rxq_t *erp,
485 __in unsigned int added,
486 __inout unsigned int *pushedp)
487 {
488 efx_nic_t *enp = erp->er_enp;
489 efx_rx_ops_t *erxop = enp->en_erxop;
490
491 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
492
493 erxop->erxo_qpush(erp, added, pushedp);
494 }
495
496 __checkReturn efx_rc_t
efx_rx_qflush(__in efx_rxq_t * erp)497 efx_rx_qflush(
498 __in efx_rxq_t *erp)
499 {
500 efx_nic_t *enp = erp->er_enp;
501 efx_rx_ops_t *erxop = enp->en_erxop;
502 efx_rc_t rc;
503
504 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
505
506 if ((rc = erxop->erxo_qflush(erp)) != 0)
507 goto fail1;
508
509 return (0);
510
511 fail1:
512 EFSYS_PROBE1(fail1, efx_rc_t, rc);
513
514 return (rc);
515 }
516
517 void
efx_rx_qenable(__in efx_rxq_t * erp)518 efx_rx_qenable(
519 __in efx_rxq_t *erp)
520 {
521 efx_nic_t *enp = erp->er_enp;
522 efx_rx_ops_t *erxop = enp->en_erxop;
523
524 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
525
526 erxop->erxo_qenable(erp);
527 }
528
529 __checkReturn efx_rc_t
efx_rx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efx_rxq_type_t type,__in efsys_mem_t * esmp,__in size_t n,__in uint32_t id,__in efx_evq_t * eep,__deref_out efx_rxq_t ** erpp)530 efx_rx_qcreate(
531 __in efx_nic_t *enp,
532 __in unsigned int index,
533 __in unsigned int label,
534 __in efx_rxq_type_t type,
535 __in efsys_mem_t *esmp,
536 __in size_t n,
537 __in uint32_t id,
538 __in efx_evq_t *eep,
539 __deref_out efx_rxq_t **erpp)
540 {
541 efx_rx_ops_t *erxop = enp->en_erxop;
542 efx_rxq_t *erp;
543 efx_rc_t rc;
544
545 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
546 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
547
548 /* Allocate an RXQ object */
549 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
550
551 if (erp == NULL) {
552 rc = ENOMEM;
553 goto fail1;
554 }
555
556 erp->er_magic = EFX_RXQ_MAGIC;
557 erp->er_enp = enp;
558 erp->er_index = index;
559 erp->er_mask = n - 1;
560 erp->er_esmp = esmp;
561
562 if ((rc = erxop->erxo_qcreate(enp, index, label, type, esmp, n, id,
563 eep, erp)) != 0)
564 goto fail2;
565
566 enp->en_rx_qcount++;
567 *erpp = erp;
568
569 return (0);
570
571 fail2:
572 EFSYS_PROBE(fail2);
573
574 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
575 fail1:
576 EFSYS_PROBE1(fail1, efx_rc_t, rc);
577
578 return (rc);
579 }
580
581 void
efx_rx_qdestroy(__in efx_rxq_t * erp)582 efx_rx_qdestroy(
583 __in efx_rxq_t *erp)
584 {
585 efx_nic_t *enp = erp->er_enp;
586 efx_rx_ops_t *erxop = enp->en_erxop;
587
588 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
589
590 erxop->erxo_qdestroy(erp);
591 }
592
593 /*
594 * Psuedo-header info for Siena/Falcon.
595 *
596 * The psuedo-header is a byte array of one of the forms:
597 *
598 * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
599 * XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.TT.TT.TT.TT
600 * XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.XX.LL.LL
601 *
602 * where:
603 *
604 * TT.TT.TT.TT is a 32-bit Toeplitz hash
605 * LL.LL is a 16-bit LFSR hash
606 *
607 * Hash values are in network (big-endian) byte order.
608 *
609 *
610 * On Huntington the pseudo-header is laid out as:
611 * (See also SF-109306-TC section 9)
612 *
613 * Toeplitz hash (32 bits, little-endian)
614 * Out-of-band outer VLAN tag
615 * (16 bits, big-endian, 0 if the packet did not have an outer VLAN tag)
616 * Out-of-band inner VLAN tag
617 * (16 bits, big-endian, 0 if the packet did not have an inner VLAN tag)
618 * Packet length (16 bits, little-endian, may be 0)
619 * MAC timestamp (32 bits, little-endian, may be 0)
620 * VLAN tag
621 * (16 bits, big-endian, 0 if the packet did not have an outer VLAN tag)
622 * VLAN tag
623 * (16 bits, big-endian, 0 if the packet did not have an inner VLAN tag)
624 */
625
626 __checkReturn efx_rc_t
efx_psuedo_hdr_pkt_length_get(__in efx_nic_t * enp,__in uint8_t * buffer,__out uint16_t * pkt_lengthp)627 efx_psuedo_hdr_pkt_length_get(
628 __in efx_nic_t *enp,
629 __in uint8_t *buffer,
630 __out uint16_t *pkt_lengthp)
631 {
632 if (enp->en_family != EFX_FAMILY_HUNTINGTON) {
633 EFSYS_ASSERT(0);
634 return (ENOTSUP);
635 }
636
637 *pkt_lengthp = buffer[8] | (buffer[9] << 8);
638
639 return (0);
640 }
641
642 #if EFSYS_OPT_RX_SCALE
643
644 uint32_t
efx_psuedo_hdr_hash_get(__in efx_nic_t * enp,__in efx_rx_hash_alg_t func,__in uint8_t * buffer)645 efx_psuedo_hdr_hash_get(
646 __in efx_nic_t *enp,
647 __in efx_rx_hash_alg_t func,
648 __in uint8_t *buffer)
649 {
650 if (func == EFX_RX_HASHALG_TOEPLITZ) {
651 switch (enp->en_family) {
652 case EFX_FAMILY_FALCON:
653 case EFX_FAMILY_SIENA:
654 return ((buffer[12] << 24) |
655 (buffer[13] << 16) |
656 (buffer[14] << 8) |
657 buffer[15]);
658 case EFX_FAMILY_HUNTINGTON:
659 return (buffer[0] |
660 (buffer[1] << 8) |
661 (buffer[2] << 16) |
662 (buffer[3] << 24));
663 default:
664 EFSYS_ASSERT(0);
665 return (0);
666 }
667 } else if (func == EFX_RX_HASHALG_LFSR) {
668 EFSYS_ASSERT(enp->en_family == EFX_FAMILY_FALCON ||
669 enp->en_family == EFX_FAMILY_SIENA);
670 return ((buffer[14] << 8) | buffer[15]);
671 } else {
672 EFSYS_ASSERT(0);
673 return (0);
674 }
675 }
676
677 #endif /* EFSYS_OPT_RX_SCALE */
678
679 #if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
680
681 static __checkReturn efx_rc_t
falconsiena_rx_init(__in efx_nic_t * enp)682 falconsiena_rx_init(
683 __in efx_nic_t *enp)
684 {
685 efx_oword_t oword;
686 unsigned int index;
687
688 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
689
690 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
691 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
692 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
693 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
694 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
695 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
696 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
697
698 /* Zero the RSS table */
699 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
700 index++) {
701 EFX_ZERO_OWORD(oword);
702 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
703 index, &oword, B_TRUE);
704 }
705
706 #if EFSYS_OPT_RX_SCALE
707 /* The RSS key and indirection table are writable. */
708 enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE;
709
710 /* Hardware can insert RX hash with/without RSS */
711 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
712 #endif /* EFSYS_OPT_RX_SCALE */
713
714 return (0);
715 }
716
717 #if EFSYS_OPT_RX_HDR_SPLIT
718 static __checkReturn efx_rc_t
falconsiena_rx_hdr_split_enable(__in efx_nic_t * enp,__in unsigned int hdr_buf_size,__in unsigned int pld_buf_size)719 falconsiena_rx_hdr_split_enable(
720 __in efx_nic_t *enp,
721 __in unsigned int hdr_buf_size,
722 __in unsigned int pld_buf_size)
723 {
724 unsigned int nhdr32;
725 unsigned int npld32;
726 efx_oword_t oword;
727 efx_rc_t rc;
728
729 nhdr32 = hdr_buf_size / 32;
730 if ((nhdr32 == 0) ||
731 (nhdr32 >= (1 << FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE_WIDTH)) ||
732 ((hdr_buf_size % 32) != 0)) {
733 rc = EINVAL;
734 goto fail1;
735 }
736
737 npld32 = pld_buf_size / 32;
738 if ((npld32 == 0) ||
739 (npld32 >= (1 << FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE_WIDTH)) ||
740 ((pld_buf_size % 32) != 0)) {
741 rc = EINVAL;
742 goto fail2;
743 }
744
745 if (enp->en_rx_qcount > 0) {
746 rc = EBUSY;
747 goto fail3;
748 }
749
750 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
751
752 EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_EN, 1);
753 EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_HDR_BUF_SIZE, nhdr32);
754 EFX_SET_OWORD_FIELD(oword, FRF_CZ_RX_HDR_SPLIT_PLD_BUF_SIZE, npld32);
755
756 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
757
758 return (0);
759
760 fail3:
761 EFSYS_PROBE(fail3);
762 fail2:
763 EFSYS_PROBE(fail2);
764 fail1:
765 EFSYS_PROBE1(fail1, efx_rc_t, rc);
766
767 return (rc);
768 }
769 #endif /* EFSYS_OPT_RX_HDR_SPLIT */
770
771 #if EFSYS_OPT_RX_SCATTER
772 static __checkReturn efx_rc_t
falconsiena_rx_scatter_enable(__in efx_nic_t * enp,__in unsigned int buf_size)773 falconsiena_rx_scatter_enable(
774 __in efx_nic_t *enp,
775 __in unsigned int buf_size)
776 {
777 unsigned int nbuf32;
778 efx_oword_t oword;
779 efx_rc_t rc;
780
781 nbuf32 = buf_size / 32;
782 if ((nbuf32 == 0) ||
783 (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
784 ((buf_size % 32) != 0)) {
785 rc = EINVAL;
786 goto fail1;
787 }
788
789 if (enp->en_rx_qcount > 0) {
790 rc = EBUSY;
791 goto fail2;
792 }
793
794 /* Set scatter buffer size */
795 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
796 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
797 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
798
799 /* Enable scatter for packets not matching a filter */
800 EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
801 EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
802 EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
803
804 return (0);
805
806 fail2:
807 EFSYS_PROBE(fail2);
808 fail1:
809 EFSYS_PROBE1(fail1, efx_rc_t, rc);
810
811 return (rc);
812 }
813 #endif /* EFSYS_OPT_RX_SCATTER */
814
815
816 #define EFX_RX_LFSR_HASH(_enp, _insert) \
817 do { \
818 efx_oword_t oword; \
819 \
820 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \
821 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \
822 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \
823 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \
824 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \
825 (_insert) ? 1 : 0); \
826 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \
827 \
828 if ((_enp)->en_family == EFX_FAMILY_SIENA) { \
829 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \
830 &oword); \
831 EFX_SET_OWORD_FIELD(oword, \
832 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \
833 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \
834 &oword); \
835 } \
836 \
837 _NOTE(CONSTANTCONDITION) \
838 } while (B_FALSE)
839
840 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \
841 do { \
842 efx_oword_t oword; \
843 \
844 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \
845 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \
846 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \
847 (_ip) ? 1 : 0); \
848 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \
849 (_tcp) ? 0 : 1); \
850 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \
851 (_insert) ? 1 : 0); \
852 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \
853 \
854 _NOTE(CONSTANTCONDITION) \
855 } while (B_FALSE)
856
857 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \
858 do { \
859 efx_oword_t oword; \
860 \
861 if ((_enp)->en_family == EFX_FAMILY_FALCON) { \
862 (_rc) = ((_ip) || (_tcp)) ? ENOTSUP : 0; \
863 break; \
864 } \
865 \
866 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
867 EFX_SET_OWORD_FIELD(oword, \
868 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \
869 EFX_SET_OWORD_FIELD(oword, \
870 FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
871 EFX_SET_OWORD_FIELD(oword, \
872 FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \
873 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
874 \
875 (_rc) = 0; \
876 \
877 _NOTE(CONSTANTCONDITION) \
878 } while (B_FALSE)
879
880
881 #if EFSYS_OPT_RX_SCALE
882
883 static __checkReturn efx_rc_t
falconsiena_rx_scale_mode_set(__in efx_nic_t * enp,__in efx_rx_hash_alg_t alg,__in efx_rx_hash_type_t type,__in boolean_t insert)884 falconsiena_rx_scale_mode_set(
885 __in efx_nic_t *enp,
886 __in efx_rx_hash_alg_t alg,
887 __in efx_rx_hash_type_t type,
888 __in boolean_t insert)
889 {
890 efx_rc_t rc;
891
892 switch (alg) {
893 case EFX_RX_HASHALG_LFSR:
894 EFX_RX_LFSR_HASH(enp, insert);
895 break;
896
897 case EFX_RX_HASHALG_TOEPLITZ:
898 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
899 type & (1 << EFX_RX_HASH_IPV4),
900 type & (1 << EFX_RX_HASH_TCPIPV4));
901
902 EFX_RX_TOEPLITZ_IPV6_HASH(enp,
903 type & (1 << EFX_RX_HASH_IPV6),
904 type & (1 << EFX_RX_HASH_TCPIPV6),
905 rc);
906 if (rc != 0)
907 goto fail1;
908
909 break;
910
911 default:
912 rc = EINVAL;
913 goto fail2;
914 }
915
916 return (0);
917
918 fail2:
919 EFSYS_PROBE(fail2);
920 fail1:
921 EFSYS_PROBE1(fail1, efx_rc_t, rc);
922
923 EFX_RX_LFSR_HASH(enp, B_FALSE);
924
925 return (rc);
926 }
927 #endif
928
929 #if EFSYS_OPT_RX_SCALE
930 static __checkReturn efx_rc_t
falconsiena_rx_scale_key_set(__in efx_nic_t * enp,__in_ecount (n)uint8_t * key,__in size_t n)931 falconsiena_rx_scale_key_set(
932 __in efx_nic_t *enp,
933 __in_ecount(n) uint8_t *key,
934 __in size_t n)
935 {
936 efx_oword_t oword;
937 unsigned int byte;
938 unsigned int offset;
939 efx_rc_t rc;
940
941 byte = 0;
942
943 /* Write Toeplitz IPv4 hash key */
944 EFX_ZERO_OWORD(oword);
945 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
946 offset > 0 && byte < n;
947 --offset)
948 oword.eo_u8[offset - 1] = key[byte++];
949
950 EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
951
952 byte = 0;
953
954 /* Verify Toeplitz IPv4 hash key */
955 EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
956 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
957 offset > 0 && byte < n;
958 --offset) {
959 if (oword.eo_u8[offset - 1] != key[byte++]) {
960 rc = EFAULT;
961 goto fail1;
962 }
963 }
964
965 if ((enp->en_features & EFX_FEATURE_IPV6) == 0)
966 goto done;
967
968 EFSYS_ASSERT3U(enp->en_family, !=, EFX_FAMILY_FALCON);
969
970 byte = 0;
971
972 /* Write Toeplitz IPv6 hash key 3 */
973 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
974 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
975 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
976 offset > 0 && byte < n;
977 --offset)
978 oword.eo_u8[offset - 1] = key[byte++];
979
980 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
981
982 /* Write Toeplitz IPv6 hash key 2 */
983 EFX_ZERO_OWORD(oword);
984 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
985 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
986 offset > 0 && byte < n;
987 --offset)
988 oword.eo_u8[offset - 1] = key[byte++];
989
990 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
991
992 /* Write Toeplitz IPv6 hash key 1 */
993 EFX_ZERO_OWORD(oword);
994 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
995 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
996 offset > 0 && byte < n;
997 --offset)
998 oword.eo_u8[offset - 1] = key[byte++];
999
1000 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
1001
1002 byte = 0;
1003
1004 /* Verify Toeplitz IPv6 hash key 3 */
1005 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
1006 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
1007 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
1008 offset > 0 && byte < n;
1009 --offset) {
1010 if (oword.eo_u8[offset - 1] != key[byte++]) {
1011 rc = EFAULT;
1012 goto fail2;
1013 }
1014 }
1015
1016 /* Verify Toeplitz IPv6 hash key 2 */
1017 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
1018 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
1019 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
1020 offset > 0 && byte < n;
1021 --offset) {
1022 if (oword.eo_u8[offset - 1] != key[byte++]) {
1023 rc = EFAULT;
1024 goto fail3;
1025 }
1026 }
1027
1028 /* Verify Toeplitz IPv6 hash key 1 */
1029 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
1030 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
1031 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
1032 offset > 0 && byte < n;
1033 --offset) {
1034 if (oword.eo_u8[offset - 1] != key[byte++]) {
1035 rc = EFAULT;
1036 goto fail4;
1037 }
1038 }
1039
1040 done:
1041 return (0);
1042
1043 fail4:
1044 EFSYS_PROBE(fail4);
1045 fail3:
1046 EFSYS_PROBE(fail3);
1047 fail2:
1048 EFSYS_PROBE(fail2);
1049 fail1:
1050 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1051
1052 return (rc);
1053 }
1054 #endif
1055
1056 #if EFSYS_OPT_RX_SCALE
1057 static __checkReturn efx_rc_t
falconsiena_rx_scale_tbl_set(__in efx_nic_t * enp,__in_ecount (n)unsigned int * table,__in size_t n)1058 falconsiena_rx_scale_tbl_set(
1059 __in efx_nic_t *enp,
1060 __in_ecount(n) unsigned int *table,
1061 __in size_t n)
1062 {
1063 efx_oword_t oword;
1064 int index;
1065 efx_rc_t rc;
1066
1067 EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
1068 EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
1069
1070 if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
1071 rc = EINVAL;
1072 goto fail1;
1073 }
1074
1075 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
1076 uint32_t byte;
1077
1078 /* Calculate the entry to place in the table */
1079 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
1080
1081 EFSYS_PROBE2(table, int, index, uint32_t, byte);
1082
1083 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
1084
1085 /* Write the table */
1086 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
1087 index, &oword, B_TRUE);
1088 }
1089
1090 for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
1091 uint32_t byte;
1092
1093 /* Determine if we're starting a new batch */
1094 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
1095
1096 /* Read the table */
1097 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
1098 index, &oword, B_TRUE);
1099
1100 /* Verify the entry */
1101 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
1102 rc = EFAULT;
1103 goto fail2;
1104 }
1105 }
1106
1107 return (0);
1108
1109 fail2:
1110 EFSYS_PROBE(fail2);
1111 fail1:
1112 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1113
1114 return (rc);
1115 }
1116 #endif
1117
1118 static void
falconsiena_rx_qpost(__in efx_rxq_t * erp,__in_ecount (n)efsys_dma_addr_t * addrp,__in size_t size,__in unsigned int n,__in unsigned int completed,__in unsigned int added)1119 falconsiena_rx_qpost(
1120 __in efx_rxq_t *erp,
1121 __in_ecount(n) efsys_dma_addr_t *addrp,
1122 __in size_t size,
1123 __in unsigned int n,
1124 __in unsigned int completed,
1125 __in unsigned int added)
1126 {
1127 efx_qword_t qword;
1128 unsigned int i;
1129 unsigned int offset;
1130 unsigned int id;
1131
1132 /* The client driver must not overfill the queue */
1133 EFSYS_ASSERT3U(added - completed + n, <=,
1134 EFX_RXQ_LIMIT(erp->er_mask + 1));
1135
1136 id = added & (erp->er_mask);
1137 for (i = 0; i < n; i++) {
1138 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
1139 unsigned int, id, efsys_dma_addr_t, addrp[i],
1140 size_t, size);
1141
1142 EFX_POPULATE_QWORD_3(qword,
1143 FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
1144 FSF_AZ_RX_KER_BUF_ADDR_DW0,
1145 (uint32_t)(addrp[i] & 0xffffffff),
1146 FSF_AZ_RX_KER_BUF_ADDR_DW1,
1147 (uint32_t)(addrp[i] >> 32));
1148
1149 offset = id * sizeof (efx_qword_t);
1150 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
1151
1152 id = (id + 1) & (erp->er_mask);
1153 }
1154 }
1155
1156 static void
falconsiena_rx_qpush(__in efx_rxq_t * erp,__in unsigned int added,__inout unsigned int * pushedp)1157 falconsiena_rx_qpush(
1158 __in efx_rxq_t *erp,
1159 __in unsigned int added,
1160 __inout unsigned int *pushedp)
1161 {
1162 efx_nic_t *enp = erp->er_enp;
1163 unsigned int pushed = *pushedp;
1164 uint32_t wptr;
1165 efx_oword_t oword;
1166 efx_dword_t dword;
1167
1168 /* All descriptors are pushed */
1169 *pushedp = added;
1170
1171 /* Push the populated descriptors out */
1172 wptr = added & erp->er_mask;
1173
1174 EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
1175
1176 /* Only write the third DWORD */
1177 EFX_POPULATE_DWORD_1(dword,
1178 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
1179
1180 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
1181 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
1182 wptr, pushed & erp->er_mask);
1183 EFSYS_PIO_WRITE_BARRIER();
1184 EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
1185 erp->er_index, &dword, B_FALSE);
1186 }
1187
1188 static __checkReturn efx_rc_t
falconsiena_rx_qflush(__in efx_rxq_t * erp)1189 falconsiena_rx_qflush(
1190 __in efx_rxq_t *erp)
1191 {
1192 efx_nic_t *enp = erp->er_enp;
1193 efx_oword_t oword;
1194 uint32_t label;
1195
1196 label = erp->er_index;
1197
1198 /* Flush the queue */
1199 EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
1200 FRF_AZ_RX_FLUSH_DESCQ, label);
1201 EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
1202
1203 return (0);
1204 }
1205
1206 static void
falconsiena_rx_qenable(__in efx_rxq_t * erp)1207 falconsiena_rx_qenable(
1208 __in efx_rxq_t *erp)
1209 {
1210 efx_nic_t *enp = erp->er_enp;
1211 efx_oword_t oword;
1212
1213 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1214
1215 EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
1216 erp->er_index, &oword, B_TRUE);
1217
1218 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
1219 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
1220 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
1221
1222 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1223 erp->er_index, &oword, B_TRUE);
1224 }
1225
1226 static __checkReturn efx_rc_t
falconsiena_rx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efx_rxq_type_t type,__in efsys_mem_t * esmp,__in size_t n,__in uint32_t id,__in efx_evq_t * eep,__in efx_rxq_t * erp)1227 falconsiena_rx_qcreate(
1228 __in efx_nic_t *enp,
1229 __in unsigned int index,
1230 __in unsigned int label,
1231 __in efx_rxq_type_t type,
1232 __in efsys_mem_t *esmp,
1233 __in size_t n,
1234 __in uint32_t id,
1235 __in efx_evq_t *eep,
1236 __in efx_rxq_t *erp)
1237 {
1238 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1239 efx_oword_t oword;
1240 uint32_t size;
1241 boolean_t split;
1242 boolean_t jumbo;
1243 efx_rc_t rc;
1244
1245 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS ==
1246 (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
1247 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1248 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
1249
1250 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
1251 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
1252
1253 if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) {
1254 rc = EINVAL;
1255 goto fail1;
1256 }
1257 if (index >= encp->enc_rxq_limit) {
1258 rc = EINVAL;
1259 goto fail2;
1260 }
1261 for (size = 0; (1 << size) <= (EFX_RXQ_MAXNDESCS / EFX_RXQ_MINNDESCS);
1262 size++)
1263 if ((1 << size) == (int)(n / EFX_RXQ_MINNDESCS))
1264 break;
1265 if (id + (1 << size) >= encp->enc_buftbl_limit) {
1266 rc = EINVAL;
1267 goto fail3;
1268 }
1269
1270 switch (type) {
1271 case EFX_RXQ_TYPE_DEFAULT:
1272 split = B_FALSE;
1273 jumbo = B_FALSE;
1274 break;
1275
1276 #if EFSYS_OPT_RX_HDR_SPLIT
1277 case EFX_RXQ_TYPE_SPLIT_HEADER:
1278 if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) != 0)) {
1279 rc = EINVAL;
1280 goto fail4;
1281 }
1282 split = B_TRUE;
1283 jumbo = B_TRUE;
1284 break;
1285
1286 case EFX_RXQ_TYPE_SPLIT_PAYLOAD:
1287 if ((enp->en_family < EFX_FAMILY_SIENA) || ((index & 1) == 0)) {
1288 rc = EINVAL;
1289 goto fail4;
1290 }
1291 split = B_FALSE;
1292 jumbo = B_TRUE;
1293 break;
1294 #endif /* EFSYS_OPT_RX_HDR_SPLIT */
1295
1296 #if EFSYS_OPT_RX_SCATTER
1297 case EFX_RXQ_TYPE_SCATTER:
1298 if (enp->en_family < EFX_FAMILY_SIENA) {
1299 rc = EINVAL;
1300 goto fail4;
1301 }
1302 split = B_FALSE;
1303 jumbo = B_TRUE;
1304 break;
1305 #endif /* EFSYS_OPT_RX_SCATTER */
1306
1307 default:
1308 rc = EINVAL;
1309 goto fail4;
1310 }
1311
1312 /* Set up the new descriptor queue */
1313 EFX_POPULATE_OWORD_10(oword,
1314 FRF_CZ_RX_HDR_SPLIT, split,
1315 FRF_AZ_RX_ISCSI_DDIG_EN, 0,
1316 FRF_AZ_RX_ISCSI_HDIG_EN, 0,
1317 FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
1318 FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
1319 FRF_AZ_RX_DESCQ_OWNER_ID, 0,
1320 FRF_AZ_RX_DESCQ_LABEL, label,
1321 FRF_AZ_RX_DESCQ_SIZE, size,
1322 FRF_AZ_RX_DESCQ_TYPE, 0,
1323 FRF_AZ_RX_DESCQ_JUMBO, jumbo);
1324
1325 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1326 erp->er_index, &oword, B_TRUE);
1327
1328 return (0);
1329
1330 fail4:
1331 EFSYS_PROBE(fail4);
1332 fail3:
1333 EFSYS_PROBE(fail3);
1334 fail2:
1335 EFSYS_PROBE(fail2);
1336 fail1:
1337 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1338
1339 return (rc);
1340 }
1341
1342 static void
falconsiena_rx_qdestroy(__in efx_rxq_t * erp)1343 falconsiena_rx_qdestroy(
1344 __in efx_rxq_t *erp)
1345 {
1346 efx_nic_t *enp = erp->er_enp;
1347 efx_oword_t oword;
1348
1349 EFSYS_ASSERT(enp->en_rx_qcount != 0);
1350 --enp->en_rx_qcount;
1351
1352 /* Purge descriptor queue */
1353 EFX_ZERO_OWORD(oword);
1354
1355 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1356 erp->er_index, &oword, B_TRUE);
1357
1358 /* Free the RXQ object */
1359 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1360 }
1361
1362 static void
falconsiena_rx_fini(__in efx_nic_t * enp)1363 falconsiena_rx_fini(
1364 __in efx_nic_t *enp)
1365 {
1366 _NOTE(ARGUNUSED(enp))
1367 }
1368
1369 #endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
1370