1 /*-
2 * Copyright (c) 2012-2016 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
29 */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD: stable/10/sys/dev/sfxge/common/ef10_rx.c 342516 2018-12-26 10:25:01Z arybchik $");
33
34 #include "efx.h"
35 #include "efx_impl.h"
36
37
38 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
39
40
41 static __checkReturn efx_rc_t
efx_mcdi_init_rxq(__in efx_nic_t * enp,__in uint32_t size,__in uint32_t target_evq,__in uint32_t label,__in uint32_t instance,__in efsys_mem_t * esmp,__in boolean_t disable_scatter)42 efx_mcdi_init_rxq(
43 __in efx_nic_t *enp,
44 __in uint32_t size,
45 __in uint32_t target_evq,
46 __in uint32_t label,
47 __in uint32_t instance,
48 __in efsys_mem_t *esmp,
49 __in boolean_t disable_scatter)
50 {
51 efx_mcdi_req_t req;
52 EFX_MCDI_DECLARE_BUF(payload,
53 MC_CMD_INIT_RXQ_IN_LEN(EFX_RXQ_NBUFS(EFX_RXQ_MAXNDESCS)),
54 MC_CMD_INIT_RXQ_OUT_LEN);
55 int npages = EFX_RXQ_NBUFS(size);
56 int i;
57 efx_qword_t *dma_addr;
58 uint64_t addr;
59 efx_rc_t rc;
60
61 /* If this changes, then the payload size might need to change. */
62 EFSYS_ASSERT3U(MC_CMD_INIT_RXQ_OUT_LEN, ==, 0);
63 EFSYS_ASSERT3U(size, <=, EFX_RXQ_MAXNDESCS);
64
65 req.emr_cmd = MC_CMD_INIT_RXQ;
66 req.emr_in_buf = payload;
67 req.emr_in_length = MC_CMD_INIT_RXQ_IN_LEN(npages);
68 req.emr_out_buf = payload;
69 req.emr_out_length = MC_CMD_INIT_RXQ_OUT_LEN;
70
71 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_SIZE, size);
72 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_TARGET_EVQ, target_evq);
73 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_LABEL, label);
74 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_INSTANCE, instance);
75 MCDI_IN_POPULATE_DWORD_6(req, INIT_RXQ_IN_FLAGS,
76 INIT_RXQ_IN_FLAG_BUFF_MODE, 0,
77 INIT_RXQ_IN_FLAG_HDR_SPLIT, 0,
78 INIT_RXQ_IN_FLAG_TIMESTAMP, 0,
79 INIT_RXQ_IN_CRC_MODE, 0,
80 INIT_RXQ_IN_FLAG_PREFIX, 1,
81 INIT_RXQ_IN_FLAG_DISABLE_SCATTER, disable_scatter);
82 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_OWNER_ID, 0);
83 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
84
85 dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR);
86 addr = EFSYS_MEM_ADDR(esmp);
87
88 for (i = 0; i < npages; i++) {
89 EFX_POPULATE_QWORD_2(*dma_addr,
90 EFX_DWORD_1, (uint32_t)(addr >> 32),
91 EFX_DWORD_0, (uint32_t)(addr & 0xffffffff));
92
93 dma_addr++;
94 addr += EFX_BUF_SIZE;
95 }
96
97 efx_mcdi_execute(enp, &req);
98
99 if (req.emr_rc != 0) {
100 rc = req.emr_rc;
101 goto fail1;
102 }
103
104 return (0);
105
106 fail1:
107 EFSYS_PROBE1(fail1, efx_rc_t, rc);
108
109 return (rc);
110 }
111
112 static __checkReturn efx_rc_t
efx_mcdi_fini_rxq(__in efx_nic_t * enp,__in uint32_t instance)113 efx_mcdi_fini_rxq(
114 __in efx_nic_t *enp,
115 __in uint32_t instance)
116 {
117 efx_mcdi_req_t req;
118 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FINI_RXQ_IN_LEN,
119 MC_CMD_FINI_RXQ_OUT_LEN);
120 efx_rc_t rc;
121
122 req.emr_cmd = MC_CMD_FINI_RXQ;
123 req.emr_in_buf = payload;
124 req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN;
125 req.emr_out_buf = payload;
126 req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN;
127
128 MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance);
129
130 efx_mcdi_execute_quiet(enp, &req);
131
132 if (req.emr_rc != 0) {
133 rc = req.emr_rc;
134 goto fail1;
135 }
136
137 return (0);
138
139 fail1:
140 /*
141 * EALREADY is not an error, but indicates that the MC has rebooted and
142 * that the RXQ has already been destroyed.
143 */
144 if (rc != EALREADY)
145 EFSYS_PROBE1(fail1, efx_rc_t, rc);
146
147 return (rc);
148 }
149
150 #if EFSYS_OPT_RX_SCALE
151 static __checkReturn efx_rc_t
efx_mcdi_rss_context_alloc(__in efx_nic_t * enp,__in efx_rx_scale_support_t scale_support,__in uint32_t num_queues,__out uint32_t * rss_contextp)152 efx_mcdi_rss_context_alloc(
153 __in efx_nic_t *enp,
154 __in efx_rx_scale_support_t scale_support,
155 __in uint32_t num_queues,
156 __out uint32_t *rss_contextp)
157 {
158 efx_mcdi_req_t req;
159 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
160 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN);
161 uint32_t rss_context;
162 uint32_t context_type;
163 efx_rc_t rc;
164
165 if (num_queues > EFX_MAXRSS) {
166 rc = EINVAL;
167 goto fail1;
168 }
169
170 switch (scale_support) {
171 case EFX_RX_SCALE_EXCLUSIVE:
172 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE;
173 break;
174 case EFX_RX_SCALE_SHARED:
175 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
176 break;
177 default:
178 rc = EINVAL;
179 goto fail2;
180 }
181
182 req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
183 req.emr_in_buf = payload;
184 req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
185 req.emr_out_buf = payload;
186 req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
187
188 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
189 EVB_PORT_ID_ASSIGNED);
190 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type);
191 /* NUM_QUEUES is only used to validate indirection table offsets */
192 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues);
193
194 efx_mcdi_execute(enp, &req);
195
196 if (req.emr_rc != 0) {
197 rc = req.emr_rc;
198 goto fail3;
199 }
200
201 if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
202 rc = EMSGSIZE;
203 goto fail4;
204 }
205
206 rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
207 if (rss_context == EF10_RSS_CONTEXT_INVALID) {
208 rc = ENOENT;
209 goto fail5;
210 }
211
212 *rss_contextp = rss_context;
213
214 return (0);
215
216 fail5:
217 EFSYS_PROBE(fail5);
218 fail4:
219 EFSYS_PROBE(fail4);
220 fail3:
221 EFSYS_PROBE(fail3);
222 fail2:
223 EFSYS_PROBE(fail2);
224 fail1:
225 EFSYS_PROBE1(fail1, efx_rc_t, rc);
226
227 return (rc);
228 }
229 #endif /* EFSYS_OPT_RX_SCALE */
230
231 #if EFSYS_OPT_RX_SCALE
232 static efx_rc_t
efx_mcdi_rss_context_free(__in efx_nic_t * enp,__in uint32_t rss_context)233 efx_mcdi_rss_context_free(
234 __in efx_nic_t *enp,
235 __in uint32_t rss_context)
236 {
237 efx_mcdi_req_t req;
238 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
239 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN);
240 efx_rc_t rc;
241
242 if (rss_context == EF10_RSS_CONTEXT_INVALID) {
243 rc = EINVAL;
244 goto fail1;
245 }
246
247 req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
248 req.emr_in_buf = payload;
249 req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
250 req.emr_out_buf = payload;
251 req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
252
253 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
254
255 efx_mcdi_execute_quiet(enp, &req);
256
257 if (req.emr_rc != 0) {
258 rc = req.emr_rc;
259 goto fail2;
260 }
261
262 return (0);
263
264 fail2:
265 EFSYS_PROBE(fail2);
266 fail1:
267 EFSYS_PROBE1(fail1, efx_rc_t, rc);
268
269 return (rc);
270 }
271 #endif /* EFSYS_OPT_RX_SCALE */
272
273 #if EFSYS_OPT_RX_SCALE
274 static efx_rc_t
efx_mcdi_rss_context_set_flags(__in efx_nic_t * enp,__in uint32_t rss_context,__in efx_rx_hash_type_t type)275 efx_mcdi_rss_context_set_flags(
276 __in efx_nic_t *enp,
277 __in uint32_t rss_context,
278 __in efx_rx_hash_type_t type)
279 {
280 efx_mcdi_req_t req;
281 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
282 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN);
283 efx_rc_t rc;
284
285 if (rss_context == EF10_RSS_CONTEXT_INVALID) {
286 rc = EINVAL;
287 goto fail1;
288 }
289
290 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
291 req.emr_in_buf = payload;
292 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
293 req.emr_out_buf = payload;
294 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
295
296 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
297 rss_context);
298
299 MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
300 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
301 (type & EFX_RX_HASH_IPV4) ? 1 : 0,
302 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
303 (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0,
304 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
305 (type & EFX_RX_HASH_IPV6) ? 1 : 0,
306 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
307 (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0);
308
309 efx_mcdi_execute(enp, &req);
310
311 if (req.emr_rc != 0) {
312 rc = req.emr_rc;
313 goto fail2;
314 }
315
316 return (0);
317
318 fail2:
319 EFSYS_PROBE(fail2);
320 fail1:
321 EFSYS_PROBE1(fail1, efx_rc_t, rc);
322
323 return (rc);
324 }
325 #endif /* EFSYS_OPT_RX_SCALE */
326
327 #if EFSYS_OPT_RX_SCALE
328 static efx_rc_t
efx_mcdi_rss_context_set_key(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (n)uint8_t * key,__in size_t n)329 efx_mcdi_rss_context_set_key(
330 __in efx_nic_t *enp,
331 __in uint32_t rss_context,
332 __in_ecount(n) uint8_t *key,
333 __in size_t n)
334 {
335 efx_mcdi_req_t req;
336 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
337 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN);
338 efx_rc_t rc;
339
340 if (rss_context == EF10_RSS_CONTEXT_INVALID) {
341 rc = EINVAL;
342 goto fail1;
343 }
344
345 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
346 req.emr_in_buf = payload;
347 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
348 req.emr_out_buf = payload;
349 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
350
351 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
352 rss_context);
353
354 EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
355 if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
356 rc = EINVAL;
357 goto fail2;
358 }
359
360 memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY),
361 key, n);
362
363 efx_mcdi_execute(enp, &req);
364
365 if (req.emr_rc != 0) {
366 rc = req.emr_rc;
367 goto fail3;
368 }
369
370 return (0);
371
372 fail3:
373 EFSYS_PROBE(fail3);
374 fail2:
375 EFSYS_PROBE(fail2);
376 fail1:
377 EFSYS_PROBE1(fail1, efx_rc_t, rc);
378
379 return (rc);
380 }
381 #endif /* EFSYS_OPT_RX_SCALE */
382
383 #if EFSYS_OPT_RX_SCALE
384 static efx_rc_t
efx_mcdi_rss_context_set_table(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (n)unsigned int * table,__in size_t n)385 efx_mcdi_rss_context_set_table(
386 __in efx_nic_t *enp,
387 __in uint32_t rss_context,
388 __in_ecount(n) unsigned int *table,
389 __in size_t n)
390 {
391 efx_mcdi_req_t req;
392 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
393 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN);
394 uint8_t *req_table;
395 int i, rc;
396
397 if (rss_context == EF10_RSS_CONTEXT_INVALID) {
398 rc = EINVAL;
399 goto fail1;
400 }
401
402 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
403 req.emr_in_buf = payload;
404 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
405 req.emr_out_buf = payload;
406 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
407
408 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
409 rss_context);
410
411 req_table =
412 MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
413
414 for (i = 0;
415 i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
416 i++) {
417 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0;
418 }
419
420 efx_mcdi_execute(enp, &req);
421
422 if (req.emr_rc != 0) {
423 rc = req.emr_rc;
424 goto fail2;
425 }
426
427 return (0);
428
429 fail2:
430 EFSYS_PROBE(fail2);
431 fail1:
432 EFSYS_PROBE1(fail1, efx_rc_t, rc);
433
434 return (rc);
435 }
436 #endif /* EFSYS_OPT_RX_SCALE */
437
438
439 __checkReturn efx_rc_t
ef10_rx_init(__in efx_nic_t * enp)440 ef10_rx_init(
441 __in efx_nic_t *enp)
442 {
443 #if EFSYS_OPT_RX_SCALE
444
445 if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS,
446 &enp->en_rss_context) == 0) {
447 /*
448 * Allocated an exclusive RSS context, which allows both the
449 * indirection table and key to be modified.
450 */
451 enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE;
452 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
453 } else {
454 /*
455 * Failed to allocate an exclusive RSS context. Continue
456 * operation without support for RSS. The pseudo-header in
457 * received packets will not contain a Toeplitz hash value.
458 */
459 enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE;
460 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
461 }
462
463 #endif /* EFSYS_OPT_RX_SCALE */
464
465 return (0);
466 }
467
468 #if EFSYS_OPT_RX_SCATTER
469 __checkReturn efx_rc_t
ef10_rx_scatter_enable(__in efx_nic_t * enp,__in unsigned int buf_size)470 ef10_rx_scatter_enable(
471 __in efx_nic_t *enp,
472 __in unsigned int buf_size)
473 {
474 _NOTE(ARGUNUSED(enp, buf_size))
475 return (0);
476 }
477 #endif /* EFSYS_OPT_RX_SCATTER */
478
479 #if EFSYS_OPT_RX_SCALE
480 __checkReturn efx_rc_t
ef10_rx_scale_mode_set(__in efx_nic_t * enp,__in efx_rx_hash_alg_t alg,__in efx_rx_hash_type_t type,__in boolean_t insert)481 ef10_rx_scale_mode_set(
482 __in efx_nic_t *enp,
483 __in efx_rx_hash_alg_t alg,
484 __in efx_rx_hash_type_t type,
485 __in boolean_t insert)
486 {
487 efx_rc_t rc;
488
489 EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ);
490 EFSYS_ASSERT3U(insert, ==, B_TRUE);
491
492 if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) {
493 rc = EINVAL;
494 goto fail1;
495 }
496
497 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
498 rc = ENOTSUP;
499 goto fail2;
500 }
501
502 if ((rc = efx_mcdi_rss_context_set_flags(enp,
503 enp->en_rss_context, type)) != 0)
504 goto fail3;
505
506 return (0);
507
508 fail3:
509 EFSYS_PROBE(fail3);
510 fail2:
511 EFSYS_PROBE(fail2);
512 fail1:
513 EFSYS_PROBE1(fail1, efx_rc_t, rc);
514
515 return (rc);
516 }
517 #endif /* EFSYS_OPT_RX_SCALE */
518
519 #if EFSYS_OPT_RX_SCALE
520 __checkReturn efx_rc_t
ef10_rx_scale_key_set(__in efx_nic_t * enp,__in_ecount (n)uint8_t * key,__in size_t n)521 ef10_rx_scale_key_set(
522 __in efx_nic_t *enp,
523 __in_ecount(n) uint8_t *key,
524 __in size_t n)
525 {
526 efx_rc_t rc;
527
528 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
529 rc = ENOTSUP;
530 goto fail1;
531 }
532
533 if ((rc = efx_mcdi_rss_context_set_key(enp,
534 enp->en_rss_context, key, n)) != 0)
535 goto fail2;
536
537 return (0);
538
539 fail2:
540 EFSYS_PROBE(fail2);
541 fail1:
542 EFSYS_PROBE1(fail1, efx_rc_t, rc);
543
544 return (rc);
545 }
546 #endif /* EFSYS_OPT_RX_SCALE */
547
548 #if EFSYS_OPT_RX_SCALE
549 __checkReturn efx_rc_t
ef10_rx_scale_tbl_set(__in efx_nic_t * enp,__in_ecount (n)unsigned int * table,__in size_t n)550 ef10_rx_scale_tbl_set(
551 __in efx_nic_t *enp,
552 __in_ecount(n) unsigned int *table,
553 __in size_t n)
554 {
555 efx_rc_t rc;
556
557 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
558 rc = ENOTSUP;
559 goto fail1;
560 }
561
562 if ((rc = efx_mcdi_rss_context_set_table(enp,
563 enp->en_rss_context, table, n)) != 0)
564 goto fail2;
565
566 return (0);
567
568 fail2:
569 EFSYS_PROBE(fail2);
570 fail1:
571 EFSYS_PROBE1(fail1, efx_rc_t, rc);
572
573 return (rc);
574 }
575 #endif /* EFSYS_OPT_RX_SCALE */
576
577
578 /*
579 * EF10 RX pseudo-header
580 * ---------------------
581 *
582 * Receive packets are prefixed by an (optional) 14 byte pseudo-header:
583 *
584 * +00: Toeplitz hash value.
585 * (32bit little-endian)
586 * +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag.
587 * (16bit big-endian)
588 * +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag.
589 * (16bit big-endian)
590 * +08: Packet Length. Zero if the RX datapath was in cut-through mode.
591 * (16bit little-endian)
592 * +10: MAC timestamp. Zero if timestamping is not enabled.
593 * (32bit little-endian)
594 *
595 * See "The RX Pseudo-header" in SF-109306-TC.
596 */
597
598 __checkReturn efx_rc_t
ef10_rx_prefix_pktlen(__in efx_nic_t * enp,__in uint8_t * buffer,__out uint16_t * lengthp)599 ef10_rx_prefix_pktlen(
600 __in efx_nic_t *enp,
601 __in uint8_t *buffer,
602 __out uint16_t *lengthp)
603 {
604 _NOTE(ARGUNUSED(enp))
605
606 /*
607 * The RX pseudo-header contains the packet length, excluding the
608 * pseudo-header. If the hardware receive datapath was operating in
609 * cut-through mode then the length in the RX pseudo-header will be
610 * zero, and the packet length must be obtained from the DMA length
611 * reported in the RX event.
612 */
613 *lengthp = buffer[8] | (buffer[9] << 8);
614 return (0);
615 }
616
617 #if EFSYS_OPT_RX_SCALE
618 __checkReturn uint32_t
ef10_rx_prefix_hash(__in efx_nic_t * enp,__in efx_rx_hash_alg_t func,__in uint8_t * buffer)619 ef10_rx_prefix_hash(
620 __in efx_nic_t *enp,
621 __in efx_rx_hash_alg_t func,
622 __in uint8_t *buffer)
623 {
624 _NOTE(ARGUNUSED(enp))
625
626 switch (func) {
627 case EFX_RX_HASHALG_TOEPLITZ:
628 return (buffer[0] |
629 (buffer[1] << 8) |
630 (buffer[2] << 16) |
631 (buffer[3] << 24));
632
633 default:
634 EFSYS_ASSERT(0);
635 return (0);
636 }
637 }
638 #endif /* EFSYS_OPT_RX_SCALE */
639
640 void
ef10_rx_qpost(__in efx_rxq_t * erp,__in_ecount (n)efsys_dma_addr_t * addrp,__in size_t size,__in unsigned int n,__in unsigned int completed,__in unsigned int added)641 ef10_rx_qpost(
642 __in efx_rxq_t *erp,
643 __in_ecount(n) efsys_dma_addr_t *addrp,
644 __in size_t size,
645 __in unsigned int n,
646 __in unsigned int completed,
647 __in unsigned int added)
648 {
649 efx_qword_t qword;
650 unsigned int i;
651 unsigned int offset;
652 unsigned int id;
653
654 _NOTE(ARGUNUSED(completed))
655
656 /* The client driver must not overfill the queue */
657 EFSYS_ASSERT3U(added - completed + n, <=,
658 EFX_RXQ_LIMIT(erp->er_mask + 1));
659
660 id = added & (erp->er_mask);
661 for (i = 0; i < n; i++) {
662 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
663 unsigned int, id, efsys_dma_addr_t, addrp[i],
664 size_t, size);
665
666 EFX_POPULATE_QWORD_3(qword,
667 ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
668 ESF_DZ_RX_KER_BUF_ADDR_DW0,
669 (uint32_t)(addrp[i] & 0xffffffff),
670 ESF_DZ_RX_KER_BUF_ADDR_DW1,
671 (uint32_t)(addrp[i] >> 32));
672
673 offset = id * sizeof (efx_qword_t);
674 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
675
676 id = (id + 1) & (erp->er_mask);
677 }
678 }
679
680 void
ef10_rx_qpush(__in efx_rxq_t * erp,__in unsigned int added,__inout unsigned int * pushedp)681 ef10_rx_qpush(
682 __in efx_rxq_t *erp,
683 __in unsigned int added,
684 __inout unsigned int *pushedp)
685 {
686 efx_nic_t *enp = erp->er_enp;
687 unsigned int pushed = *pushedp;
688 uint32_t wptr;
689 efx_dword_t dword;
690
691 /* Hardware has alignment restriction for WPTR */
692 wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN);
693 if (pushed == wptr)
694 return;
695
696 *pushedp = wptr;
697
698 /* Push the populated descriptors out */
699 wptr &= erp->er_mask;
700
701 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
702
703 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
704 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
705 wptr, pushed & erp->er_mask);
706 EFSYS_PIO_WRITE_BARRIER();
707 EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
708 erp->er_index, &dword, B_FALSE);
709 }
710
711 __checkReturn efx_rc_t
ef10_rx_qflush(__in efx_rxq_t * erp)712 ef10_rx_qflush(
713 __in efx_rxq_t *erp)
714 {
715 efx_nic_t *enp = erp->er_enp;
716 efx_rc_t rc;
717
718 if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
719 goto fail1;
720
721 return (0);
722
723 fail1:
724 /*
725 * EALREADY is not an error, but indicates that the MC has rebooted and
726 * that the RXQ has already been destroyed. Callers need to know that
727 * the RXQ flush has completed to avoid waiting until timeout for a
728 * flush done event that will not be delivered.
729 */
730 if (rc != EALREADY)
731 EFSYS_PROBE1(fail1, efx_rc_t, rc);
732
733 return (rc);
734 }
735
736 void
ef10_rx_qenable(__in efx_rxq_t * erp)737 ef10_rx_qenable(
738 __in efx_rxq_t *erp)
739 {
740 /* FIXME */
741 _NOTE(ARGUNUSED(erp))
742 /* FIXME */
743 }
744
745 __checkReturn efx_rc_t
ef10_rx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efx_rxq_type_t type,__in efsys_mem_t * esmp,__in size_t n,__in uint32_t id,__in efx_evq_t * eep,__in efx_rxq_t * erp)746 ef10_rx_qcreate(
747 __in efx_nic_t *enp,
748 __in unsigned int index,
749 __in unsigned int label,
750 __in efx_rxq_type_t type,
751 __in efsys_mem_t *esmp,
752 __in size_t n,
753 __in uint32_t id,
754 __in efx_evq_t *eep,
755 __in efx_rxq_t *erp)
756 {
757 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
758 efx_rc_t rc;
759 boolean_t disable_scatter;
760
761 _NOTE(ARGUNUSED(id, erp))
762
763 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
764 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
765 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
766
767 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
768 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
769
770 if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) {
771 rc = EINVAL;
772 goto fail1;
773 }
774 if (index >= encp->enc_rxq_limit) {
775 rc = EINVAL;
776 goto fail2;
777 }
778
779 /* Scatter can only be disabled if the firmware supports doing so */
780 if (type == EFX_RXQ_TYPE_SCATTER)
781 disable_scatter = B_FALSE;
782 else
783 disable_scatter = encp->enc_rx_disable_scatter_supported;
784
785 if ((rc = efx_mcdi_init_rxq(enp, n, eep->ee_index, label, index,
786 esmp, disable_scatter)) != 0)
787 goto fail3;
788
789 erp->er_eep = eep;
790 erp->er_label = label;
791
792 ef10_ev_rxlabel_init(eep, erp, label);
793
794 return (0);
795
796 fail3:
797 EFSYS_PROBE(fail3);
798 fail2:
799 EFSYS_PROBE(fail2);
800 fail1:
801 EFSYS_PROBE1(fail1, efx_rc_t, rc);
802
803 return (rc);
804 }
805
806 void
ef10_rx_qdestroy(__in efx_rxq_t * erp)807 ef10_rx_qdestroy(
808 __in efx_rxq_t *erp)
809 {
810 efx_nic_t *enp = erp->er_enp;
811 efx_evq_t *eep = erp->er_eep;
812 unsigned int label = erp->er_label;
813
814 ef10_ev_rxlabel_fini(eep, label);
815
816 EFSYS_ASSERT(enp->en_rx_qcount != 0);
817 --enp->en_rx_qcount;
818
819 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
820 }
821
822 void
ef10_rx_fini(__in efx_nic_t * enp)823 ef10_rx_fini(
824 __in efx_nic_t *enp)
825 {
826 #if EFSYS_OPT_RX_SCALE
827 if (enp->en_rss_support != EFX_RX_SCALE_UNAVAILABLE) {
828 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
829 }
830 enp->en_rss_context = 0;
831 enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE;
832 #else
833 _NOTE(ARGUNUSED(enp))
834 #endif /* EFSYS_OPT_RX_SCALE */
835 }
836
837 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
838