1 /*-
2 * Copyright 2007-2009 Solarflare Communications Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26 #include <sys/cdefs.h>
27 __FBSDID("$FreeBSD$");
28
29 #include "efsys.h"
30 #include "efx.h"
31 #include "efx_types.h"
32 #include "efx_regs.h"
33 #include "efx_impl.h"
34
35
36 #if EFSYS_OPT_FILTER
37
38 /* "Fudge factors" - difference between programmed value and actual depth.
39 * Due to pipelined implementation we need to program H/W with a value that
40 * is larger than the hop limit we want.
41 */
42 #define FILTER_CTL_SRCH_FUDGE_WILD 3
43 #define FILTER_CTL_SRCH_FUDGE_FULL 1
44
45 /* Hard maximum hop limit. Hardware will time-out beyond 200-something.
46 * We also need to avoid infinite loops in efx_filter_search() when the
47 * table is full.
48 */
49 #define FILTER_CTL_SRCH_MAX 200
50
51 /* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
52 * key derived from the n-tuple. */
53 static uint16_t
efx_filter_tbl_hash(__in uint32_t key)54 efx_filter_tbl_hash(
55 __in uint32_t key)
56 {
57 uint16_t tmp;
58
59 /* First 16 rounds */
60 tmp = 0x1fff ^ (uint16_t)(key >> 16);
61 tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
62 tmp = tmp ^ tmp >> 9;
63
64 /* Last 16 rounds */
65 tmp = tmp ^ tmp << 13 ^ (uint16_t)(key & 0xffff);
66 tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
67 tmp = tmp ^ tmp >> 9;
68
69 return (tmp);
70 }
71
72
73 /* To allow for hash collisions, filter search continues at these
74 * increments from the first possible entry selected by the hash. */
75 static uint16_t
efx_filter_tbl_increment(__in uint32_t key)76 efx_filter_tbl_increment(
77 __in uint32_t key)
78 {
79 return ((uint16_t)(key * 2 - 1));
80 }
81
82 static __checkReturn boolean_t
efx_filter_test_used(__in efx_filter_tbl_t * eftp,__in unsigned int index)83 efx_filter_test_used(
84 __in efx_filter_tbl_t *eftp,
85 __in unsigned int index)
86 {
87 EFSYS_ASSERT3P(eftp->eft_bitmap, !=, NULL);
88 return ((eftp->eft_bitmap[index / 32] & (1 << (index % 32))) != 0);
89 }
90
91 static void
efx_filter_set_used(__in efx_filter_tbl_t * eftp,__in unsigned int index)92 efx_filter_set_used(
93 __in efx_filter_tbl_t *eftp,
94 __in unsigned int index)
95 {
96 EFSYS_ASSERT3P(eftp->eft_bitmap, !=, NULL);
97 eftp->eft_bitmap[index / 32] |= (1 << (index % 32));
98 ++eftp->eft_used;
99 }
100
101 static void
efx_filter_clear_used(__in efx_filter_tbl_t * eftp,__in unsigned int index)102 efx_filter_clear_used(
103 __in efx_filter_tbl_t *eftp,
104 __in unsigned int index)
105 {
106 EFSYS_ASSERT3P(eftp->eft_bitmap, !=, NULL);
107 eftp->eft_bitmap[index / 32] &= ~(1 << (index % 32));
108
109 --eftp->eft_used;
110 EFSYS_ASSERT3U(eftp->eft_used, >=, 0);
111 }
112
113
114 static efx_filter_tbl_id_t
efx_filter_tbl_id(__in efx_filter_type_t type)115 efx_filter_tbl_id(
116 __in efx_filter_type_t type)
117 {
118 efx_filter_tbl_id_t tbl_id;
119
120 switch (type)
121 {
122 case EFX_FILTER_RX_TCP_FULL:
123 case EFX_FILTER_RX_TCP_WILD:
124 case EFX_FILTER_RX_UDP_FULL:
125 case EFX_FILTER_RX_UDP_WILD:
126 tbl_id = EFX_FILTER_TBL_RX_IP;
127 break;
128
129 #if EFSYS_OPT_SIENA
130 case EFX_FILTER_RX_MAC_FULL:
131 case EFX_FILTER_RX_MAC_WILD:
132 tbl_id = EFX_FILTER_TBL_RX_MAC;
133 break;
134
135 case EFX_FILTER_TX_TCP_FULL:
136 case EFX_FILTER_TX_TCP_WILD:
137 case EFX_FILTER_TX_UDP_FULL:
138 case EFX_FILTER_TX_UDP_WILD:
139 tbl_id = EFX_FILTER_TBL_TX_IP;
140 break;
141
142 case EFX_FILTER_TX_MAC_FULL:
143 case EFX_FILTER_TX_MAC_WILD:
144 tbl_id = EFX_FILTER_TBL_RX_MAC;
145 break;
146 #endif /* EFSYS_OPT_SIENA */
147
148 default:
149 EFSYS_ASSERT(B_FALSE);
150 break;
151 }
152 return (tbl_id);
153 }
154
155 static void
efx_filter_reset_search_depth(__inout efx_filter_t * efp,__in efx_filter_tbl_id_t tbl_id)156 efx_filter_reset_search_depth(
157 __inout efx_filter_t *efp,
158 __in efx_filter_tbl_id_t tbl_id)
159 {
160 switch (tbl_id)
161 {
162 case EFX_FILTER_TBL_RX_IP:
163 efp->ef_depth[EFX_FILTER_RX_TCP_FULL] = 0;
164 efp->ef_depth[EFX_FILTER_RX_TCP_WILD] = 0;
165 efp->ef_depth[EFX_FILTER_RX_UDP_FULL] = 0;
166 efp->ef_depth[EFX_FILTER_RX_UDP_WILD] = 0;
167 break;
168
169 #if EFSYS_OPT_SIENA
170 case EFX_FILTER_TBL_RX_MAC:
171 efp->ef_depth[EFX_FILTER_RX_MAC_FULL] = 0;
172 efp->ef_depth[EFX_FILTER_RX_MAC_WILD] = 0;
173 break;
174
175 case EFX_FILTER_TBL_TX_IP:
176 efp->ef_depth[EFX_FILTER_TX_TCP_FULL] = 0;
177 efp->ef_depth[EFX_FILTER_TX_TCP_WILD] = 0;
178 efp->ef_depth[EFX_FILTER_TX_UDP_FULL] = 0;
179 efp->ef_depth[EFX_FILTER_TX_UDP_WILD] = 0;
180 break;
181
182 case EFX_FILTER_TBL_TX_MAC:
183 efp->ef_depth[EFX_FILTER_TX_MAC_FULL] = 0;
184 efp->ef_depth[EFX_FILTER_TX_MAC_WILD] = 0;
185 break;
186 #endif /* EFSYS_OPT_SIENA */
187
188 default:
189 EFSYS_ASSERT(B_FALSE);
190 break;
191 }
192 }
193
194 static void
efx_filter_push_rx_limits(__in efx_nic_t * enp)195 efx_filter_push_rx_limits(
196 __in efx_nic_t *enp)
197 {
198 efx_filter_t *efp = &enp->en_filter;
199 efx_oword_t oword;
200
201 EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
202
203 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_FULL_SRCH_LIMIT,
204 efp->ef_depth[EFX_FILTER_RX_TCP_FULL] +
205 FILTER_CTL_SRCH_FUDGE_FULL);
206 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_WILD_SRCH_LIMIT,
207 efp->ef_depth[EFX_FILTER_RX_TCP_WILD] +
208 FILTER_CTL_SRCH_FUDGE_WILD);
209 EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_FULL_SRCH_LIMIT,
210 efp->ef_depth[EFX_FILTER_RX_UDP_FULL] +
211 FILTER_CTL_SRCH_FUDGE_FULL);
212 EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_WILD_SRCH_LIMIT,
213 efp->ef_depth[EFX_FILTER_RX_UDP_WILD] +
214 FILTER_CTL_SRCH_FUDGE_WILD);
215
216 #if EFSYS_OPT_SIENA
217 if (efp->ef_tbl[EFX_FILTER_TBL_RX_MAC].eft_size) {
218 EFX_SET_OWORD_FIELD(oword,
219 FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
220 efp->ef_depth[EFX_FILTER_RX_MAC_FULL] +
221 FILTER_CTL_SRCH_FUDGE_FULL);
222 EFX_SET_OWORD_FIELD(oword,
223 FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
224 efp->ef_depth[EFX_FILTER_RX_MAC_WILD] +
225 FILTER_CTL_SRCH_FUDGE_WILD);
226 }
227 #endif /* EFSYS_OPT_SIENA */
228
229 EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
230 }
231
232 static void
efx_filter_push_tx_limits(__in efx_nic_t * enp)233 efx_filter_push_tx_limits(
234 __in efx_nic_t *enp)
235 {
236 efx_filter_t *efp = &enp->en_filter;
237 efx_oword_t oword;
238
239 if (efp->ef_tbl[EFX_FILTER_TBL_TX_IP].eft_size == 0)
240 return;
241
242 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
243
244 EFX_SET_OWORD_FIELD(oword, FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE,
245 efp->ef_depth[EFX_FILTER_TX_TCP_FULL] +
246 FILTER_CTL_SRCH_FUDGE_FULL);
247 EFX_SET_OWORD_FIELD(oword, FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE,
248 efp->ef_depth[EFX_FILTER_TX_TCP_WILD] +
249 FILTER_CTL_SRCH_FUDGE_WILD);
250 EFX_SET_OWORD_FIELD(oword, FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE,
251 efp->ef_depth[EFX_FILTER_TX_UDP_FULL] +
252 FILTER_CTL_SRCH_FUDGE_FULL);
253 EFX_SET_OWORD_FIELD(oword, FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE,
254 efp->ef_depth[EFX_FILTER_TX_UDP_WILD] +
255 FILTER_CTL_SRCH_FUDGE_WILD);
256
257 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
258 }
259
260 /* Build a filter entry and return its n-tuple key. */
261 static __checkReturn uint32_t
efx_filter_build(__out efx_oword_t * filter,__in efx_filter_spec_t * spec)262 efx_filter_build(
263 __out efx_oword_t *filter,
264 __in efx_filter_spec_t *spec)
265 {
266 uint32_t dword3;
267 uint32_t key;
268 uint8_t type = spec->efs_type;
269 uint8_t flags = spec->efs_flags;
270
271 switch (efx_filter_tbl_id(type)) {
272 case EFX_FILTER_TBL_RX_IP: {
273 boolean_t is_udp = (type == EFX_FILTER_RX_UDP_FULL ||
274 type == EFX_FILTER_RX_UDP_WILD);
275 EFX_POPULATE_OWORD_7(*filter,
276 FRF_BZ_RSS_EN, (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
277 FRF_BZ_SCATTER_EN, (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
278 FRF_AZ_TCP_UDP, is_udp,
279 FRF_AZ_RXQ_ID, spec->efs_dmaq_id,
280 EFX_DWORD_2, spec->efs_dword[2],
281 EFX_DWORD_1, spec->efs_dword[1],
282 EFX_DWORD_0, spec->efs_dword[0]);
283 dword3 = is_udp;
284 break;
285 }
286
287 #if EFSYS_OPT_SIENA
288 case EFX_FILTER_TBL_RX_MAC: {
289 boolean_t is_wild = (type == EFX_FILTER_RX_MAC_WILD);
290 EFX_POPULATE_OWORD_8(*filter,
291 FRF_CZ_RMFT_RSS_EN, (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
292 FRF_CZ_RMFT_SCATTER_EN, (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
293 FRF_CZ_RMFT_IP_OVERRIDE, (flags & EFX_FILTER_FLAG_RX_OVERRIDE_IP) ? 1 : 0,
294 FRF_CZ_RMFT_RXQ_ID, spec->efs_dmaq_id,
295 FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
296 FRF_CZ_RMFT_DEST_MAC_DW1, spec->efs_dword[2],
297 FRF_CZ_RMFT_DEST_MAC_DW0, spec->efs_dword[1],
298 FRF_CZ_RMFT_VLAN_ID, spec->efs_dword[0]);
299 dword3 = is_wild;
300 break;
301 }
302 #endif /* EFSYS_OPT_SIENA */
303
304 case EFX_FILTER_TBL_TX_IP: {
305 boolean_t is_udp = (type == EFX_FILTER_TX_UDP_FULL ||
306 type == EFX_FILTER_TX_UDP_WILD);
307 EFX_POPULATE_OWORD_5(*filter,
308 FRF_CZ_TIFT_TCP_UDP, is_udp,
309 FRF_CZ_TIFT_TXQ_ID, spec->efs_dmaq_id,
310 EFX_DWORD_2, spec->efs_dword[2],
311 EFX_DWORD_1, spec->efs_dword[1],
312 EFX_DWORD_0, spec->efs_dword[0]);
313 dword3 = is_udp | spec->efs_dmaq_id << 1;
314 break;
315 }
316
317 #if EFSYS_OPT_SIENA
318 case EFX_FILTER_TBL_TX_MAC: {
319 boolean_t is_wild = (type == EFX_FILTER_TX_MAC_WILD);
320 EFX_POPULATE_OWORD_5(*filter,
321 FRF_CZ_TMFT_TXQ_ID, spec->efs_dmaq_id,
322 FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
323 FRF_CZ_TMFT_SRC_MAC_DW1, spec->efs_dword[2],
324 FRF_CZ_TMFT_SRC_MAC_DW0, spec->efs_dword[1],
325 FRF_CZ_TMFT_VLAN_ID, spec->efs_dword[0]);
326 dword3 = is_wild | spec->efs_dmaq_id << 1;
327 break;
328 }
329 #endif /* EFSYS_OPT_SIENA */
330
331 default:
332 EFSYS_ASSERT(B_FALSE);
333 }
334
335 key = spec->efs_dword[0] ^ spec->efs_dword[1] ^ spec->efs_dword[2] ^ dword3;
336 return (key);
337 }
338
339 static __checkReturn int
efx_filter_push_entry(__inout efx_nic_t * enp,__in efx_filter_type_t type,__in int index,__in efx_oword_t * eop)340 efx_filter_push_entry(
341 __inout efx_nic_t *enp,
342 __in efx_filter_type_t type,
343 __in int index,
344 __in efx_oword_t *eop)
345 {
346 int rc;
347
348 switch (type)
349 {
350 case EFX_FILTER_RX_TCP_FULL:
351 case EFX_FILTER_RX_TCP_WILD:
352 case EFX_FILTER_RX_UDP_FULL:
353 case EFX_FILTER_RX_UDP_WILD:
354 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_FILTER_TBL0, index, eop);
355 break;
356
357 #if EFSYS_OPT_SIENA
358 case EFX_FILTER_RX_MAC_FULL:
359 case EFX_FILTER_RX_MAC_WILD:
360 EFX_BAR_TBL_WRITEO(enp, FR_CZ_RX_MAC_FILTER_TBL0, index, eop);
361 break;
362
363 case EFX_FILTER_TX_TCP_FULL:
364 case EFX_FILTER_TX_TCP_WILD:
365 case EFX_FILTER_TX_UDP_FULL:
366 case EFX_FILTER_TX_UDP_WILD:
367 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_FILTER_TBL0, index, eop);
368 break;
369
370 case EFX_FILTER_TX_MAC_FULL:
371 case EFX_FILTER_TX_MAC_WILD:
372 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_MAC_FILTER_TBL0, index, eop);
373 break;
374 #endif /* EFSYS_OPT_SIENA */
375
376 default:
377 rc = ENOTSUP;
378 goto fail1;
379 }
380 return (0);
381
382 fail1:
383 return (rc);
384 }
385
386
387 static __checkReturn boolean_t
efx_filter_equal(__in const efx_filter_spec_t * left,__in const efx_filter_spec_t * right)388 efx_filter_equal(
389 __in const efx_filter_spec_t *left,
390 __in const efx_filter_spec_t *right)
391 {
392 efx_filter_tbl_id_t tbl_id = efx_filter_tbl_id(left->efs_type);
393
394 if (left->efs_type != right->efs_type)
395 return (B_FALSE);
396
397 if (memcmp(left->efs_dword, right->efs_dword, sizeof(left->efs_dword)))
398 return (B_FALSE);
399
400 if ((tbl_id == EFX_FILTER_TBL_TX_IP ||
401 tbl_id == EFX_FILTER_TBL_TX_MAC) &&
402 left->efs_dmaq_id != right->efs_dmaq_id)
403 return (B_FALSE);
404
405 return (B_TRUE);
406 }
407
408 static __checkReturn int
efx_filter_search(__in efx_filter_tbl_t * eftp,__in efx_filter_spec_t * spec,__in uint32_t key,__in boolean_t for_insert,__out int * filter_index,__out unsigned int * depth_required)409 efx_filter_search(
410 __in efx_filter_tbl_t *eftp,
411 __in efx_filter_spec_t *spec,
412 __in uint32_t key,
413 __in boolean_t for_insert,
414 __out int *filter_index,
415 __out unsigned int *depth_required)
416 {
417 unsigned hash, incr, filter_idx, depth;
418
419 hash = efx_filter_tbl_hash(key);
420 incr = efx_filter_tbl_increment(key);
421
422 filter_idx = hash & (eftp->eft_size - 1);
423 depth = 1;
424
425 for (;;) {
426 /* Return success if entry is used and matches this spec
427 * or entry is unused and we are trying to insert.
428 */
429 if (efx_filter_test_used(eftp, filter_idx) ?
430 efx_filter_equal(spec, &eftp->eft_spec[filter_idx]) :
431 for_insert) {
432 *filter_index = filter_idx;
433 *depth_required = depth;
434 return (0);
435 }
436
437 /* Return failure if we reached the maximum search depth */
438 if (depth == FILTER_CTL_SRCH_MAX)
439 return for_insert ? EBUSY : ENOENT;
440
441 filter_idx = (filter_idx + incr) & (eftp->eft_size - 1);
442 ++depth;
443 }
444 }
445
446 __checkReturn int
efx_filter_insert_filter(__in efx_nic_t * enp,__in efx_filter_spec_t * spec,__in boolean_t replace)447 efx_filter_insert_filter(
448 __in efx_nic_t *enp,
449 __in efx_filter_spec_t *spec,
450 __in boolean_t replace)
451 {
452 efx_filter_t *efp = &enp->en_filter;
453 efx_filter_tbl_id_t tbl_id = efx_filter_tbl_id(spec->efs_type);
454 efx_filter_tbl_t *eftp = &efp->ef_tbl[tbl_id];
455 efx_filter_spec_t *saved_spec;
456 efx_oword_t filter;
457 int filter_idx;
458 unsigned int depth;
459 int state;
460 uint32_t key;
461 int rc;
462
463 if (eftp->eft_size == 0)
464 return (EINVAL);
465
466 key = efx_filter_build(&filter, spec);
467
468 EFSYS_LOCK(enp->en_eslp, state);
469
470 rc = efx_filter_search(eftp, spec, key, B_TRUE, &filter_idx, &depth);
471 if (rc != 0)
472 goto done;
473
474 EFSYS_ASSERT3U(filter_idx, <, eftp->eft_size);
475 saved_spec = &eftp->eft_spec[filter_idx];
476
477 if (efx_filter_test_used(eftp, filter_idx)) {
478 if (replace == B_FALSE) {
479 rc = EEXIST;
480 goto done;
481 }
482 }
483 efx_filter_set_used(eftp, filter_idx);
484 *saved_spec = *spec;
485
486 if (efp->ef_depth[spec->efs_type] < depth) {
487 efp->ef_depth[spec->efs_type] = depth;
488 if (tbl_id == EFX_FILTER_TBL_TX_IP ||
489 tbl_id == EFX_FILTER_TBL_TX_MAC)
490 efx_filter_push_tx_limits(enp);
491 else
492 efx_filter_push_rx_limits(enp);
493 }
494
495 efx_filter_push_entry(enp, spec->efs_type, filter_idx, &filter);
496
497 done:
498 EFSYS_UNLOCK(enp->en_eslp, state);
499 return (rc);
500 }
501
502 static void
efx_filter_clear_entry(__in efx_nic_t * enp,__in efx_filter_tbl_t * eftp,__in int index)503 efx_filter_clear_entry(
504 __in efx_nic_t *enp,
505 __in efx_filter_tbl_t *eftp,
506 __in int index)
507 {
508 efx_oword_t filter;
509
510 if (efx_filter_test_used(eftp, index)) {
511 efx_filter_clear_used(eftp, index);
512
513 EFX_ZERO_OWORD(filter);
514 efx_filter_push_entry(enp, eftp->eft_spec[index].efs_type,
515 index, &filter);
516
517 memset(&eftp->eft_spec[index], 0, sizeof(eftp->eft_spec[0]));
518 }
519 }
520
521 __checkReturn int
efx_filter_remove_filter(__in efx_nic_t * enp,__in efx_filter_spec_t * spec)522 efx_filter_remove_filter(
523 __in efx_nic_t *enp,
524 __in efx_filter_spec_t *spec)
525 {
526 efx_filter_t *efp = &enp->en_filter;
527 efx_filter_tbl_id_t tbl_id = efx_filter_tbl_id(spec->efs_type);
528 efx_filter_tbl_t *eftp = &efp->ef_tbl[tbl_id];
529 efx_filter_spec_t *saved_spec;
530 efx_oword_t filter;
531 int filter_idx;
532 unsigned int depth;
533 int state;
534 uint32_t key;
535 int rc;
536
537 key = efx_filter_build(&filter, spec);
538
539 EFSYS_LOCK(enp->en_eslp, state);
540
541 rc = efx_filter_search(eftp, spec, key, B_FALSE, &filter_idx, &depth);
542 if (rc != 0)
543 goto out;
544
545 saved_spec = &eftp->eft_spec[filter_idx];
546
547 efx_filter_clear_entry(enp, eftp, filter_idx);
548 if (eftp->eft_used == 0)
549 efx_filter_reset_search_depth(efp, tbl_id);
550
551 rc = 0;
552
553 out:
554 EFSYS_UNLOCK(enp->en_eslp, state);
555 return (rc);
556 }
557
558 void
efx_filter_remove_index(__inout efx_nic_t * enp,__in efx_filter_type_t type,__in int index)559 efx_filter_remove_index(
560 __inout efx_nic_t *enp,
561 __in efx_filter_type_t type,
562 __in int index)
563 {
564 efx_filter_t *efp = &enp->en_filter;
565 efx_filter_tbl_id_t tbl_id = efx_filter_tbl_id(type);
566 efx_filter_tbl_t *eftp = &efp->ef_tbl[tbl_id];
567 int state;
568
569 if (index < 0)
570 return;
571
572 EFSYS_LOCK(enp->en_eslp, state);
573
574 efx_filter_clear_entry(enp, eftp, index);
575 if (eftp->eft_used == 0)
576 efx_filter_reset_search_depth(efp, tbl_id);
577
578 EFSYS_UNLOCK(enp->en_eslp, state);
579 }
580
581 void
efx_filter_tbl_clear(__inout efx_nic_t * enp,__in efx_filter_tbl_id_t tbl_id)582 efx_filter_tbl_clear(
583 __inout efx_nic_t *enp,
584 __in efx_filter_tbl_id_t tbl_id)
585 {
586 efx_filter_t *efp = &enp->en_filter;
587 efx_filter_tbl_t *eftp = &efp->ef_tbl[tbl_id];
588 int index;
589 int state;
590
591 EFSYS_LOCK(enp->en_eslp, state);
592
593 for (index = 0; index < eftp->eft_size; ++index) {
594 efx_filter_clear_entry(enp, eftp, index);
595 }
596
597 if (eftp->eft_used == 0)
598 efx_filter_reset_search_depth(efp, tbl_id);
599
600 EFSYS_UNLOCK(enp->en_eslp, state);
601 }
602
603 /* Restore filter state after a reset */
604 void
efx_filter_restore(__in efx_nic_t * enp)605 efx_filter_restore(
606 __in efx_nic_t *enp)
607 {
608 efx_filter_t *efp = &enp->en_filter;
609 efx_filter_tbl_id_t tbl_id;
610 efx_filter_tbl_t *eftp;
611 efx_filter_spec_t *spec;
612 efx_oword_t filter;
613 int filter_idx;
614 int state;
615
616 EFSYS_LOCK(enp->en_eslp, state);
617
618 for (tbl_id = 0; tbl_id < EFX_FILTER_NTBLS; tbl_id++) {
619 eftp = &efp->ef_tbl[tbl_id];
620 for (filter_idx = 0; filter_idx < eftp->eft_size; filter_idx++) {
621 if (!efx_filter_test_used(eftp, filter_idx))
622 continue;
623
624 spec = &eftp->eft_spec[filter_idx];
625 efx_filter_build(&filter, spec);
626 efx_filter_push_entry(enp, spec->efs_type,
627 filter_idx, &filter);
628 }
629 }
630
631 efx_filter_push_rx_limits(enp);
632 efx_filter_push_tx_limits(enp);
633
634 EFSYS_UNLOCK(enp->en_eslp, state);
635 }
636
637 void
efx_filter_redirect_index(__inout efx_nic_t * enp,__in efx_filter_type_t type,__in int filter_index,__in int rxq_index)638 efx_filter_redirect_index(
639 __inout efx_nic_t *enp,
640 __in efx_filter_type_t type,
641 __in int filter_index,
642 __in int rxq_index)
643 {
644 efx_filter_t *efp = &enp->en_filter;
645 efx_filter_tbl_t *eftp =
646 &efp->ef_tbl[efx_filter_tbl_id(type)];
647 efx_filter_spec_t *spec;
648 efx_oword_t filter;
649 int state;
650
651 EFSYS_LOCK(enp->en_eslp, state);
652
653 spec = &eftp->eft_spec[filter_index];
654 spec->efs_dmaq_id = (uint16_t)rxq_index;
655
656 efx_filter_build(&filter, spec);
657 efx_filter_push_entry(enp, spec->efs_type, filter_index, &filter);
658
659 EFSYS_UNLOCK(enp->en_eslp, state);
660 }
661
662 __checkReturn int
efx_filter_init(__in efx_nic_t * enp)663 efx_filter_init(
664 __in efx_nic_t *enp)
665 {
666 efx_filter_t *efp = &enp->en_filter;
667 efx_filter_tbl_t *eftp;
668 int tbl_id;
669 int rc;
670
671 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
672 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
673 EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_FILTER));
674
675 switch (enp->en_family)
676 {
677 #if EFSYS_OPT_FALCON
678 case EFX_FAMILY_FALCON:
679 eftp = &efp->ef_tbl[EFX_FILTER_TBL_RX_IP];
680 eftp->eft_size = FR_AZ_RX_FILTER_TBL0_ROWS;
681 break;
682 #endif /* EFSYS_OPT_FALCON */
683
684 #if EFSYS_OPT_SIENA
685 case EFX_FAMILY_SIENA:
686 eftp = &efp->ef_tbl[EFX_FILTER_TBL_RX_IP];
687 eftp->eft_size = FR_AZ_RX_FILTER_TBL0_ROWS;
688
689 eftp = &efp->ef_tbl[EFX_FILTER_TBL_RX_MAC];
690 eftp->eft_size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
691
692 eftp = &efp->ef_tbl[EFX_FILTER_TBL_TX_IP];
693 eftp->eft_size = FR_CZ_TX_FILTER_TBL0_ROWS;
694
695 eftp = &efp->ef_tbl[EFX_FILTER_TBL_TX_MAC];
696 eftp->eft_size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
697 break;
698 #endif /* EFSYS_OPT_SIENA */
699
700 default:
701 rc = ENOTSUP;
702 goto fail1;
703 }
704
705 for (tbl_id = 0; tbl_id < EFX_FILTER_NTBLS; tbl_id++) {
706 unsigned int bitmap_size;
707
708 eftp = &efp->ef_tbl[tbl_id];
709 if (eftp->eft_size == 0)
710 continue;
711
712 EFX_STATIC_ASSERT(sizeof(eftp->eft_bitmap[0]) == sizeof(uint32_t));
713 bitmap_size = (eftp->eft_size + (sizeof(uint32_t) * 8) - 1) / 8;
714
715 EFSYS_KMEM_ALLOC(enp->en_esip, bitmap_size, eftp->eft_bitmap);
716 if (!eftp->eft_bitmap) {
717 rc = ENOMEM;
718 goto fail2;
719 }
720
721 EFSYS_KMEM_ALLOC(enp->en_esip, eftp->eft_size * sizeof(*eftp->eft_spec),
722 eftp->eft_spec);
723 if (!eftp->eft_spec) {
724 rc = ENOMEM;
725 goto fail3;
726 }
727 memset(eftp->eft_spec, 0, eftp->eft_size * sizeof(*eftp->eft_spec));
728 }
729 enp->en_mod_flags |= EFX_MOD_FILTER;
730
731 return (0);
732
733 fail3:
734 EFSYS_PROBE(fail3);
735
736 fail2:
737 EFSYS_PROBE(fail2);
738 efx_filter_fini(enp);
739
740 fail1:
741 EFSYS_PROBE1(fail1, int, rc);
742 return (rc);
743 }
744
745 void
efx_filter_fini(__in efx_nic_t * enp)746 efx_filter_fini(
747 __in efx_nic_t *enp)
748 {
749 efx_filter_t *efp = &enp->en_filter;
750 efx_filter_tbl_id_t tbl_id;
751
752 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
753 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
754
755 for (tbl_id = 0; tbl_id < EFX_FILTER_NTBLS; tbl_id++) {
756 efx_filter_tbl_t *eftp = &efp->ef_tbl[tbl_id];
757 unsigned int bitmap_size;
758
759 EFX_STATIC_ASSERT(sizeof(eftp->eft_bitmap[0]) == sizeof(uint32_t));
760 bitmap_size = (eftp->eft_size + (sizeof(uint32_t) * 8) - 1) / 8;
761
762 if (eftp->eft_bitmap != NULL) {
763 EFSYS_KMEM_FREE(enp->en_esip, bitmap_size,
764 eftp->eft_bitmap);
765 eftp->eft_bitmap = NULL;
766 }
767
768 if (eftp->eft_spec != NULL) {
769 EFSYS_KMEM_FREE(enp->en_esip, eftp->eft_size *
770 sizeof(*eftp->eft_spec), eftp->eft_spec);
771 eftp->eft_spec = NULL;
772 }
773 }
774
775 enp->en_mod_flags &= ~EFX_MOD_FILTER;
776 }
777
778 extern void
efx_filter_spec_rx_ipv4_tcp_full(__inout efx_filter_spec_t * spec,__in unsigned int flags,__in uint32_t src_ip,__in uint16_t src_tcp,__in uint32_t dest_ip,__in uint16_t dest_tcp)779 efx_filter_spec_rx_ipv4_tcp_full(
780 __inout efx_filter_spec_t *spec,
781 __in unsigned int flags,
782 __in uint32_t src_ip,
783 __in uint16_t src_tcp,
784 __in uint32_t dest_ip,
785 __in uint16_t dest_tcp)
786 {
787 EFSYS_ASSERT3P(spec, !=, NULL);
788 EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
789 EFX_FILTER_FLAG_RX_SCATTER)) == 0);
790
791 spec->efs_type = EFX_FILTER_RX_TCP_FULL;
792 spec->efs_flags = (uint8_t)flags;
793 spec->efs_dword[0] = src_tcp | src_ip << 16;
794 spec->efs_dword[1] = dest_tcp << 16 | src_ip >> 16;
795 spec->efs_dword[2] = dest_ip;
796 }
797
798 extern void
efx_filter_spec_rx_ipv4_tcp_wild(__inout efx_filter_spec_t * spec,__in unsigned int flags,__in uint32_t dest_ip,__in uint16_t dest_tcp)799 efx_filter_spec_rx_ipv4_tcp_wild(
800 __inout efx_filter_spec_t *spec,
801 __in unsigned int flags,
802 __in uint32_t dest_ip,
803 __in uint16_t dest_tcp)
804 {
805 EFSYS_ASSERT3P(spec, !=, NULL);
806 EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
807 EFX_FILTER_FLAG_RX_SCATTER)) == 0);
808
809 spec->efs_type = EFX_FILTER_RX_TCP_WILD;
810 spec->efs_flags = (uint8_t)flags;
811 spec->efs_dword[0] = 0;
812 spec->efs_dword[1] = dest_tcp << 16;
813 spec->efs_dword[2] = dest_ip;
814 }
815
816 extern void
efx_filter_spec_rx_ipv4_udp_full(__inout efx_filter_spec_t * spec,__in unsigned int flags,__in uint32_t src_ip,__in uint16_t src_udp,__in uint32_t dest_ip,__in uint16_t dest_udp)817 efx_filter_spec_rx_ipv4_udp_full(
818 __inout efx_filter_spec_t *spec,
819 __in unsigned int flags,
820 __in uint32_t src_ip,
821 __in uint16_t src_udp,
822 __in uint32_t dest_ip,
823 __in uint16_t dest_udp)
824 {
825 EFSYS_ASSERT3P(spec, !=, NULL);
826 EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
827 EFX_FILTER_FLAG_RX_SCATTER)) == 0);
828
829 spec->efs_type = EFX_FILTER_RX_UDP_FULL;
830 spec->efs_flags = (uint8_t)flags;
831 spec->efs_dword[0] = src_udp | src_ip << 16;
832 spec->efs_dword[1] = dest_udp << 16 | src_ip >> 16;
833 spec->efs_dword[2] = dest_ip;
834 }
835
836 extern void
efx_filter_spec_rx_ipv4_udp_wild(__inout efx_filter_spec_t * spec,__in unsigned int flags,__in uint32_t dest_ip,__in uint16_t dest_udp)837 efx_filter_spec_rx_ipv4_udp_wild(
838 __inout efx_filter_spec_t *spec,
839 __in unsigned int flags,
840 __in uint32_t dest_ip,
841 __in uint16_t dest_udp)
842 {
843 EFSYS_ASSERT3P(spec, !=, NULL);
844 EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
845 EFX_FILTER_FLAG_RX_SCATTER)) == 0);
846
847 spec->efs_type = EFX_FILTER_RX_UDP_WILD;
848 spec->efs_flags = (uint8_t)flags;
849 spec->efs_dword[0] = dest_udp;
850 spec->efs_dword[1] = 0;
851 spec->efs_dword[2] = dest_ip;
852 }
853
854 #if EFSYS_OPT_SIENA
855 extern void
efx_filter_spec_rx_mac_full(__inout efx_filter_spec_t * spec,__in unsigned int flags,__in uint16_t vlan_id,__in uint8_t * dest_mac)856 efx_filter_spec_rx_mac_full(
857 __inout efx_filter_spec_t *spec,
858 __in unsigned int flags,
859 __in uint16_t vlan_id,
860 __in uint8_t *dest_mac)
861 {
862 EFSYS_ASSERT3P(spec, !=, NULL);
863 EFSYS_ASSERT3P(dest_mac, !=, NULL);
864 EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
865 EFX_FILTER_FLAG_RX_SCATTER |
866 EFX_FILTER_FLAG_RX_OVERRIDE_IP)) == 0);
867
868 spec->efs_type = EFX_FILTER_RX_MAC_FULL;
869 spec->efs_flags = (uint8_t)flags;
870 spec->efs_dword[0] = vlan_id;
871 spec->efs_dword[1] =
872 dest_mac[2] << 24 |
873 dest_mac[3] << 16 |
874 dest_mac[4] << 8 |
875 dest_mac[5];
876 spec->efs_dword[2] =
877 dest_mac[0] << 8 |
878 dest_mac[1];
879 }
880 #endif /* EFSYS_OPT_SIENA */
881
882 #if EFSYS_OPT_SIENA
883 extern void
efx_filter_spec_rx_mac_wild(__inout efx_filter_spec_t * spec,__in unsigned int flags,__in uint8_t * dest_mac)884 efx_filter_spec_rx_mac_wild(
885 __inout efx_filter_spec_t *spec,
886 __in unsigned int flags,
887 __in uint8_t *dest_mac)
888 {
889 EFSYS_ASSERT3P(spec, !=, NULL);
890 EFSYS_ASSERT3P(dest_mac, !=, NULL);
891 EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
892 EFX_FILTER_FLAG_RX_SCATTER |
893 EFX_FILTER_FLAG_RX_OVERRIDE_IP)) == 0);
894
895 spec->efs_type = EFX_FILTER_RX_MAC_WILD;
896 spec->efs_flags = (uint8_t)flags;
897 spec->efs_dword[0] = 0;
898 spec->efs_dword[1] =
899 dest_mac[2] << 24 |
900 dest_mac[3] << 16 |
901 dest_mac[4] << 8 |
902 dest_mac[5];
903 spec->efs_dword[2] =
904 dest_mac[0] << 8 |
905 dest_mac[1];
906 }
907 #endif /* EFSYS_OPT_SIENA */
908
909 #if EFSYS_OPT_SIENA
910 extern void
efx_filter_spec_tx_ipv4_tcp_full(__inout efx_filter_spec_t * spec,__in uint32_t src_ip,__in uint16_t src_tcp,__in uint32_t dest_ip,__in uint16_t dest_tcp)911 efx_filter_spec_tx_ipv4_tcp_full(
912 __inout efx_filter_spec_t *spec,
913 __in uint32_t src_ip,
914 __in uint16_t src_tcp,
915 __in uint32_t dest_ip,
916 __in uint16_t dest_tcp)
917 {
918 EFSYS_ASSERT3P(spec, !=, NULL);
919
920 spec->efs_type = EFX_FILTER_TX_TCP_FULL;
921 spec->efs_flags = 0;
922 spec->efs_dword[0] = src_tcp | src_ip << 16;
923 spec->efs_dword[1] = dest_tcp << 16 | src_ip >> 16;
924 spec->efs_dword[2] = dest_ip;
925 }
926 #endif /* EFSYS_OPT_SIENA */
927
928 #if EFSYS_OPT_SIENA
929 extern void
efx_filter_spec_tx_ipv4_tcp_wild(__inout efx_filter_spec_t * spec,__in uint32_t src_ip,__in uint16_t src_tcp)930 efx_filter_spec_tx_ipv4_tcp_wild(
931 __inout efx_filter_spec_t *spec,
932 __in uint32_t src_ip,
933 __in uint16_t src_tcp)
934 {
935 EFSYS_ASSERT3P(spec, !=, NULL);
936
937 spec->efs_type = EFX_FILTER_TX_TCP_WILD;
938 spec->efs_flags = 0;
939 spec->efs_dword[0] = 0;
940 spec->efs_dword[1] = src_tcp << 16;
941 spec->efs_dword[2] = src_ip;
942 }
943 #endif /* EFSYS_OPT_SIENA */
944
945 #if EFSYS_OPT_SIENA
946 extern void
efx_filter_spec_tx_ipv4_udp_full(__inout efx_filter_spec_t * spec,__in uint32_t src_ip,__in uint16_t src_udp,__in uint32_t dest_ip,__in uint16_t dest_udp)947 efx_filter_spec_tx_ipv4_udp_full(
948 __inout efx_filter_spec_t *spec,
949 __in uint32_t src_ip,
950 __in uint16_t src_udp,
951 __in uint32_t dest_ip,
952 __in uint16_t dest_udp)
953 {
954 EFSYS_ASSERT3P(spec, !=, NULL);
955
956 spec->efs_type = EFX_FILTER_TX_UDP_FULL;
957 spec->efs_flags = 0;
958 spec->efs_dword[0] = src_udp | src_ip << 16;
959 spec->efs_dword[1] = dest_udp << 16 | src_ip >> 16;
960 spec->efs_dword[2] = dest_ip;
961 }
962 #endif /* EFSYS_OPT_SIENA */
963
964 #if EFSYS_OPT_SIENA
965 extern void
efx_filter_spec_tx_ipv4_udp_wild(__inout efx_filter_spec_t * spec,__in uint32_t src_ip,__in uint16_t src_udp)966 efx_filter_spec_tx_ipv4_udp_wild(
967 __inout efx_filter_spec_t *spec,
968 __in uint32_t src_ip,
969 __in uint16_t src_udp)
970 {
971 EFSYS_ASSERT3P(spec, !=, NULL);
972
973 spec->efs_type = EFX_FILTER_TX_UDP_WILD;
974 spec->efs_flags = 0;
975 spec->efs_dword[0] = src_udp;
976 spec->efs_dword[1] = 0;
977 spec->efs_dword[2] = src_ip;
978 }
979 #endif /* EFSYS_OPT_SIENA */
980
981 #if EFSYS_OPT_SIENA
982 extern void
efx_filter_spec_tx_mac_full(__inout efx_filter_spec_t * spec,__in uint16_t vlan_id,__in uint8_t * src_mac)983 efx_filter_spec_tx_mac_full(
984 __inout efx_filter_spec_t *spec,
985 __in uint16_t vlan_id,
986 __in uint8_t *src_mac)
987 {
988 EFSYS_ASSERT3P(spec, !=, NULL);
989 EFSYS_ASSERT3P(src_mac, !=, NULL);
990
991 spec->efs_type = EFX_FILTER_TX_MAC_FULL;
992 spec->efs_flags = 0;
993 spec->efs_dword[0] = vlan_id;
994 spec->efs_dword[1] =
995 src_mac[2] << 24 |
996 src_mac[3] << 16 |
997 src_mac[4] << 8 |
998 src_mac[5];
999 spec->efs_dword[2] =
1000 src_mac[0] << 8 |
1001 src_mac[1];
1002 }
1003 #endif /* EFSYS_OPT_SIENA */
1004
1005 #if EFSYS_OPT_SIENA
1006 extern void
efx_filter_spec_tx_mac_wild(__inout efx_filter_spec_t * spec,__in uint8_t * src_mac)1007 efx_filter_spec_tx_mac_wild(
1008 __inout efx_filter_spec_t *spec,
1009 __in uint8_t *src_mac)
1010 {
1011 EFSYS_ASSERT3P(spec, !=, NULL);
1012 EFSYS_ASSERT3P(src_mac, !=, NULL);
1013
1014 spec->efs_type = EFX_FILTER_TX_MAC_WILD;
1015 spec->efs_flags = 0;
1016 spec->efs_dword[0] = 0;
1017 spec->efs_dword[1] =
1018 src_mac[2] << 24 |
1019 src_mac[3] << 16 |
1020 src_mac[4] << 8 |
1021 src_mac[5];
1022 spec->efs_dword[2] =
1023 src_mac[0] << 8 |
1024 src_mac[1];
1025 }
1026 #endif /* EFSYS_OPT_SIENA */
1027
1028
1029 #endif /* EFSYS_OPT_FILTER */
1030