1 /******************************************************************************
2
3 Copyright (c) 2013-2018, Intel Corporation
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
15
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
19
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
31
32 ******************************************************************************/
33 /*$FreeBSD: stable/12/sys/dev/ixl/ixl.h 372637 2022-10-19 23:24:40Z erj $*/
34
35 #ifndef _IXL_H_
36 #define _IXL_H_
37
38 #include "opt_inet.h"
39 #include "opt_inet6.h"
40 #include "opt_rss.h"
41 #include "opt_ixl.h"
42
43 #include <sys/param.h>
44 #include <sys/systm.h>
45 #include <sys/buf_ring.h>
46 #include <sys/mbuf.h>
47 #include <sys/protosw.h>
48 #include <sys/socket.h>
49 #include <sys/malloc.h>
50 #include <sys/kernel.h>
51 #include <sys/module.h>
52 #include <sys/sockio.h>
53 #include <sys/eventhandler.h>
54 #include <sys/syslog.h>
55 #include <sys/priv.h>
56 #include <sys/bitstring.h>
57
58 #include <net/if.h>
59 #include <net/if_var.h>
60 #include <net/if_arp.h>
61 #include <net/bpf.h>
62 #include <net/ethernet.h>
63 #include <net/if_dl.h>
64 #include <net/if_media.h>
65 #include <net/iflib.h>
66
67 #include <net/bpf.h>
68 #include <net/if_types.h>
69 #include <net/if_vlan_var.h>
70
71 #include <netinet/in_systm.h>
72 #include <netinet/in.h>
73 #include <netinet/if_ether.h>
74 #include <netinet/ip.h>
75 #include <netinet/ip6.h>
76 #include <netinet/tcp.h>
77 #include <netinet/tcp_lro.h>
78 #include <netinet/udp.h>
79 #include <netinet/sctp.h>
80
81 #include <machine/in_cksum.h>
82
83 #include <sys/bus.h>
84 #include <machine/bus.h>
85 #include <sys/rman.h>
86 #include <machine/resource.h>
87 #include <vm/vm.h>
88 #include <vm/pmap.h>
89 #include <machine/clock.h>
90 #include <dev/pci/pcivar.h>
91 #include <dev/pci/pcireg.h>
92 #include <sys/proc.h>
93 #include <sys/sysctl.h>
94 #include <sys/endian.h>
95 #include <sys/taskqueue.h>
96 #include <sys/pcpu.h>
97 #include <sys/smp.h>
98 #include <sys/sbuf.h>
99 #include <machine/smp.h>
100 #include <machine/stdarg.h>
101
102 #ifdef RSS
103 #include <net/rss_config.h>
104 #include <netinet/in_rss.h>
105 #endif
106
107 #include "ifdi_if.h"
108 #include "i40e_type.h"
109 #include "i40e_prototype.h"
110 #include "ixl_debug.h"
111
112 #define PVIDV(vendor, devid, name) \
113 PVID(vendor, devid, name " - " IXL_DRIVER_VERSION_STRING)
114
115 /* Tunables */
116
117 /*
118 * Ring Descriptors Valid Range: 32-4096 Default Value: 1024 This value is the
119 * number of tx/rx descriptors allocated by the driver. Increasing this
120 * value allows the driver to queue more operations.
121 *
122 * Tx descriptors are always 16 bytes, but Rx descriptors can be 32 bytes.
123 * The driver currently always uses 32 byte Rx descriptors.
124 */
125 #define IXL_DEFAULT_RING 1024
126 #define IXL_MAX_RING 4096
127 #define IXL_MIN_RING 64
128 #define IXL_RING_INCREMENT 32
129
130 #define IXL_AQ_LEN 256
131 #define IXL_AQ_LEN_MAX 1024
132
133 /* Alignment for rings */
134 #define DBA_ALIGN 128
135
136 #define MAX_MULTICAST_ADDR 128
137
138 #define IXL_MSIX_BAR 3
139 #define IXL_ADM_LIMIT 2
140 #define IXL_TSO_SIZE ((255*1024)-1)
141 #define IXL_TX_BUF_SZ ((u32) 1514)
142 #define IXL_AQ_BUF_SZ ((u32) 4096)
143 #define IXL_RX_ITR 0
144 #define IXL_TX_ITR 1
145 #define IXL_ITR_NONE 3
146 #define IXL_QUEUE_EOL 0x7FF
147 #define IXL_MIN_FRAME 17
148 #define IXL_MAX_FRAME 9728
149 #define IXL_MAX_TX_SEGS 8
150 #define IXL_MAX_RX_SEGS 5
151 #define IXL_MAX_TSO_SEGS 128
152 #define IXL_SPARSE_CHAIN 7
153 #define IXL_MIN_TSO_MSS 64
154 #define IXL_MAX_TSO_MSS 9668
155 #define IXL_MAX_DMA_SEG_SIZE ((16 * 1024) - 1)
156
157 #define IXL_RSS_KEY_SIZE_REG 13
158 #define IXL_RSS_KEY_SIZE (IXL_RSS_KEY_SIZE_REG * 4)
159 #define IXL_RSS_VSI_LUT_SIZE 64 /* X722 -> VSI, X710 -> VF */
160 #define IXL_RSS_VSI_LUT_ENTRY_MASK 0x3F
161 #define IXL_RSS_VF_LUT_ENTRY_MASK 0xF
162
163 #define IXL_VF_MAX_BUFFER 0x3F80
164 #define IXL_VF_MAX_HDR_BUFFER 0x840
165 #define IXL_VF_MAX_FRAME 0x3FFF
166
167 /* ERJ: hardware can support ~2k (SW5+) filters between all functions */
168 #define IXL_MAX_FILTERS 256
169
170 #define IXL_NVM_VERSION_LO_SHIFT 0
171 #define IXL_NVM_VERSION_LO_MASK (0xff << IXL_NVM_VERSION_LO_SHIFT)
172 #define IXL_NVM_VERSION_HI_SHIFT 12
173 #define IXL_NVM_VERSION_HI_MASK (0xf << IXL_NVM_VERSION_HI_SHIFT)
174
175 /*
176 * Interrupt Moderation parameters
177 * Multiply ITR values by 2 for real ITR value
178 */
179 #define IXL_MAX_ITR 0x0FF0
180 #define IXL_ITR_100K 0x0005
181 #define IXL_ITR_20K 0x0019
182 #define IXL_ITR_8K 0x003E
183 #define IXL_ITR_4K 0x007A
184 #define IXL_ITR_1K 0x01F4
185 #define IXL_ITR_DYNAMIC 0x8000
186 #define IXL_LOW_LATENCY 0
187 #define IXL_AVE_LATENCY 1
188 #define IXL_BULK_LATENCY 2
189
190 /* MacVlan Flags */
191 #define IXL_FILTER_VLAN (u16)(1 << 0)
192 #define IXL_FILTER_MC (u16)(1 << 1)
193
194 /* used in the vlan field of the filter when not a vlan */
195 #define IXL_VLAN_ANY -1
196
197 /* Maximum number of MAC/VLAN filters supported by HW */
198 #define IXL_MAX_VLAN_FILTERS 256
199
200 #define CSUM_OFFLOAD_IPV4 (CSUM_IP|CSUM_TCP|CSUM_UDP|CSUM_SCTP)
201 #define CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6|CSUM_UDP_IPV6|CSUM_SCTP_IPV6)
202 #define CSUM_OFFLOAD (CSUM_OFFLOAD_IPV4|CSUM_OFFLOAD_IPV6|CSUM_TSO)
203
204 /* Misc flags for ixl_vsi.flags */
205 #define IXL_FLAGS_KEEP_TSO4 (1 << 0)
206 #define IXL_FLAGS_KEEP_TSO6 (1 << 1)
207 #define IXL_FLAGS_USES_MSIX (1 << 2)
208 #define IXL_FLAGS_IS_VF (1 << 3)
209
210 #define IXL_VSI_IS_PF(v) ((v->flags & IXL_FLAGS_IS_VF) == 0)
211 #define IXL_VSI_IS_VF(v) ((v->flags & IXL_FLAGS_IS_VF) != 0)
212
213 #define IXL_VF_RESET_TIMEOUT 100
214
215 #define IXL_VSI_DATA_PORT 0x01
216
217 #define IAVF_MAX_QUEUES 16
218 #define IXL_MAX_VSI_QUEUES (2 * (I40E_VSILAN_QTABLE_MAX_INDEX + 1))
219
220 #define IXL_RX_CTX_BASE_UNITS 128
221 #define IXL_TX_CTX_BASE_UNITS 128
222
223 #define IXL_PF_PCI_CIAA_VF_DEVICE_STATUS 0xAA
224
225 #define IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK 0x20
226
227 #define IXL_GLGEN_VFLRSTAT_INDEX(glb_vf) ((glb_vf) / 32)
228 #define IXL_GLGEN_VFLRSTAT_MASK(glb_vf) (1 << ((glb_vf) % 32))
229
230 #define IXL_MAX_ITR_IDX 3
231
232 #define IXL_END_OF_INTR_LNKLST 0x7FF
233
234 #define IXL_DEFAULT_RSS_HENA_BASE (\
235 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \
236 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | \
237 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | \
238 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | \
239 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4) | \
240 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | \
241 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | \
242 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | \
243 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | \
244 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6) | \
245 BIT_ULL(I40E_FILTER_PCTYPE_L2_PAYLOAD))
246
247 #define IXL_DEFAULT_RSS_HENA_XL710 IXL_DEFAULT_RSS_HENA_BASE
248
249 #define IXL_DEFAULT_RSS_HENA_X722 (\
250 IXL_DEFAULT_RSS_HENA_BASE | \
251 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \
252 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \
253 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \
254 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | \
255 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \
256 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK))
257
258 #define IXL_CAPS \
259 (IFCAP_TSO4 | IFCAP_TSO6 | \
260 IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 | \
261 IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | \
262 IFCAP_VLAN_HWFILTER | IFCAP_VLAN_HWTSO | \
263 IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM | \
264 IFCAP_VLAN_MTU | IFCAP_JUMBO_MTU | IFCAP_LRO)
265
266 #define IXL_CSUM_TCP \
267 (CSUM_IP_TCP|CSUM_IP_TSO|CSUM_IP6_TSO|CSUM_IP6_TCP)
268 #define IXL_CSUM_UDP \
269 (CSUM_IP_UDP|CSUM_IP6_UDP)
270 #define IXL_CSUM_SCTP \
271 (CSUM_IP_SCTP|CSUM_IP6_SCTP)
272 #define IXL_CSUM_IPV4 \
273 (CSUM_IP|CSUM_IP_TSO)
274
275 /* Pre-11 counter(9) compatibility */
276 #if __FreeBSD_version >= 1100036
277 #define IXL_SET_IPACKETS(vsi, count) (vsi)->ipackets = (count)
278 #define IXL_SET_IERRORS(vsi, count) (vsi)->ierrors = (count)
279 #define IXL_SET_OPACKETS(vsi, count) (vsi)->opackets = (count)
280 #define IXL_SET_OERRORS(vsi, count) (vsi)->oerrors = (count)
281 #define IXL_SET_COLLISIONS(vsi, count) /* Do nothing; collisions is always 0. */
282 #define IXL_SET_IBYTES(vsi, count) (vsi)->ibytes = (count)
283 #define IXL_SET_OBYTES(vsi, count) (vsi)->obytes = (count)
284 #define IXL_SET_IMCASTS(vsi, count) (vsi)->imcasts = (count)
285 #define IXL_SET_OMCASTS(vsi, count) (vsi)->omcasts = (count)
286 #define IXL_SET_IQDROPS(vsi, count) (vsi)->iqdrops = (count)
287 #define IXL_SET_OQDROPS(vsi, count) (vsi)->oqdrops = (count)
288 #define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count)
289 #else
290 #define IXL_SET_IPACKETS(vsi, count) (vsi)->ifp->if_ipackets = (count)
291 #define IXL_SET_IERRORS(vsi, count) (vsi)->ifp->if_ierrors = (count)
292 #define IXL_SET_OPACKETS(vsi, count) (vsi)->ifp->if_opackets = (count)
293 #define IXL_SET_OERRORS(vsi, count) (vsi)->ifp->if_oerrors = (count)
294 #define IXL_SET_COLLISIONS(vsi, count) (vsi)->ifp->if_collisions = (count)
295 #define IXL_SET_IBYTES(vsi, count) (vsi)->ifp->if_ibytes = (count)
296 #define IXL_SET_OBYTES(vsi, count) (vsi)->ifp->if_obytes = (count)
297 #define IXL_SET_IMCASTS(vsi, count) (vsi)->ifp->if_imcasts = (count)
298 #define IXL_SET_OMCASTS(vsi, count) (vsi)->ifp->if_omcasts = (count)
299 #define IXL_SET_IQDROPS(vsi, count) (vsi)->ifp->if_iqdrops = (count)
300 #define IXL_SET_OQDROPS(vsi, odrops) (vsi)->ifp->if_snd.ifq_drops = (odrops)
301 #define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count)
302 #endif
303
304 /* For stats sysctl naming */
305 #define IXL_QUEUE_NAME_LEN 32
306
307 MALLOC_DECLARE(M_IXL);
308
309 #define IXL_DEV_ERR(_dev, _format, ...) \
310 device_printf(_dev, "%s: " _format " (%s:%d)\n", __func__, ##__VA_ARGS__, __FILE__, __LINE__)
311
312 /*
313 *****************************************************************************
314 * vendor_info_array
315 *
316 * This array contains the list of Subvendor/Subdevice IDs on which the driver
317 * should load.
318 *
319 *****************************************************************************
320 */
321 typedef struct _ixl_vendor_info_t {
322 unsigned int vendor_id;
323 unsigned int device_id;
324 unsigned int subvendor_id;
325 unsigned int subdevice_id;
326 unsigned int index;
327 } ixl_vendor_info_t;
328
329 /*
330 ** This struct has multiple uses, multicast
331 ** addresses, vlans, and mac filters all use it.
332 */
333 struct ixl_mac_filter {
334 LIST_ENTRY(ixl_mac_filter) ftle;
335 u8 macaddr[ETHER_ADDR_LEN];
336 s16 vlan;
337 u16 flags;
338 };
339
340 /*
341 * The Transmit ring control struct
342 */
343 struct tx_ring {
344 struct ixl_tx_queue *que;
345 u32 tail;
346 struct i40e_tx_desc *tx_base;
347 u64 tx_paddr;
348 u32 latency;
349 u32 packets;
350 u32 me;
351 /*
352 * For reporting completed packet status
353 * in descriptor writeback mode
354 */
355 qidx_t *tx_rsq;
356 qidx_t tx_rs_cidx;
357 qidx_t tx_rs_pidx;
358 qidx_t tx_cidx_processed;
359
360 /* Used for Dynamic ITR calculation */
361 u32 itr;
362 u32 bytes;
363
364 /* Soft Stats */
365 u64 tx_bytes;
366 u64 tx_packets;
367 u64 mss_too_small;
368 };
369
370
371 /*
372 * The Receive ring control struct
373 */
374 struct rx_ring {
375 struct ixl_rx_queue *que;
376 union i40e_rx_desc *rx_base;
377 uint64_t rx_paddr;
378 bool discard;
379 u32 itr;
380 u32 latency;
381 u32 mbuf_sz;
382 u32 tail;
383 u32 me;
384
385 /* Used for Dynamic ITR calculation */
386 u32 packets;
387 u32 bytes;
388
389 /* Soft stats */
390 u64 rx_packets;
391 u64 rx_bytes;
392 u64 desc_errs;
393 u64 csum_errs;
394 };
395
396 /*
397 ** Driver queue structs
398 */
399 struct ixl_tx_queue {
400 struct ixl_vsi *vsi;
401 struct tx_ring txr;
402 struct if_irq que_irq;
403 u32 msix;
404 /* Stats */
405 u64 irqs;
406 u64 tso;
407 };
408
409 struct ixl_rx_queue {
410 struct ixl_vsi *vsi;
411 struct rx_ring rxr;
412 struct if_irq que_irq;
413 u32 msix; /* This queue's MSIX vector */
414 /* Stats */
415 u64 irqs;
416 };
417
418 /*
419 ** Virtual Station Interface
420 */
421 LIST_HEAD(ixl_ftl_head, ixl_mac_filter);
422 struct ixl_vsi {
423 if_ctx_t ctx;
424 if_softc_ctx_t shared;
425 struct ifnet *ifp;
426 device_t dev;
427 struct i40e_hw *hw;
428 struct ifmedia *media;
429
430 int num_rx_queues;
431 int num_tx_queues;
432
433 void *back;
434 enum i40e_vsi_type type;
435 int id;
436 u32 rx_itr_setting;
437 u32 tx_itr_setting;
438 bool enable_head_writeback;
439
440 u16 vsi_num;
441 bool link_active;
442 u16 seid;
443 u16 uplink_seid;
444 u16 downlink_seid;
445
446 struct ixl_tx_queue *tx_queues; /* TX queue array */
447 struct ixl_rx_queue *rx_queues; /* RX queue array */
448 struct if_irq irq;
449 u32 link_speed;
450
451 /* MAC/VLAN Filter list */
452 struct ixl_ftl_head ftl;
453 u16 num_macs;
454 u64 num_hw_filters;
455
456 /* Contains readylist & stat counter id */
457 struct i40e_aqc_vsi_properties_data info;
458
459 #define IXL_VLANS_MAP_LEN EVL_VLID_MASK + 1
460 bitstr_t bit_decl(vlans_map, IXL_VLANS_MAP_LEN);
461 u16 num_vlans;
462
463 /* Per-VSI stats from hardware */
464 struct i40e_eth_stats eth_stats;
465 struct i40e_eth_stats eth_stats_offsets;
466 bool stat_offsets_loaded;
467 /* VSI stat counters */
468 u64 ipackets;
469 u64 ierrors;
470 u64 opackets;
471 u64 oerrors;
472 u64 ibytes;
473 u64 obytes;
474 u64 imcasts;
475 u64 omcasts;
476 u64 iqdrops;
477 u64 oqdrops;
478 u64 noproto;
479
480 /* Misc. */
481 u64 flags;
482 /* Stats sysctls for this VSI */
483 struct sysctl_oid *vsi_node;
484 struct sysctl_ctx_list sysctl_ctx;
485 };
486
487 struct ixl_add_maddr_arg {
488 struct ixl_ftl_head to_add;
489 struct ixl_vsi *vsi;
490 };
491
492 /*
493 ** Compare two ethernet addresses
494 */
495 static inline bool
ixl_ether_is_equal(const u8 * ea1,const u8 * ea2)496 ixl_ether_is_equal(const u8 *ea1, const u8 *ea2)
497 {
498 return (bcmp(ea1, ea2, ETHER_ADDR_LEN) == 0);
499 }
500
501 /*
502 * Return next largest power of 2, unsigned
503 *
504 * Public domain, from Bit Twiddling Hacks
505 */
506 static inline u32
next_power_of_two(u32 n)507 next_power_of_two(u32 n)
508 {
509 n--;
510 n |= n >> 1;
511 n |= n >> 2;
512 n |= n >> 4;
513 n |= n >> 8;
514 n |= n >> 16;
515 n++;
516
517 /* Next power of two > 0 is 1 */
518 n += (n == 0);
519
520 return (n);
521 }
522
523 /*
524 * Info for stats sysctls
525 */
526 struct ixl_sysctl_info {
527 u64 *stat;
528 char *name;
529 char *description;
530 };
531
532 extern const uint8_t ixl_bcast_addr[ETHER_ADDR_LEN];
533
534 /* Common function prototypes between PF/VF driver */
535 void ixl_debug_core(device_t dev, u32 enabled_mask, u32 mask, char *fmt, ...);
536 void ixl_init_tx_ring(struct ixl_vsi *vsi, struct ixl_tx_queue *que);
537 void ixl_get_default_rss_key(u32 *);
538 const char * i40e_vc_stat_str(struct i40e_hw *hw,
539 enum virtchnl_status_code stat_err);
540 void ixl_init_tx_rsqs(struct ixl_vsi *vsi);
541 void ixl_init_tx_cidx(struct ixl_vsi *vsi);
542 u64 ixl_max_vc_speed_to_value(u8 link_speeds);
543 void ixl_add_vsi_sysctls(device_t dev, struct ixl_vsi *vsi,
544 struct sysctl_ctx_list *ctx, const char *sysctl_name);
545 void ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx,
546 struct sysctl_oid_list *child,
547 struct i40e_eth_stats *eth_stats);
548 void ixl_vsi_add_queues_stats(struct ixl_vsi *vsi,
549 struct sysctl_ctx_list *ctx);
550 #endif /* _IXL_H_ */
551