1 /******************************************************************************
2
3 Copyright (c) 2001-2015, Intel Corporation
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
15
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
19
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
31
32 ******************************************************************************/
33 /*$FreeBSD$*/
34
35 /*
36 * Uncomment the following extensions for better performance in a VM,
37 * especially if you have support in the hypervisor.
38 * See http://info.iet.unipi.it/~luigi/netmap/
39 */
40 // #define BATCH_DISPATCH
41 // #define NIC_SEND_COMBINING
42 // #define NIC_PARAVIRT /* enable virtio-like synchronization */
43
44 #include "opt_inet.h"
45 #include "opt_inet6.h"
46
47 #ifdef HAVE_KERNEL_OPTION_HEADERS
48 #include "opt_device_polling.h"
49 #endif
50
51 #include <sys/param.h>
52 #include <sys/systm.h>
53 #include <sys/buf_ring.h>
54 #include <sys/bus.h>
55 #include <sys/endian.h>
56 #include <sys/kernel.h>
57 #include <sys/kthread.h>
58 #include <sys/malloc.h>
59 #include <sys/mbuf.h>
60 #include <sys/module.h>
61 #include <sys/rman.h>
62 #include <sys/socket.h>
63 #include <sys/sockio.h>
64 #include <sys/sysctl.h>
65 #include <sys/taskqueue.h>
66 #include <sys/eventhandler.h>
67 #include <machine/bus.h>
68 #include <machine/resource.h>
69
70 #include <net/bpf.h>
71 #include <net/ethernet.h>
72 #include <net/if.h>
73 #include <net/if_var.h>
74 #include <net/if_arp.h>
75 #include <net/if_dl.h>
76 #include <net/if_media.h>
77
78 #include <net/if_types.h>
79 #include <net/if_vlan_var.h>
80
81 #include <netinet/in_systm.h>
82 #include <netinet/in.h>
83 #include <netinet/if_ether.h>
84 #include <netinet/ip.h>
85 #include <netinet/ip6.h>
86 #include <netinet/tcp.h>
87 #include <netinet/udp.h>
88
89 #include <machine/in_cksum.h>
90 #include <dev/led/led.h>
91 #include <dev/pci/pcivar.h>
92 #include <dev/pci/pcireg.h>
93
94 #include "e1000_api.h"
95 #include "if_lem.h"
96
97 /*********************************************************************
98 * Legacy Em Driver version:
99 *********************************************************************/
100 char lem_driver_version[] = "1.1.0";
101
102 /*********************************************************************
103 * PCI Device ID Table
104 *
105 * Used by probe to select devices to load on
106 * Last field stores an index into e1000_strings
107 * Last entry must be all 0s
108 *
109 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
110 *********************************************************************/
111
112 static em_vendor_info_t lem_vendor_info_array[] =
113 {
114 /* Intel(R) PRO/1000 Network Connection */
115 { 0x8086, E1000_DEV_ID_82540EM, PCI_ANY_ID, PCI_ANY_ID, 0},
116 { 0x8086, E1000_DEV_ID_82540EM_LOM, PCI_ANY_ID, PCI_ANY_ID, 0},
117 { 0x8086, E1000_DEV_ID_82540EP, PCI_ANY_ID, PCI_ANY_ID, 0},
118 { 0x8086, E1000_DEV_ID_82540EP_LOM, PCI_ANY_ID, PCI_ANY_ID, 0},
119 { 0x8086, E1000_DEV_ID_82540EP_LP, PCI_ANY_ID, PCI_ANY_ID, 0},
120
121 { 0x8086, E1000_DEV_ID_82541EI, PCI_ANY_ID, PCI_ANY_ID, 0},
122 { 0x8086, E1000_DEV_ID_82541ER, PCI_ANY_ID, PCI_ANY_ID, 0},
123 { 0x8086, E1000_DEV_ID_82541ER_LOM, PCI_ANY_ID, PCI_ANY_ID, 0},
124 { 0x8086, E1000_DEV_ID_82541EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0},
125 { 0x8086, E1000_DEV_ID_82541GI, PCI_ANY_ID, PCI_ANY_ID, 0},
126 { 0x8086, E1000_DEV_ID_82541GI_LF, PCI_ANY_ID, PCI_ANY_ID, 0},
127 { 0x8086, E1000_DEV_ID_82541GI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0},
128
129 { 0x8086, E1000_DEV_ID_82542, PCI_ANY_ID, PCI_ANY_ID, 0},
130
131 { 0x8086, E1000_DEV_ID_82543GC_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
132 { 0x8086, E1000_DEV_ID_82543GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
133
134 { 0x8086, E1000_DEV_ID_82544EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
135 { 0x8086, E1000_DEV_ID_82544EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
136 { 0x8086, E1000_DEV_ID_82544GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
137 { 0x8086, E1000_DEV_ID_82544GC_LOM, PCI_ANY_ID, PCI_ANY_ID, 0},
138
139 { 0x8086, E1000_DEV_ID_82545EM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
140 { 0x8086, E1000_DEV_ID_82545EM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
141 { 0x8086, E1000_DEV_ID_82545GM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
142 { 0x8086, E1000_DEV_ID_82545GM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
143 { 0x8086, E1000_DEV_ID_82545GM_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
144
145 { 0x8086, E1000_DEV_ID_82546EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
146 { 0x8086, E1000_DEV_ID_82546EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
147 { 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
148 { 0x8086, E1000_DEV_ID_82546GB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
149 { 0x8086, E1000_DEV_ID_82546GB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
150 { 0x8086, E1000_DEV_ID_82546GB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
151 { 0x8086, E1000_DEV_ID_82546GB_PCIE, PCI_ANY_ID, PCI_ANY_ID, 0},
152 { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
153 { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3,
154 PCI_ANY_ID, PCI_ANY_ID, 0},
155
156 { 0x8086, E1000_DEV_ID_82547EI, PCI_ANY_ID, PCI_ANY_ID, 0},
157 { 0x8086, E1000_DEV_ID_82547EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0},
158 { 0x8086, E1000_DEV_ID_82547GI, PCI_ANY_ID, PCI_ANY_ID, 0},
159 /* required last entry */
160 { 0, 0, 0, 0, 0}
161 };
162
163 /*********************************************************************
164 * Table of branding strings for all supported NICs.
165 *********************************************************************/
166
167 static char *lem_strings[] = {
168 "Intel(R) PRO/1000 Legacy Network Connection"
169 };
170
171 /*********************************************************************
172 * Function prototypes
173 *********************************************************************/
174 static int lem_probe(device_t);
175 static int lem_attach(device_t);
176 static int lem_detach(device_t);
177 static int lem_shutdown(device_t);
178 static int lem_suspend(device_t);
179 static int lem_resume(device_t);
180 static void lem_start(if_t);
181 static void lem_start_locked(if_t ifp);
182 static int lem_ioctl(if_t, u_long, caddr_t);
183 static uint64_t lem_get_counter(if_t, ift_counter);
184 static void lem_init(void *);
185 static void lem_init_locked(struct adapter *);
186 static void lem_stop(void *);
187 static void lem_media_status(if_t, struct ifmediareq *);
188 static int lem_media_change(if_t);
189 static void lem_identify_hardware(struct adapter *);
190 static int lem_allocate_pci_resources(struct adapter *);
191 static int lem_allocate_irq(struct adapter *adapter);
192 static void lem_free_pci_resources(struct adapter *);
193 static void lem_local_timer(void *);
194 static int lem_hardware_init(struct adapter *);
195 static int lem_setup_interface(device_t, struct adapter *);
196 static void lem_setup_transmit_structures(struct adapter *);
197 static void lem_initialize_transmit_unit(struct adapter *);
198 static int lem_setup_receive_structures(struct adapter *);
199 static void lem_initialize_receive_unit(struct adapter *);
200 static void lem_enable_intr(struct adapter *);
201 static void lem_disable_intr(struct adapter *);
202 static void lem_free_transmit_structures(struct adapter *);
203 static void lem_free_receive_structures(struct adapter *);
204 static void lem_update_stats_counters(struct adapter *);
205 static void lem_add_hw_stats(struct adapter *adapter);
206 static void lem_txeof(struct adapter *);
207 static void lem_tx_purge(struct adapter *);
208 static int lem_allocate_receive_structures(struct adapter *);
209 static int lem_allocate_transmit_structures(struct adapter *);
210 static bool lem_rxeof(struct adapter *, int, int *);
211 #ifndef __NO_STRICT_ALIGNMENT
212 static int lem_fixup_rx(struct adapter *);
213 #endif
214 static void lem_receive_checksum(struct adapter *, struct e1000_rx_desc *,
215 struct mbuf *);
216 static void lem_transmit_checksum_setup(struct adapter *, struct mbuf *,
217 u32 *, u32 *);
218 static void lem_set_promisc(struct adapter *);
219 static void lem_disable_promisc(struct adapter *);
220 static void lem_set_multi(struct adapter *);
221 static void lem_update_link_status(struct adapter *);
222 static int lem_get_buf(struct adapter *, int);
223 static void lem_register_vlan(void *, if_t, u16);
224 static void lem_unregister_vlan(void *, if_t, u16);
225 static void lem_setup_vlan_hw_support(struct adapter *);
226 static int lem_xmit(struct adapter *, struct mbuf **);
227 static void lem_smartspeed(struct adapter *);
228 static int lem_82547_fifo_workaround(struct adapter *, int);
229 static void lem_82547_update_fifo_head(struct adapter *, int);
230 static int lem_82547_tx_fifo_reset(struct adapter *);
231 static void lem_82547_move_tail(void *);
232 static int lem_dma_malloc(struct adapter *, bus_size_t,
233 struct em_dma_alloc *, int);
234 static void lem_dma_free(struct adapter *, struct em_dma_alloc *);
235 static int lem_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
236 static void lem_print_nvm_info(struct adapter *);
237 static int lem_is_valid_ether_addr(u8 *);
238 static u32 lem_fill_descriptors (bus_addr_t address, u32 length,
239 PDESC_ARRAY desc_array);
240 static int lem_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
241 static void lem_add_int_delay_sysctl(struct adapter *, const char *,
242 const char *, struct em_int_delay_info *, int, int);
243 static void lem_set_flow_cntrl(struct adapter *, const char *,
244 const char *, int *, int);
245 /* Management and WOL Support */
246 static void lem_init_manageability(struct adapter *);
247 static void lem_release_manageability(struct adapter *);
248 static void lem_get_hw_control(struct adapter *);
249 static void lem_release_hw_control(struct adapter *);
250 static void lem_get_wakeup(device_t);
251 static void lem_enable_wakeup(device_t);
252 static int lem_enable_phy_wakeup(struct adapter *);
253 static void lem_led_func(void *, int);
254
255 static void lem_intr(void *);
256 static int lem_irq_fast(void *);
257 static void lem_handle_rxtx(void *context, int pending);
258 static void lem_handle_link(void *context, int pending);
259 static void lem_add_rx_process_limit(struct adapter *, const char *,
260 const char *, int *, int);
261
262 #ifdef DEVICE_POLLING
263 static poll_handler_t lem_poll;
264 #endif /* POLLING */
265
266 /*********************************************************************
267 * FreeBSD Device Interface Entry Points
268 *********************************************************************/
269
270 static device_method_t lem_methods[] = {
271 /* Device interface */
272 DEVMETHOD(device_probe, lem_probe),
273 DEVMETHOD(device_attach, lem_attach),
274 DEVMETHOD(device_detach, lem_detach),
275 DEVMETHOD(device_shutdown, lem_shutdown),
276 DEVMETHOD(device_suspend, lem_suspend),
277 DEVMETHOD(device_resume, lem_resume),
278 DEVMETHOD_END
279 };
280
281 static driver_t lem_driver = {
282 "em", lem_methods, sizeof(struct adapter),
283 };
284
285 extern devclass_t em_devclass;
286 DRIVER_MODULE(lem, pci, lem_driver, em_devclass, 0, 0);
287 MODULE_DEPEND(lem, pci, 1, 1, 1);
288 MODULE_DEPEND(lem, ether, 1, 1, 1);
289 #ifdef DEV_NETMAP
290 MODULE_DEPEND(lem, netmap, 1, 1, 1);
291 #endif /* DEV_NETMAP */
292
293 /*********************************************************************
294 * Tunable default values.
295 *********************************************************************/
296
297 #define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000)
298 #define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024)
299
300 #define MAX_INTS_PER_SEC 8000
301 #define DEFAULT_ITR (1000000000/(MAX_INTS_PER_SEC * 256))
302
303 static int lem_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV);
304 static int lem_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR);
305 static int lem_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV);
306 static int lem_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV);
307 /*
308 * increase lem_rxd and lem_txd to at least 2048 in netmap mode
309 * for better performance.
310 */
311 static int lem_rxd = EM_DEFAULT_RXD;
312 static int lem_txd = EM_DEFAULT_TXD;
313 static int lem_smart_pwr_down = FALSE;
314
315 /* Controls whether promiscuous also shows bad packets */
316 static int lem_debug_sbp = FALSE;
317
318 TUNABLE_INT("hw.em.tx_int_delay", &lem_tx_int_delay_dflt);
319 TUNABLE_INT("hw.em.rx_int_delay", &lem_rx_int_delay_dflt);
320 TUNABLE_INT("hw.em.tx_abs_int_delay", &lem_tx_abs_int_delay_dflt);
321 TUNABLE_INT("hw.em.rx_abs_int_delay", &lem_rx_abs_int_delay_dflt);
322 TUNABLE_INT("hw.em.rxd", &lem_rxd);
323 TUNABLE_INT("hw.em.txd", &lem_txd);
324 TUNABLE_INT("hw.em.smart_pwr_down", &lem_smart_pwr_down);
325 TUNABLE_INT("hw.em.sbp", &lem_debug_sbp);
326
327 /* Interrupt style - default to fast */
328 static int lem_use_legacy_irq = 0;
329 TUNABLE_INT("hw.em.use_legacy_irq", &lem_use_legacy_irq);
330
331 /* How many packets rxeof tries to clean at a time */
332 static int lem_rx_process_limit = 100;
333 TUNABLE_INT("hw.em.rx_process_limit", &lem_rx_process_limit);
334
335 /* Flow control setting - default to FULL */
336 static int lem_fc_setting = e1000_fc_full;
337 TUNABLE_INT("hw.em.fc_setting", &lem_fc_setting);
338
339 /* Global used in WOL setup with multiport cards */
340 static int global_quad_port_a = 0;
341
342 #ifdef DEV_NETMAP /* see ixgbe.c for details */
343 #include <dev/netmap/if_lem_netmap.h>
344 #endif /* DEV_NETMAP */
345
346 /*********************************************************************
347 * Device identification routine
348 *
349 * em_probe determines if the driver should be loaded on
350 * adapter based on PCI vendor/device id of the adapter.
351 *
352 * return BUS_PROBE_DEFAULT on success, positive on failure
353 *********************************************************************/
354
355 static int
lem_probe(device_t dev)356 lem_probe(device_t dev)
357 {
358 char adapter_name[60];
359 u16 pci_vendor_id = 0;
360 u16 pci_device_id = 0;
361 u16 pci_subvendor_id = 0;
362 u16 pci_subdevice_id = 0;
363 em_vendor_info_t *ent;
364
365 INIT_DEBUGOUT("em_probe: begin");
366
367 pci_vendor_id = pci_get_vendor(dev);
368 if (pci_vendor_id != EM_VENDOR_ID)
369 return (ENXIO);
370
371 pci_device_id = pci_get_device(dev);
372 pci_subvendor_id = pci_get_subvendor(dev);
373 pci_subdevice_id = pci_get_subdevice(dev);
374
375 ent = lem_vendor_info_array;
376 while (ent->vendor_id != 0) {
377 if ((pci_vendor_id == ent->vendor_id) &&
378 (pci_device_id == ent->device_id) &&
379
380 ((pci_subvendor_id == ent->subvendor_id) ||
381 (ent->subvendor_id == PCI_ANY_ID)) &&
382
383 ((pci_subdevice_id == ent->subdevice_id) ||
384 (ent->subdevice_id == PCI_ANY_ID))) {
385 sprintf(adapter_name, "%s %s",
386 lem_strings[ent->index],
387 lem_driver_version);
388 device_set_desc_copy(dev, adapter_name);
389 return (BUS_PROBE_DEFAULT);
390 }
391 ent++;
392 }
393
394 return (ENXIO);
395 }
396
397 /*********************************************************************
398 * Device initialization routine
399 *
400 * The attach entry point is called when the driver is being loaded.
401 * This routine identifies the type of hardware, allocates all resources
402 * and initializes the hardware.
403 *
404 * return 0 on success, positive on failure
405 *********************************************************************/
406
407 static int
lem_attach(device_t dev)408 lem_attach(device_t dev)
409 {
410 struct adapter *adapter;
411 int tsize, rsize;
412 int error = 0;
413
414 INIT_DEBUGOUT("lem_attach: begin");
415
416 adapter = device_get_softc(dev);
417 adapter->dev = adapter->osdep.dev = dev;
418 EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
419 EM_TX_LOCK_INIT(adapter, device_get_nameunit(dev));
420 EM_RX_LOCK_INIT(adapter, device_get_nameunit(dev));
421
422 /* SYSCTL stuff */
423 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
424 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
425 OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
426 lem_sysctl_nvm_info, "I", "NVM Information");
427
428 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
429 callout_init_mtx(&adapter->tx_fifo_timer, &adapter->tx_mtx, 0);
430
431 /* Determine hardware and mac info */
432 lem_identify_hardware(adapter);
433
434 /* Setup PCI resources */
435 if (lem_allocate_pci_resources(adapter)) {
436 device_printf(dev, "Allocation of PCI resources failed\n");
437 error = ENXIO;
438 goto err_pci;
439 }
440
441 /* Do Shared Code initialization */
442 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
443 device_printf(dev, "Setup of Shared code failed\n");
444 error = ENXIO;
445 goto err_pci;
446 }
447
448 e1000_get_bus_info(&adapter->hw);
449
450 /* Set up some sysctls for the tunable interrupt delays */
451 lem_add_int_delay_sysctl(adapter, "rx_int_delay",
452 "receive interrupt delay in usecs", &adapter->rx_int_delay,
453 E1000_REGISTER(&adapter->hw, E1000_RDTR), lem_rx_int_delay_dflt);
454 lem_add_int_delay_sysctl(adapter, "tx_int_delay",
455 "transmit interrupt delay in usecs", &adapter->tx_int_delay,
456 E1000_REGISTER(&adapter->hw, E1000_TIDV), lem_tx_int_delay_dflt);
457 if (adapter->hw.mac.type >= e1000_82540) {
458 lem_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
459 "receive interrupt delay limit in usecs",
460 &adapter->rx_abs_int_delay,
461 E1000_REGISTER(&adapter->hw, E1000_RADV),
462 lem_rx_abs_int_delay_dflt);
463 lem_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
464 "transmit interrupt delay limit in usecs",
465 &adapter->tx_abs_int_delay,
466 E1000_REGISTER(&adapter->hw, E1000_TADV),
467 lem_tx_abs_int_delay_dflt);
468 lem_add_int_delay_sysctl(adapter, "itr",
469 "interrupt delay limit in usecs/4",
470 &adapter->tx_itr,
471 E1000_REGISTER(&adapter->hw, E1000_ITR),
472 DEFAULT_ITR);
473 }
474
475 /* Sysctls for limiting the amount of work done in the taskqueue */
476 lem_add_rx_process_limit(adapter, "rx_processing_limit",
477 "max number of rx packets to process", &adapter->rx_process_limit,
478 lem_rx_process_limit);
479
480 #ifdef NIC_SEND_COMBINING
481 /* Sysctls to control mitigation */
482 lem_add_rx_process_limit(adapter, "sc_enable",
483 "driver TDT mitigation", &adapter->sc_enable, 0);
484 #endif /* NIC_SEND_COMBINING */
485 #ifdef BATCH_DISPATCH
486 lem_add_rx_process_limit(adapter, "batch_enable",
487 "driver rx batch", &adapter->batch_enable, 0);
488 #endif /* BATCH_DISPATCH */
489 #ifdef NIC_PARAVIRT
490 lem_add_rx_process_limit(adapter, "rx_retries",
491 "driver rx retries", &adapter->rx_retries, 0);
492 #endif /* NIC_PARAVIRT */
493
494 /* Sysctl for setting the interface flow control */
495 lem_set_flow_cntrl(adapter, "flow_control",
496 "flow control setting",
497 &adapter->fc_setting, lem_fc_setting);
498
499 /*
500 * Validate number of transmit and receive descriptors. It
501 * must not exceed hardware maximum, and must be multiple
502 * of E1000_DBA_ALIGN.
503 */
504 if (((lem_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 ||
505 (adapter->hw.mac.type >= e1000_82544 && lem_txd > EM_MAX_TXD) ||
506 (adapter->hw.mac.type < e1000_82544 && lem_txd > EM_MAX_TXD_82543) ||
507 (lem_txd < EM_MIN_TXD)) {
508 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
509 EM_DEFAULT_TXD, lem_txd);
510 adapter->num_tx_desc = EM_DEFAULT_TXD;
511 } else
512 adapter->num_tx_desc = lem_txd;
513 if (((lem_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 ||
514 (adapter->hw.mac.type >= e1000_82544 && lem_rxd > EM_MAX_RXD) ||
515 (adapter->hw.mac.type < e1000_82544 && lem_rxd > EM_MAX_RXD_82543) ||
516 (lem_rxd < EM_MIN_RXD)) {
517 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
518 EM_DEFAULT_RXD, lem_rxd);
519 adapter->num_rx_desc = EM_DEFAULT_RXD;
520 } else
521 adapter->num_rx_desc = lem_rxd;
522
523 adapter->hw.mac.autoneg = DO_AUTO_NEG;
524 adapter->hw.phy.autoneg_wait_to_complete = FALSE;
525 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
526 adapter->rx_buffer_len = 2048;
527
528 e1000_init_script_state_82541(&adapter->hw, TRUE);
529 e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE);
530
531 /* Copper options */
532 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
533 adapter->hw.phy.mdix = AUTO_ALL_MODES;
534 adapter->hw.phy.disable_polarity_correction = FALSE;
535 adapter->hw.phy.ms_type = EM_MASTER_SLAVE;
536 }
537
538 /*
539 * Set the frame limits assuming
540 * standard ethernet sized frames.
541 */
542 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
543 adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
544
545 /*
546 * This controls when hardware reports transmit completion
547 * status.
548 */
549 adapter->hw.mac.report_tx_early = 1;
550
551 #ifdef NIC_PARAVIRT
552 device_printf(dev, "driver supports paravirt, subdev 0x%x\n",
553 adapter->hw.subsystem_device_id);
554 if (adapter->hw.subsystem_device_id == E1000_PARA_SUBDEV) {
555 uint64_t bus_addr;
556
557 device_printf(dev, "paravirt support on dev %p\n", adapter);
558 tsize = 4096; // XXX one page for the csb
559 if (lem_dma_malloc(adapter, tsize, &adapter->csb_mem, BUS_DMA_NOWAIT)) {
560 device_printf(dev, "Unable to allocate csb memory\n");
561 error = ENOMEM;
562 goto err_csb;
563 }
564 /* Setup the Base of the CSB */
565 adapter->csb = (struct paravirt_csb *)adapter->csb_mem.dma_vaddr;
566 /* force the first kick */
567 adapter->csb->host_need_txkick = 1; /* txring empty */
568 adapter->csb->guest_need_rxkick = 1; /* no rx packets */
569 bus_addr = adapter->csb_mem.dma_paddr;
570 lem_add_rx_process_limit(adapter, "csb_on",
571 "enable paravirt.", &adapter->csb->guest_csb_on, 0);
572 lem_add_rx_process_limit(adapter, "txc_lim",
573 "txc_lim", &adapter->csb->host_txcycles_lim, 1);
574
575 /* some stats */
576 #define PA_SC(name, var, val) \
577 lem_add_rx_process_limit(adapter, name, name, var, val)
578 PA_SC("host_need_txkick",&adapter->csb->host_need_txkick, 1);
579 PA_SC("host_rxkick_at",&adapter->csb->host_rxkick_at, ~0);
580 PA_SC("guest_need_txkick",&adapter->csb->guest_need_txkick, 0);
581 PA_SC("guest_need_rxkick",&adapter->csb->guest_need_rxkick, 1);
582 PA_SC("tdt_reg_count",&adapter->tdt_reg_count, 0);
583 PA_SC("tdt_csb_count",&adapter->tdt_csb_count, 0);
584 PA_SC("tdt_int_count",&adapter->tdt_int_count, 0);
585 PA_SC("guest_need_kick_count",&adapter->guest_need_kick_count, 0);
586 /* tell the host where the block is */
587 E1000_WRITE_REG(&adapter->hw, E1000_CSBAH,
588 (u32)(bus_addr >> 32));
589 E1000_WRITE_REG(&adapter->hw, E1000_CSBAL,
590 (u32)bus_addr);
591 }
592 #endif /* NIC_PARAVIRT */
593
594 tsize = roundup2(adapter->num_tx_desc * sizeof(struct e1000_tx_desc),
595 EM_DBA_ALIGN);
596
597 /* Allocate Transmit Descriptor ring */
598 if (lem_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) {
599 device_printf(dev, "Unable to allocate tx_desc memory\n");
600 error = ENOMEM;
601 goto err_tx_desc;
602 }
603 adapter->tx_desc_base =
604 (struct e1000_tx_desc *)adapter->txdma.dma_vaddr;
605
606 rsize = roundup2(adapter->num_rx_desc * sizeof(struct e1000_rx_desc),
607 EM_DBA_ALIGN);
608
609 /* Allocate Receive Descriptor ring */
610 if (lem_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) {
611 device_printf(dev, "Unable to allocate rx_desc memory\n");
612 error = ENOMEM;
613 goto err_rx_desc;
614 }
615 adapter->rx_desc_base =
616 (struct e1000_rx_desc *)adapter->rxdma.dma_vaddr;
617
618 /* Allocate multicast array memory. */
619 adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
620 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
621 if (adapter->mta == NULL) {
622 device_printf(dev, "Can not allocate multicast setup array\n");
623 error = ENOMEM;
624 goto err_hw_init;
625 }
626
627 /*
628 ** Start from a known state, this is
629 ** important in reading the nvm and
630 ** mac from that.
631 */
632 e1000_reset_hw(&adapter->hw);
633
634 /* Make sure we have a good EEPROM before we read from it */
635 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
636 /*
637 ** Some PCI-E parts fail the first check due to
638 ** the link being in sleep state, call it again,
639 ** if it fails a second time its a real issue.
640 */
641 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
642 device_printf(dev,
643 "The EEPROM Checksum Is Not Valid\n");
644 error = EIO;
645 goto err_hw_init;
646 }
647 }
648
649 /* Copy the permanent MAC address out of the EEPROM */
650 if (e1000_read_mac_addr(&adapter->hw) < 0) {
651 device_printf(dev, "EEPROM read error while reading MAC"
652 " address\n");
653 error = EIO;
654 goto err_hw_init;
655 }
656
657 if (!lem_is_valid_ether_addr(adapter->hw.mac.addr)) {
658 device_printf(dev, "Invalid MAC address\n");
659 error = EIO;
660 goto err_hw_init;
661 }
662
663 /* Initialize the hardware */
664 if (lem_hardware_init(adapter)) {
665 device_printf(dev, "Unable to initialize the hardware\n");
666 error = EIO;
667 goto err_hw_init;
668 }
669
670 /* Allocate transmit descriptors and buffers */
671 if (lem_allocate_transmit_structures(adapter)) {
672 device_printf(dev, "Could not setup transmit structures\n");
673 error = ENOMEM;
674 goto err_tx_struct;
675 }
676
677 /* Allocate receive descriptors and buffers */
678 if (lem_allocate_receive_structures(adapter)) {
679 device_printf(dev, "Could not setup receive structures\n");
680 error = ENOMEM;
681 goto err_rx_struct;
682 }
683
684 /*
685 ** Do interrupt configuration
686 */
687 error = lem_allocate_irq(adapter);
688 if (error)
689 goto err_rx_struct;
690
691 /*
692 * Get Wake-on-Lan and Management info for later use
693 */
694 lem_get_wakeup(dev);
695
696 /* Setup OS specific network interface */
697 if (lem_setup_interface(dev, adapter) != 0)
698 goto err_rx_struct;
699
700 /* Initialize statistics */
701 lem_update_stats_counters(adapter);
702
703 adapter->hw.mac.get_link_status = 1;
704 lem_update_link_status(adapter);
705
706 /* Indicate SOL/IDER usage */
707 if (e1000_check_reset_block(&adapter->hw))
708 device_printf(dev,
709 "PHY reset is blocked due to SOL/IDER session.\n");
710
711 /* Do we need workaround for 82544 PCI-X adapter? */
712 if (adapter->hw.bus.type == e1000_bus_type_pcix &&
713 adapter->hw.mac.type == e1000_82544)
714 adapter->pcix_82544 = TRUE;
715 else
716 adapter->pcix_82544 = FALSE;
717
718 /* Register for VLAN events */
719 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
720 lem_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
721 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
722 lem_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
723
724 lem_add_hw_stats(adapter);
725
726 /* Non-AMT based hardware can now take control from firmware */
727 if (adapter->has_manage && !adapter->has_amt)
728 lem_get_hw_control(adapter);
729
730 /* Tell the stack that the interface is not active */
731 if_setdrvflagbits(adapter->ifp, 0, IFF_DRV_OACTIVE | IFF_DRV_RUNNING);
732
733 adapter->led_dev = led_create(lem_led_func, adapter,
734 device_get_nameunit(dev));
735
736 #ifdef DEV_NETMAP
737 lem_netmap_attach(adapter);
738 #endif /* DEV_NETMAP */
739 INIT_DEBUGOUT("lem_attach: end");
740
741 return (0);
742
743 err_rx_struct:
744 lem_free_transmit_structures(adapter);
745 err_tx_struct:
746 err_hw_init:
747 lem_release_hw_control(adapter);
748 lem_dma_free(adapter, &adapter->rxdma);
749 err_rx_desc:
750 lem_dma_free(adapter, &adapter->txdma);
751 err_tx_desc:
752 #ifdef NIC_PARAVIRT
753 lem_dma_free(adapter, &adapter->csb_mem);
754 err_csb:
755 #endif /* NIC_PARAVIRT */
756
757 err_pci:
758 if (adapter->ifp != (void *)NULL)
759 if_free(adapter->ifp);
760 lem_free_pci_resources(adapter);
761 free(adapter->mta, M_DEVBUF);
762 EM_TX_LOCK_DESTROY(adapter);
763 EM_RX_LOCK_DESTROY(adapter);
764 EM_CORE_LOCK_DESTROY(adapter);
765
766 return (error);
767 }
768
769 /*********************************************************************
770 * Device removal routine
771 *
772 * The detach entry point is called when the driver is being removed.
773 * This routine stops the adapter and deallocates all the resources
774 * that were allocated for driver operation.
775 *
776 * return 0 on success, positive on failure
777 *********************************************************************/
778
779 static int
lem_detach(device_t dev)780 lem_detach(device_t dev)
781 {
782 struct adapter *adapter = device_get_softc(dev);
783 if_t ifp = adapter->ifp;
784
785 INIT_DEBUGOUT("em_detach: begin");
786
787 /* Make sure VLANS are not using driver */
788 if (if_vlantrunkinuse(ifp)) {
789 device_printf(dev,"Vlan in use, detach first\n");
790 return (EBUSY);
791 }
792
793 #ifdef DEVICE_POLLING
794 if (if_getcapenable(ifp) & IFCAP_POLLING)
795 ether_poll_deregister(ifp);
796 #endif
797
798 if (adapter->led_dev != NULL)
799 led_destroy(adapter->led_dev);
800
801 EM_CORE_LOCK(adapter);
802 EM_TX_LOCK(adapter);
803 adapter->in_detach = 1;
804 lem_stop(adapter);
805 e1000_phy_hw_reset(&adapter->hw);
806
807 lem_release_manageability(adapter);
808
809 EM_TX_UNLOCK(adapter);
810 EM_CORE_UNLOCK(adapter);
811
812 /* Unregister VLAN events */
813 if (adapter->vlan_attach != NULL)
814 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
815 if (adapter->vlan_detach != NULL)
816 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
817
818 ether_ifdetach(adapter->ifp);
819 callout_drain(&adapter->timer);
820 callout_drain(&adapter->tx_fifo_timer);
821
822 #ifdef DEV_NETMAP
823 netmap_detach(ifp);
824 #endif /* DEV_NETMAP */
825 lem_free_pci_resources(adapter);
826 bus_generic_detach(dev);
827 if_free(ifp);
828
829 lem_free_transmit_structures(adapter);
830 lem_free_receive_structures(adapter);
831
832 /* Free Transmit Descriptor ring */
833 if (adapter->tx_desc_base) {
834 lem_dma_free(adapter, &adapter->txdma);
835 adapter->tx_desc_base = NULL;
836 }
837
838 /* Free Receive Descriptor ring */
839 if (adapter->rx_desc_base) {
840 lem_dma_free(adapter, &adapter->rxdma);
841 adapter->rx_desc_base = NULL;
842 }
843
844 #ifdef NIC_PARAVIRT
845 if (adapter->csb) {
846 lem_dma_free(adapter, &adapter->csb_mem);
847 adapter->csb = NULL;
848 }
849 #endif /* NIC_PARAVIRT */
850 lem_release_hw_control(adapter);
851 free(adapter->mta, M_DEVBUF);
852 EM_TX_LOCK_DESTROY(adapter);
853 EM_RX_LOCK_DESTROY(adapter);
854 EM_CORE_LOCK_DESTROY(adapter);
855
856 return (0);
857 }
858
859 /*********************************************************************
860 *
861 * Shutdown entry point
862 *
863 **********************************************************************/
864
865 static int
lem_shutdown(device_t dev)866 lem_shutdown(device_t dev)
867 {
868 return lem_suspend(dev);
869 }
870
871 /*
872 * Suspend/resume device methods.
873 */
874 static int
lem_suspend(device_t dev)875 lem_suspend(device_t dev)
876 {
877 struct adapter *adapter = device_get_softc(dev);
878
879 EM_CORE_LOCK(adapter);
880
881 lem_release_manageability(adapter);
882 lem_release_hw_control(adapter);
883 lem_enable_wakeup(dev);
884
885 EM_CORE_UNLOCK(adapter);
886
887 return bus_generic_suspend(dev);
888 }
889
890 static int
lem_resume(device_t dev)891 lem_resume(device_t dev)
892 {
893 struct adapter *adapter = device_get_softc(dev);
894 if_t ifp = adapter->ifp;
895
896 EM_CORE_LOCK(adapter);
897 lem_init_locked(adapter);
898 lem_init_manageability(adapter);
899 EM_CORE_UNLOCK(adapter);
900 lem_start(ifp);
901
902 return bus_generic_resume(dev);
903 }
904
905
906 static void
lem_start_locked(if_t ifp)907 lem_start_locked(if_t ifp)
908 {
909 struct adapter *adapter = if_getsoftc(ifp);
910 struct mbuf *m_head;
911
912 EM_TX_LOCK_ASSERT(adapter);
913
914 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
915 IFF_DRV_RUNNING)
916 return;
917 if (!adapter->link_active)
918 return;
919
920 /*
921 * Force a cleanup if number of TX descriptors
922 * available hits the threshold
923 */
924 if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
925 lem_txeof(adapter);
926 /* Now do we at least have a minimal? */
927 if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) {
928 adapter->no_tx_desc_avail1++;
929 return;
930 }
931 }
932
933 while (!if_sendq_empty(ifp)) {
934 m_head = if_dequeue(ifp);
935
936 if (m_head == NULL)
937 break;
938 /*
939 * Encapsulation can modify our pointer, and or make it
940 * NULL on failure. In that event, we can't requeue.
941 */
942 if (lem_xmit(adapter, &m_head)) {
943 if (m_head == NULL)
944 break;
945 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0);
946 if_sendq_prepend(ifp, m_head);
947 break;
948 }
949
950 /* Send a copy of the frame to the BPF listener */
951 if_etherbpfmtap(ifp, m_head);
952
953 /* Set timeout in case hardware has problems transmitting. */
954 adapter->watchdog_check = TRUE;
955 adapter->watchdog_time = ticks;
956 }
957 if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD)
958 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0);
959 #ifdef NIC_PARAVIRT
960 if (if_getdrvflags(ifp) & IFF_DRV_OACTIVE && adapter->csb &&
961 adapter->csb->guest_csb_on &&
962 !(adapter->csb->guest_need_txkick & 1)) {
963 adapter->csb->guest_need_txkick = 1;
964 adapter->guest_need_kick_count++;
965 // XXX memory barrier
966 lem_txeof(adapter); // XXX possibly clear IFF_DRV_OACTIVE
967 }
968 #endif /* NIC_PARAVIRT */
969
970 return;
971 }
972
973 static void
lem_start(if_t ifp)974 lem_start(if_t ifp)
975 {
976 struct adapter *adapter = if_getsoftc(ifp);
977
978 EM_TX_LOCK(adapter);
979 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
980 lem_start_locked(ifp);
981 EM_TX_UNLOCK(adapter);
982 }
983
984 /*********************************************************************
985 * Ioctl entry point
986 *
987 * em_ioctl is called when the user wants to configure the
988 * interface.
989 *
990 * return 0 on success, positive on failure
991 **********************************************************************/
992
993 static int
lem_ioctl(if_t ifp,u_long command,caddr_t data)994 lem_ioctl(if_t ifp, u_long command, caddr_t data)
995 {
996 struct adapter *adapter = if_getsoftc(ifp);
997 struct ifreq *ifr = (struct ifreq *)data;
998 #if defined(INET) || defined(INET6)
999 struct ifaddr *ifa = (struct ifaddr *)data;
1000 #endif
1001 bool avoid_reset = FALSE;
1002 int error = 0;
1003
1004 if (adapter->in_detach)
1005 return (error);
1006
1007 switch (command) {
1008 case SIOCSIFADDR:
1009 #ifdef INET
1010 if (ifa->ifa_addr->sa_family == AF_INET)
1011 avoid_reset = TRUE;
1012 #endif
1013 #ifdef INET6
1014 if (ifa->ifa_addr->sa_family == AF_INET6)
1015 avoid_reset = TRUE;
1016 #endif
1017 /*
1018 ** Calling init results in link renegotiation,
1019 ** so we avoid doing it when possible.
1020 */
1021 if (avoid_reset) {
1022 if_setflagbits(ifp, IFF_UP, 0);
1023 if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))
1024 lem_init(adapter);
1025 #ifdef INET
1026 if (!(if_getflags(ifp) & IFF_NOARP))
1027 arp_ifinit(ifp, ifa);
1028 #endif
1029 } else
1030 error = ether_ioctl(ifp, command, data);
1031 break;
1032 case SIOCSIFMTU:
1033 {
1034 int max_frame_size;
1035
1036 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
1037
1038 EM_CORE_LOCK(adapter);
1039 switch (adapter->hw.mac.type) {
1040 case e1000_82542:
1041 max_frame_size = ETHER_MAX_LEN;
1042 break;
1043 default:
1044 max_frame_size = MAX_JUMBO_FRAME_SIZE;
1045 }
1046 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1047 ETHER_CRC_LEN) {
1048 EM_CORE_UNLOCK(adapter);
1049 error = EINVAL;
1050 break;
1051 }
1052
1053 if_setmtu(ifp, ifr->ifr_mtu);
1054 adapter->max_frame_size =
1055 if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN;
1056 lem_init_locked(adapter);
1057 EM_CORE_UNLOCK(adapter);
1058 break;
1059 }
1060 case SIOCSIFFLAGS:
1061 IOCTL_DEBUGOUT("ioctl rcv'd:\
1062 SIOCSIFFLAGS (Set Interface Flags)");
1063 EM_CORE_LOCK(adapter);
1064 if (if_getflags(ifp) & IFF_UP) {
1065 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING)) {
1066 if ((if_getflags(ifp) ^ adapter->if_flags) &
1067 (IFF_PROMISC | IFF_ALLMULTI)) {
1068 lem_disable_promisc(adapter);
1069 lem_set_promisc(adapter);
1070 }
1071 } else
1072 lem_init_locked(adapter);
1073 } else
1074 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1075 EM_TX_LOCK(adapter);
1076 lem_stop(adapter);
1077 EM_TX_UNLOCK(adapter);
1078 }
1079 adapter->if_flags = if_getflags(ifp);
1080 EM_CORE_UNLOCK(adapter);
1081 break;
1082 case SIOCADDMULTI:
1083 case SIOCDELMULTI:
1084 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1085 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1086 EM_CORE_LOCK(adapter);
1087 lem_disable_intr(adapter);
1088 lem_set_multi(adapter);
1089 if (adapter->hw.mac.type == e1000_82542 &&
1090 adapter->hw.revision_id == E1000_REVISION_2) {
1091 lem_initialize_receive_unit(adapter);
1092 }
1093 #ifdef DEVICE_POLLING
1094 if (!(if_getcapenable(ifp) & IFCAP_POLLING))
1095 #endif
1096 lem_enable_intr(adapter);
1097 EM_CORE_UNLOCK(adapter);
1098 }
1099 break;
1100 case SIOCSIFMEDIA:
1101 /* Check SOL/IDER usage */
1102 EM_CORE_LOCK(adapter);
1103 if (e1000_check_reset_block(&adapter->hw)) {
1104 EM_CORE_UNLOCK(adapter);
1105 device_printf(adapter->dev, "Media change is"
1106 " blocked due to SOL/IDER session.\n");
1107 break;
1108 }
1109 EM_CORE_UNLOCK(adapter);
1110 case SIOCGIFMEDIA:
1111 IOCTL_DEBUGOUT("ioctl rcv'd: \
1112 SIOCxIFMEDIA (Get/Set Interface Media)");
1113 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1114 break;
1115 case SIOCSIFCAP:
1116 {
1117 int mask, reinit;
1118
1119 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1120 reinit = 0;
1121 mask = ifr->ifr_reqcap ^ if_getcapenable(ifp);
1122 #ifdef DEVICE_POLLING
1123 if (mask & IFCAP_POLLING) {
1124 if (ifr->ifr_reqcap & IFCAP_POLLING) {
1125 error = ether_poll_register(lem_poll, ifp);
1126 if (error)
1127 return (error);
1128 EM_CORE_LOCK(adapter);
1129 lem_disable_intr(adapter);
1130 if_setcapenablebit(ifp, IFCAP_POLLING, 0);
1131 EM_CORE_UNLOCK(adapter);
1132 } else {
1133 error = ether_poll_deregister(ifp);
1134 /* Enable interrupt even in error case */
1135 EM_CORE_LOCK(adapter);
1136 lem_enable_intr(adapter);
1137 if_setcapenablebit(ifp, 0, IFCAP_POLLING);
1138 EM_CORE_UNLOCK(adapter);
1139 }
1140 }
1141 #endif
1142 if (mask & IFCAP_HWCSUM) {
1143 if_togglecapenable(ifp, IFCAP_HWCSUM);
1144 reinit = 1;
1145 }
1146 if (mask & IFCAP_VLAN_HWTAGGING) {
1147 if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING);
1148 reinit = 1;
1149 }
1150 if ((mask & IFCAP_WOL) &&
1151 (if_getcapabilities(ifp) & IFCAP_WOL) != 0) {
1152 if (mask & IFCAP_WOL_MCAST)
1153 if_togglecapenable(ifp, IFCAP_WOL_MCAST);
1154 if (mask & IFCAP_WOL_MAGIC)
1155 if_togglecapenable(ifp, IFCAP_WOL_MAGIC);
1156 }
1157 if (reinit && (if_getdrvflags(ifp) & IFF_DRV_RUNNING))
1158 lem_init(adapter);
1159 if_vlancap(ifp);
1160 break;
1161 }
1162
1163 default:
1164 error = ether_ioctl(ifp, command, data);
1165 break;
1166 }
1167
1168 return (error);
1169 }
1170
1171
1172 /*********************************************************************
1173 * Init entry point
1174 *
1175 * This routine is used in two ways. It is used by the stack as
1176 * init entry point in network interface structure. It is also used
1177 * by the driver as a hw/sw initialization routine to get to a
1178 * consistent state.
1179 *
1180 * return 0 on success, positive on failure
1181 **********************************************************************/
1182
1183 static void
lem_init_locked(struct adapter * adapter)1184 lem_init_locked(struct adapter *adapter)
1185 {
1186 if_t ifp = adapter->ifp;
1187 device_t dev = adapter->dev;
1188 u32 pba;
1189
1190 INIT_DEBUGOUT("lem_init: begin");
1191
1192 EM_CORE_LOCK_ASSERT(adapter);
1193
1194 EM_TX_LOCK(adapter);
1195 lem_stop(adapter);
1196 EM_TX_UNLOCK(adapter);
1197
1198 /*
1199 * Packet Buffer Allocation (PBA)
1200 * Writing PBA sets the receive portion of the buffer
1201 * the remainder is used for the transmit buffer.
1202 *
1203 * Devices before the 82547 had a Packet Buffer of 64K.
1204 * Default allocation: PBA=48K for Rx, leaving 16K for Tx.
1205 * After the 82547 the buffer was reduced to 40K.
1206 * Default allocation: PBA=30K for Rx, leaving 10K for Tx.
1207 * Note: default does not leave enough room for Jumbo Frame >10k.
1208 */
1209 switch (adapter->hw.mac.type) {
1210 case e1000_82547:
1211 case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */
1212 if (adapter->max_frame_size > 8192)
1213 pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */
1214 else
1215 pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */
1216 adapter->tx_fifo_head = 0;
1217 adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT;
1218 adapter->tx_fifo_size =
1219 (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT;
1220 break;
1221 default:
1222 /* Devices before 82547 had a Packet Buffer of 64K. */
1223 if (adapter->max_frame_size > 8192)
1224 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
1225 else
1226 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
1227 }
1228
1229 INIT_DEBUGOUT1("lem_init: pba=%dK",pba);
1230 E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
1231
1232 /* Get the latest mac address, User can use a LAA */
1233 bcopy(if_getlladdr(adapter->ifp), adapter->hw.mac.addr,
1234 ETHER_ADDR_LEN);
1235
1236 /* Put the address into the Receive Address Array */
1237 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1238
1239 /* Initialize the hardware */
1240 if (lem_hardware_init(adapter)) {
1241 device_printf(dev, "Unable to initialize the hardware\n");
1242 return;
1243 }
1244 lem_update_link_status(adapter);
1245
1246 /* Setup VLAN support, basic and offload if available */
1247 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1248
1249 /* Set hardware offload abilities */
1250 if_clearhwassist(ifp);
1251 if (adapter->hw.mac.type >= e1000_82543) {
1252 if (if_getcapenable(ifp) & IFCAP_TXCSUM)
1253 if_sethwassistbits(ifp, CSUM_TCP | CSUM_UDP, 0);
1254 }
1255
1256 /* Configure for OS presence */
1257 lem_init_manageability(adapter);
1258
1259 /* Prepare transmit descriptors and buffers */
1260 lem_setup_transmit_structures(adapter);
1261 lem_initialize_transmit_unit(adapter);
1262
1263 /* Setup Multicast table */
1264 lem_set_multi(adapter);
1265
1266 /* Prepare receive descriptors and buffers */
1267 if (lem_setup_receive_structures(adapter)) {
1268 device_printf(dev, "Could not setup receive structures\n");
1269 EM_TX_LOCK(adapter);
1270 lem_stop(adapter);
1271 EM_TX_UNLOCK(adapter);
1272 return;
1273 }
1274 lem_initialize_receive_unit(adapter);
1275
1276 /* Use real VLAN Filter support? */
1277 if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) {
1278 if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
1279 /* Use real VLAN Filter support */
1280 lem_setup_vlan_hw_support(adapter);
1281 else {
1282 u32 ctrl;
1283 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
1284 ctrl |= E1000_CTRL_VME;
1285 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
1286 }
1287 }
1288
1289 /* Don't lose promiscuous settings */
1290 lem_set_promisc(adapter);
1291
1292 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
1293
1294 callout_reset(&adapter->timer, hz, lem_local_timer, adapter);
1295 e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1296
1297 #ifdef DEVICE_POLLING
1298 /*
1299 * Only enable interrupts if we are not polling, make sure
1300 * they are off otherwise.
1301 */
1302 if (if_getcapenable(ifp) & IFCAP_POLLING)
1303 lem_disable_intr(adapter);
1304 else
1305 #endif /* DEVICE_POLLING */
1306 lem_enable_intr(adapter);
1307
1308 /* AMT based hardware can now take control from firmware */
1309 if (adapter->has_manage && adapter->has_amt)
1310 lem_get_hw_control(adapter);
1311 }
1312
1313 static void
lem_init(void * arg)1314 lem_init(void *arg)
1315 {
1316 struct adapter *adapter = arg;
1317
1318 EM_CORE_LOCK(adapter);
1319 lem_init_locked(adapter);
1320 EM_CORE_UNLOCK(adapter);
1321 }
1322
1323
1324 #ifdef DEVICE_POLLING
1325 /*********************************************************************
1326 *
1327 * Legacy polling routine
1328 *
1329 *********************************************************************/
1330 static int
lem_poll(if_t ifp,enum poll_cmd cmd,int count)1331 lem_poll(if_t ifp, enum poll_cmd cmd, int count)
1332 {
1333 struct adapter *adapter = if_getsoftc(ifp);
1334 u32 reg_icr, rx_done = 0;
1335
1336 EM_CORE_LOCK(adapter);
1337 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
1338 EM_CORE_UNLOCK(adapter);
1339 return (rx_done);
1340 }
1341
1342 if (cmd == POLL_AND_CHECK_STATUS) {
1343 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1344 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1345 callout_stop(&adapter->timer);
1346 adapter->hw.mac.get_link_status = 1;
1347 lem_update_link_status(adapter);
1348 callout_reset(&adapter->timer, hz,
1349 lem_local_timer, adapter);
1350 }
1351 }
1352 EM_CORE_UNLOCK(adapter);
1353
1354 lem_rxeof(adapter, count, &rx_done);
1355
1356 EM_TX_LOCK(adapter);
1357 lem_txeof(adapter);
1358 if(!if_sendq_empty(ifp))
1359 lem_start_locked(ifp);
1360 EM_TX_UNLOCK(adapter);
1361 return (rx_done);
1362 }
1363 #endif /* DEVICE_POLLING */
1364
1365 /*********************************************************************
1366 *
1367 * Legacy Interrupt Service routine
1368 *
1369 *********************************************************************/
1370 static void
lem_intr(void * arg)1371 lem_intr(void *arg)
1372 {
1373 struct adapter *adapter = arg;
1374 if_t ifp = adapter->ifp;
1375 u32 reg_icr;
1376
1377
1378 if ((if_getcapenable(ifp) & IFCAP_POLLING) ||
1379 ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0))
1380 return;
1381
1382 EM_CORE_LOCK(adapter);
1383 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1384 if (reg_icr & E1000_ICR_RXO)
1385 adapter->rx_overruns++;
1386
1387 if ((reg_icr == 0xffffffff) || (reg_icr == 0)) {
1388 EM_CORE_UNLOCK(adapter);
1389 return;
1390 }
1391
1392 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1393 callout_stop(&adapter->timer);
1394 adapter->hw.mac.get_link_status = 1;
1395 lem_update_link_status(adapter);
1396 /* Deal with TX cruft when link lost */
1397 lem_tx_purge(adapter);
1398 callout_reset(&adapter->timer, hz,
1399 lem_local_timer, adapter);
1400 EM_CORE_UNLOCK(adapter);
1401 return;
1402 }
1403
1404 EM_CORE_UNLOCK(adapter);
1405 lem_rxeof(adapter, -1, NULL);
1406
1407 EM_TX_LOCK(adapter);
1408 lem_txeof(adapter);
1409 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) &&
1410 (!if_sendq_empty(ifp)))
1411 lem_start_locked(ifp);
1412 EM_TX_UNLOCK(adapter);
1413 return;
1414 }
1415
1416
1417 static void
lem_handle_link(void * context,int pending)1418 lem_handle_link(void *context, int pending)
1419 {
1420 struct adapter *adapter = context;
1421 if_t ifp = adapter->ifp;
1422
1423 if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))
1424 return;
1425
1426 EM_CORE_LOCK(adapter);
1427 callout_stop(&adapter->timer);
1428 lem_update_link_status(adapter);
1429 /* Deal with TX cruft when link lost */
1430 lem_tx_purge(adapter);
1431 callout_reset(&adapter->timer, hz, lem_local_timer, adapter);
1432 EM_CORE_UNLOCK(adapter);
1433 }
1434
1435
1436 /* Combined RX/TX handler, used by Legacy and MSI */
1437 static void
lem_handle_rxtx(void * context,int pending)1438 lem_handle_rxtx(void *context, int pending)
1439 {
1440 struct adapter *adapter = context;
1441 if_t ifp = adapter->ifp;
1442
1443
1444 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1445 bool more = lem_rxeof(adapter, adapter->rx_process_limit, NULL);
1446 EM_TX_LOCK(adapter);
1447 lem_txeof(adapter);
1448 if(!if_sendq_empty(ifp))
1449 lem_start_locked(ifp);
1450 EM_TX_UNLOCK(adapter);
1451 if (more) {
1452 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1453 return;
1454 }
1455 }
1456
1457 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
1458 lem_enable_intr(adapter);
1459 }
1460
1461 /*********************************************************************
1462 *
1463 * Fast Legacy/MSI Combined Interrupt Service routine
1464 *
1465 *********************************************************************/
1466 static int
lem_irq_fast(void * arg)1467 lem_irq_fast(void *arg)
1468 {
1469 struct adapter *adapter = arg;
1470 if_t ifp;
1471 u32 reg_icr;
1472
1473 ifp = adapter->ifp;
1474
1475 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1476
1477 /* Hot eject? */
1478 if (reg_icr == 0xffffffff)
1479 return FILTER_STRAY;
1480
1481 /* Definitely not our interrupt. */
1482 if (reg_icr == 0x0)
1483 return FILTER_STRAY;
1484
1485 /*
1486 * Mask interrupts until the taskqueue is finished running. This is
1487 * cheap, just assume that it is needed. This also works around the
1488 * MSI message reordering errata on certain systems.
1489 */
1490 lem_disable_intr(adapter);
1491 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1492
1493 /* Link status change */
1494 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1495 adapter->hw.mac.get_link_status = 1;
1496 taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
1497 }
1498
1499 if (reg_icr & E1000_ICR_RXO)
1500 adapter->rx_overruns++;
1501 return FILTER_HANDLED;
1502 }
1503
1504
1505 /*********************************************************************
1506 *
1507 * Media Ioctl callback
1508 *
1509 * This routine is called whenever the user queries the status of
1510 * the interface using ifconfig.
1511 *
1512 **********************************************************************/
1513 static void
lem_media_status(if_t ifp,struct ifmediareq * ifmr)1514 lem_media_status(if_t ifp, struct ifmediareq *ifmr)
1515 {
1516 struct adapter *adapter = if_getsoftc(ifp);
1517 u_char fiber_type = IFM_1000_SX;
1518
1519 INIT_DEBUGOUT("lem_media_status: begin");
1520
1521 EM_CORE_LOCK(adapter);
1522 lem_update_link_status(adapter);
1523
1524 ifmr->ifm_status = IFM_AVALID;
1525 ifmr->ifm_active = IFM_ETHER;
1526
1527 if (!adapter->link_active) {
1528 EM_CORE_UNLOCK(adapter);
1529 return;
1530 }
1531
1532 ifmr->ifm_status |= IFM_ACTIVE;
1533
1534 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
1535 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
1536 if (adapter->hw.mac.type == e1000_82545)
1537 fiber_type = IFM_1000_LX;
1538 ifmr->ifm_active |= fiber_type | IFM_FDX;
1539 } else {
1540 switch (adapter->link_speed) {
1541 case 10:
1542 ifmr->ifm_active |= IFM_10_T;
1543 break;
1544 case 100:
1545 ifmr->ifm_active |= IFM_100_TX;
1546 break;
1547 case 1000:
1548 ifmr->ifm_active |= IFM_1000_T;
1549 break;
1550 }
1551 if (adapter->link_duplex == FULL_DUPLEX)
1552 ifmr->ifm_active |= IFM_FDX;
1553 else
1554 ifmr->ifm_active |= IFM_HDX;
1555 }
1556 EM_CORE_UNLOCK(adapter);
1557 }
1558
1559 /*********************************************************************
1560 *
1561 * Media Ioctl callback
1562 *
1563 * This routine is called when the user changes speed/duplex using
1564 * media/mediopt option with ifconfig.
1565 *
1566 **********************************************************************/
1567 static int
lem_media_change(if_t ifp)1568 lem_media_change(if_t ifp)
1569 {
1570 struct adapter *adapter = if_getsoftc(ifp);
1571 struct ifmedia *ifm = &adapter->media;
1572
1573 INIT_DEBUGOUT("lem_media_change: begin");
1574
1575 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1576 return (EINVAL);
1577
1578 EM_CORE_LOCK(adapter);
1579 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1580 case IFM_AUTO:
1581 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1582 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1583 break;
1584 case IFM_1000_LX:
1585 case IFM_1000_SX:
1586 case IFM_1000_T:
1587 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1588 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1589 break;
1590 case IFM_100_TX:
1591 adapter->hw.mac.autoneg = FALSE;
1592 adapter->hw.phy.autoneg_advertised = 0;
1593 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1594 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1595 else
1596 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1597 break;
1598 case IFM_10_T:
1599 adapter->hw.mac.autoneg = FALSE;
1600 adapter->hw.phy.autoneg_advertised = 0;
1601 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1602 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1603 else
1604 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1605 break;
1606 default:
1607 device_printf(adapter->dev, "Unsupported media type\n");
1608 }
1609
1610 lem_init_locked(adapter);
1611 EM_CORE_UNLOCK(adapter);
1612
1613 return (0);
1614 }
1615
1616 /*********************************************************************
1617 *
1618 * This routine maps the mbufs to tx descriptors.
1619 *
1620 * return 0 on success, positive on failure
1621 **********************************************************************/
1622
1623 static int
lem_xmit(struct adapter * adapter,struct mbuf ** m_headp)1624 lem_xmit(struct adapter *adapter, struct mbuf **m_headp)
1625 {
1626 bus_dma_segment_t segs[EM_MAX_SCATTER];
1627 bus_dmamap_t map;
1628 struct em_buffer *tx_buffer, *tx_buffer_mapped;
1629 struct e1000_tx_desc *ctxd = NULL;
1630 struct mbuf *m_head;
1631 u32 txd_upper, txd_lower, txd_used, txd_saved;
1632 int error, nsegs, i, j, first, last = 0;
1633
1634 m_head = *m_headp;
1635 txd_upper = txd_lower = txd_used = txd_saved = 0;
1636
1637 /*
1638 ** When doing checksum offload, it is critical to
1639 ** make sure the first mbuf has more than header,
1640 ** because that routine expects data to be present.
1641 */
1642 if ((m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) &&
1643 (m_head->m_len < ETHER_HDR_LEN + sizeof(struct ip))) {
1644 m_head = m_pullup(m_head, ETHER_HDR_LEN + sizeof(struct ip));
1645 *m_headp = m_head;
1646 if (m_head == NULL)
1647 return (ENOBUFS);
1648 }
1649
1650 /*
1651 * Map the packet for DMA
1652 *
1653 * Capture the first descriptor index,
1654 * this descriptor will have the index
1655 * of the EOP which is the only one that
1656 * now gets a DONE bit writeback.
1657 */
1658 first = adapter->next_avail_tx_desc;
1659 tx_buffer = &adapter->tx_buffer_area[first];
1660 tx_buffer_mapped = tx_buffer;
1661 map = tx_buffer->map;
1662
1663 error = bus_dmamap_load_mbuf_sg(adapter->txtag, map,
1664 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1665
1666 /*
1667 * There are two types of errors we can (try) to handle:
1668 * - EFBIG means the mbuf chain was too long and bus_dma ran
1669 * out of segments. Defragment the mbuf chain and try again.
1670 * - ENOMEM means bus_dma could not obtain enough bounce buffers
1671 * at this point in time. Defer sending and try again later.
1672 * All other errors, in particular EINVAL, are fatal and prevent the
1673 * mbuf chain from ever going through. Drop it and report error.
1674 */
1675 if (error == EFBIG) {
1676 struct mbuf *m;
1677
1678 m = m_defrag(*m_headp, M_NOWAIT);
1679 if (m == NULL) {
1680 adapter->mbuf_alloc_failed++;
1681 m_freem(*m_headp);
1682 *m_headp = NULL;
1683 return (ENOBUFS);
1684 }
1685 *m_headp = m;
1686
1687 /* Try it again */
1688 error = bus_dmamap_load_mbuf_sg(adapter->txtag, map,
1689 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1690
1691 if (error) {
1692 adapter->no_tx_dma_setup++;
1693 m_freem(*m_headp);
1694 *m_headp = NULL;
1695 return (error);
1696 }
1697 } else if (error != 0) {
1698 adapter->no_tx_dma_setup++;
1699 return (error);
1700 }
1701
1702 if (nsegs > (adapter->num_tx_desc_avail - 2)) {
1703 adapter->no_tx_desc_avail2++;
1704 bus_dmamap_unload(adapter->txtag, map);
1705 return (ENOBUFS);
1706 }
1707 m_head = *m_headp;
1708
1709 /* Do hardware assists */
1710 if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)
1711 lem_transmit_checksum_setup(adapter, m_head,
1712 &txd_upper, &txd_lower);
1713
1714 i = adapter->next_avail_tx_desc;
1715 if (adapter->pcix_82544)
1716 txd_saved = i;
1717
1718 /* Set up our transmit descriptors */
1719 for (j = 0; j < nsegs; j++) {
1720 bus_size_t seg_len;
1721 bus_addr_t seg_addr;
1722 /* If adapter is 82544 and on PCIX bus */
1723 if(adapter->pcix_82544) {
1724 DESC_ARRAY desc_array;
1725 u32 array_elements, counter;
1726 /*
1727 * Check the Address and Length combination and
1728 * split the data accordingly
1729 */
1730 array_elements = lem_fill_descriptors(segs[j].ds_addr,
1731 segs[j].ds_len, &desc_array);
1732 for (counter = 0; counter < array_elements; counter++) {
1733 if (txd_used == adapter->num_tx_desc_avail) {
1734 adapter->next_avail_tx_desc = txd_saved;
1735 adapter->no_tx_desc_avail2++;
1736 bus_dmamap_unload(adapter->txtag, map);
1737 return (ENOBUFS);
1738 }
1739 tx_buffer = &adapter->tx_buffer_area[i];
1740 ctxd = &adapter->tx_desc_base[i];
1741 ctxd->buffer_addr = htole64(
1742 desc_array.descriptor[counter].address);
1743 ctxd->lower.data = htole32(
1744 (adapter->txd_cmd | txd_lower | (u16)
1745 desc_array.descriptor[counter].length));
1746 ctxd->upper.data =
1747 htole32((txd_upper));
1748 last = i;
1749 if (++i == adapter->num_tx_desc)
1750 i = 0;
1751 tx_buffer->m_head = NULL;
1752 tx_buffer->next_eop = -1;
1753 txd_used++;
1754 }
1755 } else {
1756 tx_buffer = &adapter->tx_buffer_area[i];
1757 ctxd = &adapter->tx_desc_base[i];
1758 seg_addr = segs[j].ds_addr;
1759 seg_len = segs[j].ds_len;
1760 ctxd->buffer_addr = htole64(seg_addr);
1761 ctxd->lower.data = htole32(
1762 adapter->txd_cmd | txd_lower | seg_len);
1763 ctxd->upper.data =
1764 htole32(txd_upper);
1765 last = i;
1766 if (++i == adapter->num_tx_desc)
1767 i = 0;
1768 tx_buffer->m_head = NULL;
1769 tx_buffer->next_eop = -1;
1770 }
1771 }
1772
1773 adapter->next_avail_tx_desc = i;
1774
1775 if (adapter->pcix_82544)
1776 adapter->num_tx_desc_avail -= txd_used;
1777 else
1778 adapter->num_tx_desc_avail -= nsegs;
1779
1780 if (m_head->m_flags & M_VLANTAG) {
1781 /* Set the vlan id. */
1782 ctxd->upper.fields.special =
1783 htole16(m_head->m_pkthdr.ether_vtag);
1784 /* Tell hardware to add tag */
1785 ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE);
1786 }
1787
1788 tx_buffer->m_head = m_head;
1789 tx_buffer_mapped->map = tx_buffer->map;
1790 tx_buffer->map = map;
1791 bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE);
1792
1793 /*
1794 * Last Descriptor of Packet
1795 * needs End Of Packet (EOP)
1796 * and Report Status (RS)
1797 */
1798 ctxd->lower.data |=
1799 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
1800 /*
1801 * Keep track in the first buffer which
1802 * descriptor will be written back
1803 */
1804 tx_buffer = &adapter->tx_buffer_area[first];
1805 tx_buffer->next_eop = last;
1806 adapter->watchdog_time = ticks;
1807
1808 /*
1809 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
1810 * that this frame is available to transmit.
1811 */
1812 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
1813 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1814
1815 #ifdef NIC_PARAVIRT
1816 if (adapter->csb) {
1817 adapter->csb->guest_tdt = i;
1818 /* XXX memory barrier ? */
1819 if (adapter->csb->guest_csb_on &&
1820 !(adapter->csb->host_need_txkick & 1)) {
1821 /* XXX maybe useless
1822 * clean the ring. maybe do it before ?
1823 * maybe a little bit of histeresys ?
1824 */
1825 if (adapter->num_tx_desc_avail <= 64) {// XXX
1826 lem_txeof(adapter);
1827 }
1828 return (0);
1829 }
1830 }
1831 #endif /* NIC_PARAVIRT */
1832
1833 #ifdef NIC_SEND_COMBINING
1834 if (adapter->sc_enable) {
1835 if (adapter->shadow_tdt & MIT_PENDING_INT) {
1836 /* signal intr and data pending */
1837 adapter->shadow_tdt = MIT_PENDING_TDT | (i & 0xffff);
1838 return (0);
1839 } else {
1840 adapter->shadow_tdt = MIT_PENDING_INT;
1841 }
1842 }
1843 #endif /* NIC_SEND_COMBINING */
1844
1845 if (adapter->hw.mac.type == e1000_82547 &&
1846 adapter->link_duplex == HALF_DUPLEX)
1847 lem_82547_move_tail(adapter);
1848 else {
1849 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i);
1850 if (adapter->hw.mac.type == e1000_82547)
1851 lem_82547_update_fifo_head(adapter,
1852 m_head->m_pkthdr.len);
1853 }
1854
1855 return (0);
1856 }
1857
1858 /*********************************************************************
1859 *
1860 * 82547 workaround to avoid controller hang in half-duplex environment.
1861 * The workaround is to avoid queuing a large packet that would span
1862 * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
1863 * in this case. We do that only when FIFO is quiescent.
1864 *
1865 **********************************************************************/
1866 static void
lem_82547_move_tail(void * arg)1867 lem_82547_move_tail(void *arg)
1868 {
1869 struct adapter *adapter = arg;
1870 struct e1000_tx_desc *tx_desc;
1871 u16 hw_tdt, sw_tdt, length = 0;
1872 bool eop = 0;
1873
1874 EM_TX_LOCK_ASSERT(adapter);
1875
1876 hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0));
1877 sw_tdt = adapter->next_avail_tx_desc;
1878
1879 while (hw_tdt != sw_tdt) {
1880 tx_desc = &adapter->tx_desc_base[hw_tdt];
1881 length += tx_desc->lower.flags.length;
1882 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
1883 if (++hw_tdt == adapter->num_tx_desc)
1884 hw_tdt = 0;
1885
1886 if (eop) {
1887 if (lem_82547_fifo_workaround(adapter, length)) {
1888 adapter->tx_fifo_wrk_cnt++;
1889 callout_reset(&adapter->tx_fifo_timer, 1,
1890 lem_82547_move_tail, adapter);
1891 break;
1892 }
1893 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt);
1894 lem_82547_update_fifo_head(adapter, length);
1895 length = 0;
1896 }
1897 }
1898 }
1899
1900 static int
lem_82547_fifo_workaround(struct adapter * adapter,int len)1901 lem_82547_fifo_workaround(struct adapter *adapter, int len)
1902 {
1903 int fifo_space, fifo_pkt_len;
1904
1905 fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
1906
1907 if (adapter->link_duplex == HALF_DUPLEX) {
1908 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
1909
1910 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
1911 if (lem_82547_tx_fifo_reset(adapter))
1912 return (0);
1913 else
1914 return (1);
1915 }
1916 }
1917
1918 return (0);
1919 }
1920
1921 static void
lem_82547_update_fifo_head(struct adapter * adapter,int len)1922 lem_82547_update_fifo_head(struct adapter *adapter, int len)
1923 {
1924 int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
1925
1926 /* tx_fifo_head is always 16 byte aligned */
1927 adapter->tx_fifo_head += fifo_pkt_len;
1928 if (adapter->tx_fifo_head >= adapter->tx_fifo_size) {
1929 adapter->tx_fifo_head -= adapter->tx_fifo_size;
1930 }
1931 }
1932
1933
1934 static int
lem_82547_tx_fifo_reset(struct adapter * adapter)1935 lem_82547_tx_fifo_reset(struct adapter *adapter)
1936 {
1937 u32 tctl;
1938
1939 if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) ==
1940 E1000_READ_REG(&adapter->hw, E1000_TDH(0))) &&
1941 (E1000_READ_REG(&adapter->hw, E1000_TDFT) ==
1942 E1000_READ_REG(&adapter->hw, E1000_TDFH)) &&
1943 (E1000_READ_REG(&adapter->hw, E1000_TDFTS) ==
1944 E1000_READ_REG(&adapter->hw, E1000_TDFHS)) &&
1945 (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) {
1946 /* Disable TX unit */
1947 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
1948 E1000_WRITE_REG(&adapter->hw, E1000_TCTL,
1949 tctl & ~E1000_TCTL_EN);
1950
1951 /* Reset FIFO pointers */
1952 E1000_WRITE_REG(&adapter->hw, E1000_TDFT,
1953 adapter->tx_head_addr);
1954 E1000_WRITE_REG(&adapter->hw, E1000_TDFH,
1955 adapter->tx_head_addr);
1956 E1000_WRITE_REG(&adapter->hw, E1000_TDFTS,
1957 adapter->tx_head_addr);
1958 E1000_WRITE_REG(&adapter->hw, E1000_TDFHS,
1959 adapter->tx_head_addr);
1960
1961 /* Re-enable TX unit */
1962 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
1963 E1000_WRITE_FLUSH(&adapter->hw);
1964
1965 adapter->tx_fifo_head = 0;
1966 adapter->tx_fifo_reset_cnt++;
1967
1968 return (TRUE);
1969 }
1970 else {
1971 return (FALSE);
1972 }
1973 }
1974
1975 static void
lem_set_promisc(struct adapter * adapter)1976 lem_set_promisc(struct adapter *adapter)
1977 {
1978 if_t ifp = adapter->ifp;
1979 u32 reg_rctl;
1980
1981 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1982
1983 if (if_getflags(ifp) & IFF_PROMISC) {
1984 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1985 /* Turn this on if you want to see bad packets */
1986 if (lem_debug_sbp)
1987 reg_rctl |= E1000_RCTL_SBP;
1988 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1989 } else if (if_getflags(ifp) & IFF_ALLMULTI) {
1990 reg_rctl |= E1000_RCTL_MPE;
1991 reg_rctl &= ~E1000_RCTL_UPE;
1992 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1993 }
1994 }
1995
1996 static void
lem_disable_promisc(struct adapter * adapter)1997 lem_disable_promisc(struct adapter *adapter)
1998 {
1999 if_t ifp = adapter->ifp;
2000 u32 reg_rctl;
2001 int mcnt = 0;
2002
2003 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2004 reg_rctl &= (~E1000_RCTL_UPE);
2005 if (if_getflags(ifp) & IFF_ALLMULTI)
2006 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
2007 else
2008 mcnt = if_multiaddr_count(ifp, MAX_NUM_MULTICAST_ADDRESSES);
2009
2010 /* Don't disable if in MAX groups */
2011 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
2012 reg_rctl &= (~E1000_RCTL_MPE);
2013 reg_rctl &= (~E1000_RCTL_SBP);
2014 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2015 }
2016
2017
2018 /*********************************************************************
2019 * Multicast Update
2020 *
2021 * This routine is called whenever multicast address list is updated.
2022 *
2023 **********************************************************************/
2024
2025 static void
lem_set_multi(struct adapter * adapter)2026 lem_set_multi(struct adapter *adapter)
2027 {
2028 if_t ifp = adapter->ifp;
2029 u32 reg_rctl = 0;
2030 u8 *mta; /* Multicast array memory */
2031 int mcnt = 0;
2032
2033 IOCTL_DEBUGOUT("lem_set_multi: begin");
2034
2035 mta = adapter->mta;
2036 bzero(mta, sizeof(u8) * ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES);
2037
2038 if (adapter->hw.mac.type == e1000_82542 &&
2039 adapter->hw.revision_id == E1000_REVISION_2) {
2040 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2041 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2042 e1000_pci_clear_mwi(&adapter->hw);
2043 reg_rctl |= E1000_RCTL_RST;
2044 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2045 msec_delay(5);
2046 }
2047
2048 if_multiaddr_array(ifp, mta, &mcnt, MAX_NUM_MULTICAST_ADDRESSES);
2049
2050 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
2051 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2052 reg_rctl |= E1000_RCTL_MPE;
2053 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2054 } else
2055 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
2056
2057 if (adapter->hw.mac.type == e1000_82542 &&
2058 adapter->hw.revision_id == E1000_REVISION_2) {
2059 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2060 reg_rctl &= ~E1000_RCTL_RST;
2061 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2062 msec_delay(5);
2063 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2064 e1000_pci_set_mwi(&adapter->hw);
2065 }
2066 }
2067
2068
2069 /*********************************************************************
2070 * Timer routine
2071 *
2072 * This routine checks for link status and updates statistics.
2073 *
2074 **********************************************************************/
2075
2076 static void
lem_local_timer(void * arg)2077 lem_local_timer(void *arg)
2078 {
2079 struct adapter *adapter = arg;
2080
2081 EM_CORE_LOCK_ASSERT(adapter);
2082
2083 lem_update_link_status(adapter);
2084 lem_update_stats_counters(adapter);
2085
2086 lem_smartspeed(adapter);
2087
2088 #ifdef NIC_PARAVIRT
2089 /* recover space if needed */
2090 if (adapter->csb && adapter->csb->guest_csb_on &&
2091 (adapter->watchdog_check == TRUE) &&
2092 (ticks - adapter->watchdog_time > EM_WATCHDOG) &&
2093 (adapter->num_tx_desc_avail != adapter->num_tx_desc) ) {
2094 lem_txeof(adapter);
2095 /*
2096 * lem_txeof() normally (except when space in the queue
2097 * runs low XXX) cleans watchdog_check so that
2098 * we do not hung.
2099 */
2100 }
2101 #endif /* NIC_PARAVIRT */
2102 /*
2103 * We check the watchdog: the time since
2104 * the last TX descriptor was cleaned.
2105 * This implies a functional TX engine.
2106 */
2107 if ((adapter->watchdog_check == TRUE) &&
2108 (ticks - adapter->watchdog_time > EM_WATCHDOG))
2109 goto hung;
2110
2111 callout_reset(&adapter->timer, hz, lem_local_timer, adapter);
2112 return;
2113 hung:
2114 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2115 if_setdrvflagbits(adapter->ifp, 0, IFF_DRV_RUNNING);
2116 adapter->watchdog_events++;
2117 lem_init_locked(adapter);
2118 }
2119
2120 static void
lem_update_link_status(struct adapter * adapter)2121 lem_update_link_status(struct adapter *adapter)
2122 {
2123 struct e1000_hw *hw = &adapter->hw;
2124 if_t ifp = adapter->ifp;
2125 device_t dev = adapter->dev;
2126 u32 link_check = 0;
2127
2128 /* Get the cached link value or read phy for real */
2129 switch (hw->phy.media_type) {
2130 case e1000_media_type_copper:
2131 if (hw->mac.get_link_status) {
2132 /* Do the work to read phy */
2133 e1000_check_for_link(hw);
2134 link_check = !hw->mac.get_link_status;
2135 if (link_check) /* ESB2 fix */
2136 e1000_cfg_on_link_up(hw);
2137 } else
2138 link_check = TRUE;
2139 break;
2140 case e1000_media_type_fiber:
2141 e1000_check_for_link(hw);
2142 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2143 E1000_STATUS_LU);
2144 break;
2145 case e1000_media_type_internal_serdes:
2146 e1000_check_for_link(hw);
2147 link_check = adapter->hw.mac.serdes_has_link;
2148 break;
2149 default:
2150 case e1000_media_type_unknown:
2151 break;
2152 }
2153
2154 /* Now check for a transition */
2155 if (link_check && (adapter->link_active == 0)) {
2156 e1000_get_speed_and_duplex(hw, &adapter->link_speed,
2157 &adapter->link_duplex);
2158 if (bootverbose)
2159 device_printf(dev, "Link is up %d Mbps %s\n",
2160 adapter->link_speed,
2161 ((adapter->link_duplex == FULL_DUPLEX) ?
2162 "Full Duplex" : "Half Duplex"));
2163 adapter->link_active = 1;
2164 adapter->smartspeed = 0;
2165 if_setbaudrate(ifp, adapter->link_speed * 1000000);
2166 if_link_state_change(ifp, LINK_STATE_UP);
2167 } else if (!link_check && (adapter->link_active == 1)) {
2168 if_setbaudrate(ifp, 0);
2169 adapter->link_speed = 0;
2170 adapter->link_duplex = 0;
2171 if (bootverbose)
2172 device_printf(dev, "Link is Down\n");
2173 adapter->link_active = 0;
2174 /* Link down, disable watchdog */
2175 adapter->watchdog_check = FALSE;
2176 if_link_state_change(ifp, LINK_STATE_DOWN);
2177 }
2178 }
2179
2180 /*********************************************************************
2181 *
2182 * This routine disables all traffic on the adapter by issuing a
2183 * global reset on the MAC and deallocates TX/RX buffers.
2184 *
2185 * This routine should always be called with BOTH the CORE
2186 * and TX locks.
2187 **********************************************************************/
2188
2189 static void
lem_stop(void * arg)2190 lem_stop(void *arg)
2191 {
2192 struct adapter *adapter = arg;
2193 if_t ifp = adapter->ifp;
2194
2195 EM_CORE_LOCK_ASSERT(adapter);
2196 EM_TX_LOCK_ASSERT(adapter);
2197
2198 INIT_DEBUGOUT("lem_stop: begin");
2199
2200 lem_disable_intr(adapter);
2201 callout_stop(&adapter->timer);
2202 callout_stop(&adapter->tx_fifo_timer);
2203
2204 /* Tell the stack that the interface is no longer active */
2205 if_setdrvflagbits(ifp, 0, (IFF_DRV_RUNNING | IFF_DRV_OACTIVE));
2206
2207 e1000_reset_hw(&adapter->hw);
2208 if (adapter->hw.mac.type >= e1000_82544)
2209 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2210
2211 e1000_led_off(&adapter->hw);
2212 e1000_cleanup_led(&adapter->hw);
2213 }
2214
2215
2216 /*********************************************************************
2217 *
2218 * Determine hardware revision.
2219 *
2220 **********************************************************************/
2221 static void
lem_identify_hardware(struct adapter * adapter)2222 lem_identify_hardware(struct adapter *adapter)
2223 {
2224 device_t dev = adapter->dev;
2225
2226 /* Make sure our PCI config space has the necessary stuff set */
2227 pci_enable_busmaster(dev);
2228 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2229
2230 /* Save off the information about this board */
2231 adapter->hw.vendor_id = pci_get_vendor(dev);
2232 adapter->hw.device_id = pci_get_device(dev);
2233 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2234 adapter->hw.subsystem_vendor_id =
2235 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2236 adapter->hw.subsystem_device_id =
2237 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2238
2239 /* Do Shared Code Init and Setup */
2240 if (e1000_set_mac_type(&adapter->hw)) {
2241 device_printf(dev, "Setup init failure\n");
2242 return;
2243 }
2244 }
2245
2246 static int
lem_allocate_pci_resources(struct adapter * adapter)2247 lem_allocate_pci_resources(struct adapter *adapter)
2248 {
2249 device_t dev = adapter->dev;
2250 int val, rid, error = E1000_SUCCESS;
2251
2252 rid = PCIR_BAR(0);
2253 adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2254 &rid, RF_ACTIVE);
2255 if (adapter->memory == NULL) {
2256 device_printf(dev, "Unable to allocate bus resource: memory\n");
2257 return (ENXIO);
2258 }
2259 adapter->osdep.mem_bus_space_tag =
2260 rman_get_bustag(adapter->memory);
2261 adapter->osdep.mem_bus_space_handle =
2262 rman_get_bushandle(adapter->memory);
2263 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2264
2265 /* Only older adapters use IO mapping */
2266 if (adapter->hw.mac.type > e1000_82543) {
2267 /* Figure our where our IO BAR is ? */
2268 for (rid = PCIR_BAR(0); rid < PCIR_CIS;) {
2269 val = pci_read_config(dev, rid, 4);
2270 if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) {
2271 adapter->io_rid = rid;
2272 break;
2273 }
2274 rid += 4;
2275 /* check for 64bit BAR */
2276 if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT)
2277 rid += 4;
2278 }
2279 if (rid >= PCIR_CIS) {
2280 device_printf(dev, "Unable to locate IO BAR\n");
2281 return (ENXIO);
2282 }
2283 adapter->ioport = bus_alloc_resource_any(dev,
2284 SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE);
2285 if (adapter->ioport == NULL) {
2286 device_printf(dev, "Unable to allocate bus resource: "
2287 "ioport\n");
2288 return (ENXIO);
2289 }
2290 adapter->hw.io_base = 0;
2291 adapter->osdep.io_bus_space_tag =
2292 rman_get_bustag(adapter->ioport);
2293 adapter->osdep.io_bus_space_handle =
2294 rman_get_bushandle(adapter->ioport);
2295 }
2296
2297 adapter->hw.back = &adapter->osdep;
2298
2299 return (error);
2300 }
2301
2302 /*********************************************************************
2303 *
2304 * Setup the Legacy or MSI Interrupt handler
2305 *
2306 **********************************************************************/
2307 int
lem_allocate_irq(struct adapter * adapter)2308 lem_allocate_irq(struct adapter *adapter)
2309 {
2310 device_t dev = adapter->dev;
2311 int error, rid = 0;
2312
2313 /* Manually turn off all interrupts */
2314 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2315
2316 /* We allocate a single interrupt resource */
2317 adapter->res[0] = bus_alloc_resource_any(dev,
2318 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2319 if (adapter->res[0] == NULL) {
2320 device_printf(dev, "Unable to allocate bus resource: "
2321 "interrupt\n");
2322 return (ENXIO);
2323 }
2324
2325 /* Do Legacy setup? */
2326 if (lem_use_legacy_irq) {
2327 if ((error = bus_setup_intr(dev, adapter->res[0],
2328 INTR_TYPE_NET | INTR_MPSAFE, NULL, lem_intr, adapter,
2329 &adapter->tag[0])) != 0) {
2330 device_printf(dev,
2331 "Failed to register interrupt handler");
2332 return (error);
2333 }
2334 return (0);
2335 }
2336
2337 /*
2338 * Use a Fast interrupt and the associated
2339 * deferred processing contexts.
2340 */
2341 TASK_INIT(&adapter->rxtx_task, 0, lem_handle_rxtx, adapter);
2342 TASK_INIT(&adapter->link_task, 0, lem_handle_link, adapter);
2343 adapter->tq = taskqueue_create_fast("lem_taskq", M_NOWAIT,
2344 taskqueue_thread_enqueue, &adapter->tq);
2345 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
2346 device_get_nameunit(adapter->dev));
2347 if ((error = bus_setup_intr(dev, adapter->res[0],
2348 INTR_TYPE_NET, lem_irq_fast, NULL, adapter,
2349 &adapter->tag[0])) != 0) {
2350 device_printf(dev, "Failed to register fast interrupt "
2351 "handler: %d\n", error);
2352 taskqueue_free(adapter->tq);
2353 adapter->tq = NULL;
2354 return (error);
2355 }
2356
2357 return (0);
2358 }
2359
2360
2361 static void
lem_free_pci_resources(struct adapter * adapter)2362 lem_free_pci_resources(struct adapter *adapter)
2363 {
2364 device_t dev = adapter->dev;
2365
2366
2367 if (adapter->tag[0] != NULL) {
2368 bus_teardown_intr(dev, adapter->res[0],
2369 adapter->tag[0]);
2370 adapter->tag[0] = NULL;
2371 }
2372
2373 if (adapter->res[0] != NULL) {
2374 bus_release_resource(dev, SYS_RES_IRQ,
2375 0, adapter->res[0]);
2376 }
2377
2378 if (adapter->memory != NULL)
2379 bus_release_resource(dev, SYS_RES_MEMORY,
2380 PCIR_BAR(0), adapter->memory);
2381
2382 if (adapter->ioport != NULL)
2383 bus_release_resource(dev, SYS_RES_IOPORT,
2384 adapter->io_rid, adapter->ioport);
2385 }
2386
2387
2388 /*********************************************************************
2389 *
2390 * Initialize the hardware to a configuration
2391 * as specified by the adapter structure.
2392 *
2393 **********************************************************************/
2394 static int
lem_hardware_init(struct adapter * adapter)2395 lem_hardware_init(struct adapter *adapter)
2396 {
2397 device_t dev = adapter->dev;
2398 u16 rx_buffer_size;
2399
2400 INIT_DEBUGOUT("lem_hardware_init: begin");
2401
2402 /* Issue a global reset */
2403 e1000_reset_hw(&adapter->hw);
2404
2405 /* When hardware is reset, fifo_head is also reset */
2406 adapter->tx_fifo_head = 0;
2407
2408 /*
2409 * These parameters control the automatic generation (Tx) and
2410 * response (Rx) to Ethernet PAUSE frames.
2411 * - High water mark should allow for at least two frames to be
2412 * received after sending an XOFF.
2413 * - Low water mark works best when it is very near the high water mark.
2414 * This allows the receiver to restart by sending XON when it has
2415 * drained a bit. Here we use an arbitary value of 1500 which will
2416 * restart after one full frame is pulled from the buffer. There
2417 * could be several smaller frames in the buffer and if so they will
2418 * not trigger the XON until their total number reduces the buffer
2419 * by 1500.
2420 * - The pause time is fairly large at 1000 x 512ns = 512 usec.
2421 */
2422 rx_buffer_size = ((E1000_READ_REG(&adapter->hw, E1000_PBA) &
2423 0xffff) << 10 );
2424
2425 adapter->hw.fc.high_water = rx_buffer_size -
2426 roundup2(adapter->max_frame_size, 1024);
2427 adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500;
2428
2429 adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME;
2430 adapter->hw.fc.send_xon = TRUE;
2431
2432 /* Set Flow control, use the tunable location if sane */
2433 if ((lem_fc_setting >= 0) && (lem_fc_setting < 4))
2434 adapter->hw.fc.requested_mode = lem_fc_setting;
2435 else
2436 adapter->hw.fc.requested_mode = e1000_fc_none;
2437
2438 if (e1000_init_hw(&adapter->hw) < 0) {
2439 device_printf(dev, "Hardware Initialization Failed\n");
2440 return (EIO);
2441 }
2442
2443 e1000_check_for_link(&adapter->hw);
2444
2445 return (0);
2446 }
2447
2448 /*********************************************************************
2449 *
2450 * Setup networking device structure and register an interface.
2451 *
2452 **********************************************************************/
2453 static int
lem_setup_interface(device_t dev,struct adapter * adapter)2454 lem_setup_interface(device_t dev, struct adapter *adapter)
2455 {
2456 if_t ifp;
2457
2458 INIT_DEBUGOUT("lem_setup_interface: begin");
2459
2460 ifp = adapter->ifp = if_gethandle(IFT_ETHER);
2461 if (ifp == (void *)NULL) {
2462 device_printf(dev, "can not allocate ifnet structure\n");
2463 return (-1);
2464 }
2465 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2466 if_setinitfn(ifp, lem_init);
2467 if_setsoftc(ifp, adapter);
2468 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
2469 if_setioctlfn(ifp, lem_ioctl);
2470 if_setstartfn(ifp, lem_start);
2471 if_setgetcounterfn(ifp, lem_get_counter);
2472 if_setsendqlen(ifp, adapter->num_tx_desc - 1);
2473 if_setsendqready(ifp);
2474
2475 ether_ifattach(ifp, adapter->hw.mac.addr);
2476
2477 if_setcapabilities(ifp, 0);
2478
2479 if (adapter->hw.mac.type >= e1000_82543) {
2480 if_setcapabilitiesbit(ifp, IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM, 0);
2481 if_setcapenablebit(ifp, IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM, 0);
2482 }
2483
2484 /*
2485 * Tell the upper layer(s) we support long frames.
2486 */
2487 if_setifheaderlen(ifp, sizeof(struct ether_vlan_header));
2488 if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU, 0);
2489 if_setcapenablebit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU, 0);
2490
2491 /*
2492 ** Dont turn this on by default, if vlans are
2493 ** created on another pseudo device (eg. lagg)
2494 ** then vlan events are not passed thru, breaking
2495 ** operation, but with HW FILTER off it works. If
2496 ** using vlans directly on the em driver you can
2497 ** enable this and get full hardware tag filtering.
2498 */
2499 if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWFILTER, 0);
2500
2501 #ifdef DEVICE_POLLING
2502 if_setcapabilitiesbit(ifp, IFCAP_POLLING, 0);
2503 #endif
2504
2505 /* Enable only WOL MAGIC by default */
2506 if (adapter->wol) {
2507 if_setcapabilitiesbit(ifp, IFCAP_WOL, 0);
2508 if_setcapenablebit(ifp, IFCAP_WOL_MAGIC, 0);
2509 }
2510
2511 /*
2512 * Specify the media types supported by this adapter and register
2513 * callbacks to update media and link information
2514 */
2515 ifmedia_init(&adapter->media, IFM_IMASK,
2516 lem_media_change, lem_media_status);
2517 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
2518 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
2519 u_char fiber_type = IFM_1000_SX; /* default type */
2520
2521 if (adapter->hw.mac.type == e1000_82545)
2522 fiber_type = IFM_1000_LX;
2523 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
2524 0, NULL);
2525 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
2526 } else {
2527 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
2528 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
2529 0, NULL);
2530 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
2531 0, NULL);
2532 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
2533 0, NULL);
2534 if (adapter->hw.phy.type != e1000_phy_ife) {
2535 ifmedia_add(&adapter->media,
2536 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2537 ifmedia_add(&adapter->media,
2538 IFM_ETHER | IFM_1000_T, 0, NULL);
2539 }
2540 }
2541 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2542 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2543 return (0);
2544 }
2545
2546
2547 /*********************************************************************
2548 *
2549 * Workaround for SmartSpeed on 82541 and 82547 controllers
2550 *
2551 **********************************************************************/
2552 static void
lem_smartspeed(struct adapter * adapter)2553 lem_smartspeed(struct adapter *adapter)
2554 {
2555 u16 phy_tmp;
2556
2557 if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) ||
2558 adapter->hw.mac.autoneg == 0 ||
2559 (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0)
2560 return;
2561
2562 if (adapter->smartspeed == 0) {
2563 /* If Master/Slave config fault is asserted twice,
2564 * we assume back-to-back */
2565 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
2566 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
2567 return;
2568 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
2569 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
2570 e1000_read_phy_reg(&adapter->hw,
2571 PHY_1000T_CTRL, &phy_tmp);
2572 if(phy_tmp & CR_1000T_MS_ENABLE) {
2573 phy_tmp &= ~CR_1000T_MS_ENABLE;
2574 e1000_write_phy_reg(&adapter->hw,
2575 PHY_1000T_CTRL, phy_tmp);
2576 adapter->smartspeed++;
2577 if(adapter->hw.mac.autoneg &&
2578 !e1000_copper_link_autoneg(&adapter->hw) &&
2579 !e1000_read_phy_reg(&adapter->hw,
2580 PHY_CONTROL, &phy_tmp)) {
2581 phy_tmp |= (MII_CR_AUTO_NEG_EN |
2582 MII_CR_RESTART_AUTO_NEG);
2583 e1000_write_phy_reg(&adapter->hw,
2584 PHY_CONTROL, phy_tmp);
2585 }
2586 }
2587 }
2588 return;
2589 } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
2590 /* If still no link, perhaps using 2/3 pair cable */
2591 e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
2592 phy_tmp |= CR_1000T_MS_ENABLE;
2593 e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
2594 if(adapter->hw.mac.autoneg &&
2595 !e1000_copper_link_autoneg(&adapter->hw) &&
2596 !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) {
2597 phy_tmp |= (MII_CR_AUTO_NEG_EN |
2598 MII_CR_RESTART_AUTO_NEG);
2599 e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp);
2600 }
2601 }
2602 /* Restart process after EM_SMARTSPEED_MAX iterations */
2603 if(adapter->smartspeed++ == EM_SMARTSPEED_MAX)
2604 adapter->smartspeed = 0;
2605 }
2606
2607
2608 /*
2609 * Manage DMA'able memory.
2610 */
2611 static void
lem_dmamap_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)2612 lem_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
2613 {
2614 if (error)
2615 return;
2616 *(bus_addr_t *) arg = segs[0].ds_addr;
2617 }
2618
2619 static int
lem_dma_malloc(struct adapter * adapter,bus_size_t size,struct em_dma_alloc * dma,int mapflags)2620 lem_dma_malloc(struct adapter *adapter, bus_size_t size,
2621 struct em_dma_alloc *dma, int mapflags)
2622 {
2623 int error;
2624
2625 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
2626 EM_DBA_ALIGN, 0, /* alignment, bounds */
2627 BUS_SPACE_MAXADDR, /* lowaddr */
2628 BUS_SPACE_MAXADDR, /* highaddr */
2629 NULL, NULL, /* filter, filterarg */
2630 size, /* maxsize */
2631 1, /* nsegments */
2632 size, /* maxsegsize */
2633 0, /* flags */
2634 NULL, /* lockfunc */
2635 NULL, /* lockarg */
2636 &dma->dma_tag);
2637 if (error) {
2638 device_printf(adapter->dev,
2639 "%s: bus_dma_tag_create failed: %d\n",
2640 __func__, error);
2641 goto fail_0;
2642 }
2643
2644 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
2645 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
2646 if (error) {
2647 device_printf(adapter->dev,
2648 "%s: bus_dmamem_alloc(%ju) failed: %d\n",
2649 __func__, (uintmax_t)size, error);
2650 goto fail_2;
2651 }
2652
2653 dma->dma_paddr = 0;
2654 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2655 size, lem_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
2656 if (error || dma->dma_paddr == 0) {
2657 device_printf(adapter->dev,
2658 "%s: bus_dmamap_load failed: %d\n",
2659 __func__, error);
2660 goto fail_3;
2661 }
2662
2663 return (0);
2664
2665 fail_3:
2666 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2667 fail_2:
2668 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2669 bus_dma_tag_destroy(dma->dma_tag);
2670 fail_0:
2671 dma->dma_tag = NULL;
2672
2673 return (error);
2674 }
2675
2676 static void
lem_dma_free(struct adapter * adapter,struct em_dma_alloc * dma)2677 lem_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
2678 {
2679 if (dma->dma_tag == NULL)
2680 return;
2681 if (dma->dma_paddr != 0) {
2682 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2683 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2684 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2685 dma->dma_paddr = 0;
2686 }
2687 if (dma->dma_vaddr != NULL) {
2688 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2689 dma->dma_vaddr = NULL;
2690 }
2691 bus_dma_tag_destroy(dma->dma_tag);
2692 dma->dma_tag = NULL;
2693 }
2694
2695
2696 /*********************************************************************
2697 *
2698 * Allocate memory for tx_buffer structures. The tx_buffer stores all
2699 * the information needed to transmit a packet on the wire.
2700 *
2701 **********************************************************************/
2702 static int
lem_allocate_transmit_structures(struct adapter * adapter)2703 lem_allocate_transmit_structures(struct adapter *adapter)
2704 {
2705 device_t dev = adapter->dev;
2706 struct em_buffer *tx_buffer;
2707 int error;
2708
2709 /*
2710 * Create DMA tags for tx descriptors
2711 */
2712 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
2713 1, 0, /* alignment, bounds */
2714 BUS_SPACE_MAXADDR, /* lowaddr */
2715 BUS_SPACE_MAXADDR, /* highaddr */
2716 NULL, NULL, /* filter, filterarg */
2717 MCLBYTES * EM_MAX_SCATTER, /* maxsize */
2718 EM_MAX_SCATTER, /* nsegments */
2719 MCLBYTES, /* maxsegsize */
2720 0, /* flags */
2721 NULL, /* lockfunc */
2722 NULL, /* lockarg */
2723 &adapter->txtag)) != 0) {
2724 device_printf(dev, "Unable to allocate TX DMA tag\n");
2725 goto fail;
2726 }
2727
2728 adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) *
2729 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
2730 if (adapter->tx_buffer_area == NULL) {
2731 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2732 error = ENOMEM;
2733 goto fail;
2734 }
2735
2736 /* Create the descriptor buffer dma maps */
2737 for (int i = 0; i < adapter->num_tx_desc; i++) {
2738 tx_buffer = &adapter->tx_buffer_area[i];
2739 error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map);
2740 if (error != 0) {
2741 device_printf(dev, "Unable to create TX DMA map\n");
2742 goto fail;
2743 }
2744 tx_buffer->next_eop = -1;
2745 }
2746
2747 return (0);
2748 fail:
2749 lem_free_transmit_structures(adapter);
2750 return (error);
2751 }
2752
2753 /*********************************************************************
2754 *
2755 * (Re)Initialize transmit structures.
2756 *
2757 **********************************************************************/
2758 static void
lem_setup_transmit_structures(struct adapter * adapter)2759 lem_setup_transmit_structures(struct adapter *adapter)
2760 {
2761 struct em_buffer *tx_buffer;
2762 #ifdef DEV_NETMAP
2763 /* we are already locked */
2764 struct netmap_adapter *na = netmap_getna(adapter->ifp);
2765 struct netmap_slot *slot = netmap_reset(na, NR_TX, 0, 0);
2766 #endif /* DEV_NETMAP */
2767
2768 /* Clear the old ring contents */
2769 bzero(adapter->tx_desc_base,
2770 (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc);
2771
2772 /* Free any existing TX buffers */
2773 for (int i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
2774 tx_buffer = &adapter->tx_buffer_area[i];
2775 bus_dmamap_sync(adapter->txtag, tx_buffer->map,
2776 BUS_DMASYNC_POSTWRITE);
2777 bus_dmamap_unload(adapter->txtag, tx_buffer->map);
2778 m_freem(tx_buffer->m_head);
2779 tx_buffer->m_head = NULL;
2780 #ifdef DEV_NETMAP
2781 if (slot) {
2782 /* the i-th NIC entry goes to slot si */
2783 int si = netmap_idx_n2k(&na->tx_rings[0], i);
2784 uint64_t paddr;
2785 void *addr;
2786
2787 addr = PNMB(na, slot + si, &paddr);
2788 adapter->tx_desc_base[i].buffer_addr = htole64(paddr);
2789 /* reload the map for netmap mode */
2790 netmap_load_map(na, adapter->txtag, tx_buffer->map, addr);
2791 }
2792 #endif /* DEV_NETMAP */
2793 tx_buffer->next_eop = -1;
2794 }
2795
2796 /* Reset state */
2797 adapter->last_hw_offload = 0;
2798 adapter->next_avail_tx_desc = 0;
2799 adapter->next_tx_to_clean = 0;
2800 adapter->num_tx_desc_avail = adapter->num_tx_desc;
2801
2802 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
2803 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2804
2805 return;
2806 }
2807
2808 /*********************************************************************
2809 *
2810 * Enable transmit unit.
2811 *
2812 **********************************************************************/
2813 static void
lem_initialize_transmit_unit(struct adapter * adapter)2814 lem_initialize_transmit_unit(struct adapter *adapter)
2815 {
2816 u32 tctl, tipg = 0;
2817 u64 bus_addr;
2818
2819 INIT_DEBUGOUT("lem_initialize_transmit_unit: begin");
2820 /* Setup the Base and Length of the Tx Descriptor Ring */
2821 bus_addr = adapter->txdma.dma_paddr;
2822 E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0),
2823 adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
2824 E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0),
2825 (u32)(bus_addr >> 32));
2826 E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0),
2827 (u32)bus_addr);
2828 /* Setup the HW Tx Head and Tail descriptor pointers */
2829 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0);
2830 E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0);
2831
2832 HW_DEBUGOUT2("Base = %x, Length = %x\n",
2833 E1000_READ_REG(&adapter->hw, E1000_TDBAL(0)),
2834 E1000_READ_REG(&adapter->hw, E1000_TDLEN(0)));
2835
2836 /* Set the default values for the Tx Inter Packet Gap timer */
2837 switch (adapter->hw.mac.type) {
2838 case e1000_82542:
2839 tipg = DEFAULT_82542_TIPG_IPGT;
2840 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
2841 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2842 break;
2843 default:
2844 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
2845 (adapter->hw.phy.media_type ==
2846 e1000_media_type_internal_serdes))
2847 tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
2848 else
2849 tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
2850 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
2851 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2852 }
2853
2854 E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
2855 E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value);
2856 if(adapter->hw.mac.type >= e1000_82540)
2857 E1000_WRITE_REG(&adapter->hw, E1000_TADV,
2858 adapter->tx_abs_int_delay.value);
2859
2860 /* Program the Transmit Control Register */
2861 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
2862 tctl &= ~E1000_TCTL_CT;
2863 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
2864 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
2865
2866 /* This write will effectively turn on the transmit unit. */
2867 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
2868
2869 /* Setup Transmit Descriptor Base Settings */
2870 adapter->txd_cmd = E1000_TXD_CMD_IFCS;
2871
2872 if (adapter->tx_int_delay.value > 0)
2873 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
2874 }
2875
2876 /*********************************************************************
2877 *
2878 * Free all transmit related data structures.
2879 *
2880 **********************************************************************/
2881 static void
lem_free_transmit_structures(struct adapter * adapter)2882 lem_free_transmit_structures(struct adapter *adapter)
2883 {
2884 struct em_buffer *tx_buffer;
2885
2886 INIT_DEBUGOUT("free_transmit_structures: begin");
2887
2888 if (adapter->tx_buffer_area != NULL) {
2889 for (int i = 0; i < adapter->num_tx_desc; i++) {
2890 tx_buffer = &adapter->tx_buffer_area[i];
2891 if (tx_buffer->m_head != NULL) {
2892 bus_dmamap_sync(adapter->txtag, tx_buffer->map,
2893 BUS_DMASYNC_POSTWRITE);
2894 bus_dmamap_unload(adapter->txtag,
2895 tx_buffer->map);
2896 m_freem(tx_buffer->m_head);
2897 tx_buffer->m_head = NULL;
2898 } else if (tx_buffer->map != NULL)
2899 bus_dmamap_unload(adapter->txtag,
2900 tx_buffer->map);
2901 if (tx_buffer->map != NULL) {
2902 bus_dmamap_destroy(adapter->txtag,
2903 tx_buffer->map);
2904 tx_buffer->map = NULL;
2905 }
2906 }
2907 }
2908 if (adapter->tx_buffer_area != NULL) {
2909 free(adapter->tx_buffer_area, M_DEVBUF);
2910 adapter->tx_buffer_area = NULL;
2911 }
2912 if (adapter->txtag != NULL) {
2913 bus_dma_tag_destroy(adapter->txtag);
2914 adapter->txtag = NULL;
2915 }
2916 }
2917
2918 /*********************************************************************
2919 *
2920 * The offload context needs to be set when we transfer the first
2921 * packet of a particular protocol (TCP/UDP). This routine has been
2922 * enhanced to deal with inserted VLAN headers, and IPV6 (not complete)
2923 *
2924 * Added back the old method of keeping the current context type
2925 * and not setting if unnecessary, as this is reported to be a
2926 * big performance win. -jfv
2927 **********************************************************************/
2928 static void
lem_transmit_checksum_setup(struct adapter * adapter,struct mbuf * mp,u32 * txd_upper,u32 * txd_lower)2929 lem_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp,
2930 u32 *txd_upper, u32 *txd_lower)
2931 {
2932 struct e1000_context_desc *TXD = NULL;
2933 struct em_buffer *tx_buffer;
2934 struct ether_vlan_header *eh;
2935 struct ip *ip = NULL;
2936 struct ip6_hdr *ip6;
2937 int curr_txd, ehdrlen;
2938 u32 cmd, hdr_len, ip_hlen;
2939 u16 etype;
2940 u8 ipproto;
2941
2942
2943 cmd = hdr_len = ipproto = 0;
2944 *txd_upper = *txd_lower = 0;
2945 curr_txd = adapter->next_avail_tx_desc;
2946
2947 /*
2948 * Determine where frame payload starts.
2949 * Jump over vlan headers if already present,
2950 * helpful for QinQ too.
2951 */
2952 eh = mtod(mp, struct ether_vlan_header *);
2953 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
2954 etype = ntohs(eh->evl_proto);
2955 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
2956 } else {
2957 etype = ntohs(eh->evl_encap_proto);
2958 ehdrlen = ETHER_HDR_LEN;
2959 }
2960
2961 /*
2962 * We only support TCP/UDP for IPv4 and IPv6 for the moment.
2963 * TODO: Support SCTP too when it hits the tree.
2964 */
2965 switch (etype) {
2966 case ETHERTYPE_IP:
2967 ip = (struct ip *)(mp->m_data + ehdrlen);
2968 ip_hlen = ip->ip_hl << 2;
2969
2970 /* Setup of IP header checksum. */
2971 if (mp->m_pkthdr.csum_flags & CSUM_IP) {
2972 /*
2973 * Start offset for header checksum calculation.
2974 * End offset for header checksum calculation.
2975 * Offset of place to put the checksum.
2976 */
2977 TXD = (struct e1000_context_desc *)
2978 &adapter->tx_desc_base[curr_txd];
2979 TXD->lower_setup.ip_fields.ipcss = ehdrlen;
2980 TXD->lower_setup.ip_fields.ipcse =
2981 htole16(ehdrlen + ip_hlen);
2982 TXD->lower_setup.ip_fields.ipcso =
2983 ehdrlen + offsetof(struct ip, ip_sum);
2984 cmd |= E1000_TXD_CMD_IP;
2985 *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
2986 }
2987
2988 hdr_len = ehdrlen + ip_hlen;
2989 ipproto = ip->ip_p;
2990
2991 break;
2992 case ETHERTYPE_IPV6:
2993 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
2994 ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */
2995
2996 /* IPv6 doesn't have a header checksum. */
2997
2998 hdr_len = ehdrlen + ip_hlen;
2999 ipproto = ip6->ip6_nxt;
3000 break;
3001
3002 default:
3003 return;
3004 }
3005
3006 switch (ipproto) {
3007 case IPPROTO_TCP:
3008 if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
3009 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3010 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3011 /* no need for context if already set */
3012 if (adapter->last_hw_offload == CSUM_TCP)
3013 return;
3014 adapter->last_hw_offload = CSUM_TCP;
3015 /*
3016 * Start offset for payload checksum calculation.
3017 * End offset for payload checksum calculation.
3018 * Offset of place to put the checksum.
3019 */
3020 TXD = (struct e1000_context_desc *)
3021 &adapter->tx_desc_base[curr_txd];
3022 TXD->upper_setup.tcp_fields.tucss = hdr_len;
3023 TXD->upper_setup.tcp_fields.tucse = htole16(0);
3024 TXD->upper_setup.tcp_fields.tucso =
3025 hdr_len + offsetof(struct tcphdr, th_sum);
3026 cmd |= E1000_TXD_CMD_TCP;
3027 }
3028 break;
3029 case IPPROTO_UDP:
3030 {
3031 if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
3032 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3033 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3034 /* no need for context if already set */
3035 if (adapter->last_hw_offload == CSUM_UDP)
3036 return;
3037 adapter->last_hw_offload = CSUM_UDP;
3038 /*
3039 * Start offset for header checksum calculation.
3040 * End offset for header checksum calculation.
3041 * Offset of place to put the checksum.
3042 */
3043 TXD = (struct e1000_context_desc *)
3044 &adapter->tx_desc_base[curr_txd];
3045 TXD->upper_setup.tcp_fields.tucss = hdr_len;
3046 TXD->upper_setup.tcp_fields.tucse = htole16(0);
3047 TXD->upper_setup.tcp_fields.tucso =
3048 hdr_len + offsetof(struct udphdr, uh_sum);
3049 }
3050 /* Fall Thru */
3051 }
3052 default:
3053 break;
3054 }
3055
3056 if (TXD == NULL)
3057 return;
3058 TXD->tcp_seg_setup.data = htole32(0);
3059 TXD->cmd_and_length =
3060 htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd);
3061 tx_buffer = &adapter->tx_buffer_area[curr_txd];
3062 tx_buffer->m_head = NULL;
3063 tx_buffer->next_eop = -1;
3064
3065 if (++curr_txd == adapter->num_tx_desc)
3066 curr_txd = 0;
3067
3068 adapter->num_tx_desc_avail--;
3069 adapter->next_avail_tx_desc = curr_txd;
3070 }
3071
3072
3073 /**********************************************************************
3074 *
3075 * Examine each tx_buffer in the used queue. If the hardware is done
3076 * processing the packet then free associated resources. The
3077 * tx_buffer is put back on the free queue.
3078 *
3079 **********************************************************************/
3080 static void
lem_txeof(struct adapter * adapter)3081 lem_txeof(struct adapter *adapter)
3082 {
3083 int first, last, done, num_avail;
3084 struct em_buffer *tx_buffer;
3085 struct e1000_tx_desc *tx_desc, *eop_desc;
3086 if_t ifp = adapter->ifp;
3087
3088 EM_TX_LOCK_ASSERT(adapter);
3089
3090 #ifdef DEV_NETMAP
3091 if (netmap_tx_irq(ifp, 0))
3092 return;
3093 #endif /* DEV_NETMAP */
3094 if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
3095 return;
3096
3097 num_avail = adapter->num_tx_desc_avail;
3098 first = adapter->next_tx_to_clean;
3099 tx_desc = &adapter->tx_desc_base[first];
3100 tx_buffer = &adapter->tx_buffer_area[first];
3101 last = tx_buffer->next_eop;
3102 eop_desc = &adapter->tx_desc_base[last];
3103
3104 /*
3105 * What this does is get the index of the
3106 * first descriptor AFTER the EOP of the
3107 * first packet, that way we can do the
3108 * simple comparison on the inner while loop.
3109 */
3110 if (++last == adapter->num_tx_desc)
3111 last = 0;
3112 done = last;
3113
3114 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
3115 BUS_DMASYNC_POSTREAD);
3116
3117 while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
3118 /* We clean the range of the packet */
3119 while (first != done) {
3120 tx_desc->upper.data = 0;
3121 tx_desc->lower.data = 0;
3122 tx_desc->buffer_addr = 0;
3123 ++num_avail;
3124
3125 if (tx_buffer->m_head) {
3126 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
3127 bus_dmamap_sync(adapter->txtag,
3128 tx_buffer->map,
3129 BUS_DMASYNC_POSTWRITE);
3130 bus_dmamap_unload(adapter->txtag,
3131 tx_buffer->map);
3132
3133 m_freem(tx_buffer->m_head);
3134 tx_buffer->m_head = NULL;
3135 }
3136 tx_buffer->next_eop = -1;
3137 adapter->watchdog_time = ticks;
3138
3139 if (++first == adapter->num_tx_desc)
3140 first = 0;
3141
3142 tx_buffer = &adapter->tx_buffer_area[first];
3143 tx_desc = &adapter->tx_desc_base[first];
3144 }
3145 /* See if we can continue to the next packet */
3146 last = tx_buffer->next_eop;
3147 if (last != -1) {
3148 eop_desc = &adapter->tx_desc_base[last];
3149 /* Get new done point */
3150 if (++last == adapter->num_tx_desc) last = 0;
3151 done = last;
3152 } else
3153 break;
3154 }
3155 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
3156 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3157
3158 adapter->next_tx_to_clean = first;
3159 adapter->num_tx_desc_avail = num_avail;
3160
3161 #ifdef NIC_SEND_COMBINING
3162 if ((adapter->shadow_tdt & MIT_PENDING_TDT) == MIT_PENDING_TDT) {
3163 /* a tdt write is pending, do it */
3164 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0),
3165 0xffff & adapter->shadow_tdt);
3166 adapter->shadow_tdt = MIT_PENDING_INT;
3167 } else {
3168 adapter->shadow_tdt = 0; // disable
3169 }
3170 #endif /* NIC_SEND_COMBINING */
3171 /*
3172 * If we have enough room, clear IFF_DRV_OACTIVE to
3173 * tell the stack that it is OK to send packets.
3174 * If there are no pending descriptors, clear the watchdog.
3175 */
3176 if (adapter->num_tx_desc_avail > EM_TX_CLEANUP_THRESHOLD) {
3177 if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE);
3178 #ifdef NIC_PARAVIRT
3179 if (adapter->csb) { // XXX also csb_on ?
3180 adapter->csb->guest_need_txkick = 2; /* acked */
3181 // XXX memory barrier
3182 }
3183 #endif /* NIC_PARAVIRT */
3184 if (adapter->num_tx_desc_avail == adapter->num_tx_desc) {
3185 adapter->watchdog_check = FALSE;
3186 return;
3187 }
3188 }
3189 }
3190
3191 /*********************************************************************
3192 *
3193 * When Link is lost sometimes there is work still in the TX ring
3194 * which may result in a watchdog, rather than allow that we do an
3195 * attempted cleanup and then reinit here. Note that this has been
3196 * seens mostly with fiber adapters.
3197 *
3198 **********************************************************************/
3199 static void
lem_tx_purge(struct adapter * adapter)3200 lem_tx_purge(struct adapter *adapter)
3201 {
3202 if ((!adapter->link_active) && (adapter->watchdog_check)) {
3203 EM_TX_LOCK(adapter);
3204 lem_txeof(adapter);
3205 EM_TX_UNLOCK(adapter);
3206 if (adapter->watchdog_check) /* Still outstanding? */
3207 lem_init_locked(adapter);
3208 }
3209 }
3210
3211 /*********************************************************************
3212 *
3213 * Get a buffer from system mbuf buffer pool.
3214 *
3215 **********************************************************************/
3216 static int
lem_get_buf(struct adapter * adapter,int i)3217 lem_get_buf(struct adapter *adapter, int i)
3218 {
3219 struct mbuf *m;
3220 bus_dma_segment_t segs[1];
3221 bus_dmamap_t map;
3222 struct em_buffer *rx_buffer;
3223 int error, nsegs;
3224
3225 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
3226 if (m == NULL) {
3227 adapter->mbuf_cluster_failed++;
3228 return (ENOBUFS);
3229 }
3230 m->m_len = m->m_pkthdr.len = MCLBYTES;
3231
3232 if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
3233 m_adj(m, ETHER_ALIGN);
3234
3235 /*
3236 * Using memory from the mbuf cluster pool, invoke the
3237 * bus_dma machinery to arrange the memory mapping.
3238 */
3239 error = bus_dmamap_load_mbuf_sg(adapter->rxtag,
3240 adapter->rx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT);
3241 if (error != 0) {
3242 m_free(m);
3243 return (error);
3244 }
3245
3246 /* If nsegs is wrong then the stack is corrupt. */
3247 KASSERT(nsegs == 1, ("Too many segments returned!"));
3248
3249 rx_buffer = &adapter->rx_buffer_area[i];
3250 if (rx_buffer->m_head != NULL)
3251 bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
3252
3253 map = rx_buffer->map;
3254 rx_buffer->map = adapter->rx_sparemap;
3255 adapter->rx_sparemap = map;
3256 bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD);
3257 rx_buffer->m_head = m;
3258
3259 adapter->rx_desc_base[i].buffer_addr = htole64(segs[0].ds_addr);
3260 return (0);
3261 }
3262
3263 /*********************************************************************
3264 *
3265 * Allocate memory for rx_buffer structures. Since we use one
3266 * rx_buffer per received packet, the maximum number of rx_buffer's
3267 * that we'll need is equal to the number of receive descriptors
3268 * that we've allocated.
3269 *
3270 **********************************************************************/
3271 static int
lem_allocate_receive_structures(struct adapter * adapter)3272 lem_allocate_receive_structures(struct adapter *adapter)
3273 {
3274 device_t dev = adapter->dev;
3275 struct em_buffer *rx_buffer;
3276 int i, error;
3277
3278 adapter->rx_buffer_area = malloc(sizeof(struct em_buffer) *
3279 adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
3280 if (adapter->rx_buffer_area == NULL) {
3281 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3282 return (ENOMEM);
3283 }
3284
3285 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
3286 1, 0, /* alignment, bounds */
3287 BUS_SPACE_MAXADDR, /* lowaddr */
3288 BUS_SPACE_MAXADDR, /* highaddr */
3289 NULL, NULL, /* filter, filterarg */
3290 MCLBYTES, /* maxsize */
3291 1, /* nsegments */
3292 MCLBYTES, /* maxsegsize */
3293 0, /* flags */
3294 NULL, /* lockfunc */
3295 NULL, /* lockarg */
3296 &adapter->rxtag);
3297 if (error) {
3298 device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
3299 __func__, error);
3300 goto fail;
3301 }
3302
3303 /* Create the spare map (used by getbuf) */
3304 error = bus_dmamap_create(adapter->rxtag, 0, &adapter->rx_sparemap);
3305 if (error) {
3306 device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
3307 __func__, error);
3308 goto fail;
3309 }
3310
3311 rx_buffer = adapter->rx_buffer_area;
3312 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
3313 error = bus_dmamap_create(adapter->rxtag, 0, &rx_buffer->map);
3314 if (error) {
3315 device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
3316 __func__, error);
3317 goto fail;
3318 }
3319 }
3320
3321 return (0);
3322
3323 fail:
3324 lem_free_receive_structures(adapter);
3325 return (error);
3326 }
3327
3328 /*********************************************************************
3329 *
3330 * (Re)initialize receive structures.
3331 *
3332 **********************************************************************/
3333 static int
lem_setup_receive_structures(struct adapter * adapter)3334 lem_setup_receive_structures(struct adapter *adapter)
3335 {
3336 struct em_buffer *rx_buffer;
3337 int i, error;
3338 #ifdef DEV_NETMAP
3339 /* we are already under lock */
3340 struct netmap_adapter *na = netmap_getna(adapter->ifp);
3341 struct netmap_slot *slot = netmap_reset(na, NR_RX, 0, 0);
3342 #endif
3343
3344 /* Reset descriptor ring */
3345 bzero(adapter->rx_desc_base,
3346 (sizeof(struct e1000_rx_desc)) * adapter->num_rx_desc);
3347
3348 /* Free current RX buffers. */
3349 rx_buffer = adapter->rx_buffer_area;
3350 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
3351 if (rx_buffer->m_head != NULL) {
3352 bus_dmamap_sync(adapter->rxtag, rx_buffer->map,
3353 BUS_DMASYNC_POSTREAD);
3354 bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
3355 m_freem(rx_buffer->m_head);
3356 rx_buffer->m_head = NULL;
3357 }
3358 }
3359
3360 /* Allocate new ones. */
3361 for (i = 0; i < adapter->num_rx_desc; i++) {
3362 #ifdef DEV_NETMAP
3363 if (slot) {
3364 /* the i-th NIC entry goes to slot si */
3365 int si = netmap_idx_n2k(&na->rx_rings[0], i);
3366 uint64_t paddr;
3367 void *addr;
3368
3369 addr = PNMB(na, slot + si, &paddr);
3370 netmap_load_map(na, adapter->rxtag, rx_buffer->map, addr);
3371 /* Update descriptor */
3372 adapter->rx_desc_base[i].buffer_addr = htole64(paddr);
3373 continue;
3374 }
3375 #endif /* DEV_NETMAP */
3376 error = lem_get_buf(adapter, i);
3377 if (error)
3378 return (error);
3379 }
3380
3381 /* Setup our descriptor pointers */
3382 adapter->next_rx_desc_to_check = 0;
3383 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
3384 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3385
3386 return (0);
3387 }
3388
3389 /*********************************************************************
3390 *
3391 * Enable receive unit.
3392 *
3393 **********************************************************************/
3394
3395 static void
lem_initialize_receive_unit(struct adapter * adapter)3396 lem_initialize_receive_unit(struct adapter *adapter)
3397 {
3398 if_t ifp = adapter->ifp;
3399 u64 bus_addr;
3400 u32 rctl, rxcsum;
3401
3402 INIT_DEBUGOUT("lem_initialize_receive_unit: begin");
3403
3404 /*
3405 * Make sure receives are disabled while setting
3406 * up the descriptor ring
3407 */
3408 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
3409 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
3410
3411 if (adapter->hw.mac.type >= e1000_82540) {
3412 E1000_WRITE_REG(&adapter->hw, E1000_RADV,
3413 adapter->rx_abs_int_delay.value);
3414 /*
3415 * Set the interrupt throttling rate. Value is calculated
3416 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
3417 */
3418 E1000_WRITE_REG(&adapter->hw, E1000_ITR, DEFAULT_ITR);
3419 }
3420
3421 /* Setup the Base and Length of the Rx Descriptor Ring */
3422 bus_addr = adapter->rxdma.dma_paddr;
3423 E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(0),
3424 adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
3425 E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(0),
3426 (u32)(bus_addr >> 32));
3427 E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(0),
3428 (u32)bus_addr);
3429
3430 /* Setup the Receive Control Register */
3431 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
3432 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
3433 E1000_RCTL_RDMTS_HALF |
3434 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
3435
3436 /* Make sure VLAN Filters are off */
3437 rctl &= ~E1000_RCTL_VFE;
3438
3439 if (e1000_tbi_sbp_enabled_82543(&adapter->hw))
3440 rctl |= E1000_RCTL_SBP;
3441 else
3442 rctl &= ~E1000_RCTL_SBP;
3443
3444 switch (adapter->rx_buffer_len) {
3445 default:
3446 case 2048:
3447 rctl |= E1000_RCTL_SZ_2048;
3448 break;
3449 case 4096:
3450 rctl |= E1000_RCTL_SZ_4096 |
3451 E1000_RCTL_BSEX | E1000_RCTL_LPE;
3452 break;
3453 case 8192:
3454 rctl |= E1000_RCTL_SZ_8192 |
3455 E1000_RCTL_BSEX | E1000_RCTL_LPE;
3456 break;
3457 case 16384:
3458 rctl |= E1000_RCTL_SZ_16384 |
3459 E1000_RCTL_BSEX | E1000_RCTL_LPE;
3460 break;
3461 }
3462
3463 if (if_getmtu(ifp) > ETHERMTU)
3464 rctl |= E1000_RCTL_LPE;
3465 else
3466 rctl &= ~E1000_RCTL_LPE;
3467
3468 /* Enable 82543 Receive Checksum Offload for TCP and UDP */
3469 if ((adapter->hw.mac.type >= e1000_82543) &&
3470 (if_getcapenable(ifp) & IFCAP_RXCSUM)) {
3471 rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM);
3472 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
3473 E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum);
3474 }
3475
3476 /* Enable Receives */
3477 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
3478
3479 /*
3480 * Setup the HW Rx Head and
3481 * Tail Descriptor Pointers
3482 */
3483 E1000_WRITE_REG(&adapter->hw, E1000_RDH(0), 0);
3484 rctl = adapter->num_rx_desc - 1; /* default RDT value */
3485 #ifdef DEV_NETMAP
3486 /* preserve buffers already made available to clients */
3487 if (if_getcapenable(ifp) & IFCAP_NETMAP) {
3488 struct netmap_adapter *na = netmap_getna(adapter->ifp);
3489 rctl -= nm_kr_rxspace(&na->rx_rings[0]);
3490 }
3491 #endif /* DEV_NETMAP */
3492 E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), rctl);
3493
3494 return;
3495 }
3496
3497 /*********************************************************************
3498 *
3499 * Free receive related data structures.
3500 *
3501 **********************************************************************/
3502 static void
lem_free_receive_structures(struct adapter * adapter)3503 lem_free_receive_structures(struct adapter *adapter)
3504 {
3505 struct em_buffer *rx_buffer;
3506 int i;
3507
3508 INIT_DEBUGOUT("free_receive_structures: begin");
3509
3510 if (adapter->rx_sparemap) {
3511 bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap);
3512 adapter->rx_sparemap = NULL;
3513 }
3514
3515 /* Cleanup any existing buffers */
3516 if (adapter->rx_buffer_area != NULL) {
3517 rx_buffer = adapter->rx_buffer_area;
3518 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
3519 if (rx_buffer->m_head != NULL) {
3520 bus_dmamap_sync(adapter->rxtag, rx_buffer->map,
3521 BUS_DMASYNC_POSTREAD);
3522 bus_dmamap_unload(adapter->rxtag,
3523 rx_buffer->map);
3524 m_freem(rx_buffer->m_head);
3525 rx_buffer->m_head = NULL;
3526 } else if (rx_buffer->map != NULL)
3527 bus_dmamap_unload(adapter->rxtag,
3528 rx_buffer->map);
3529 if (rx_buffer->map != NULL) {
3530 bus_dmamap_destroy(adapter->rxtag,
3531 rx_buffer->map);
3532 rx_buffer->map = NULL;
3533 }
3534 }
3535 }
3536
3537 if (adapter->rx_buffer_area != NULL) {
3538 free(adapter->rx_buffer_area, M_DEVBUF);
3539 adapter->rx_buffer_area = NULL;
3540 }
3541
3542 if (adapter->rxtag != NULL) {
3543 bus_dma_tag_destroy(adapter->rxtag);
3544 adapter->rxtag = NULL;
3545 }
3546 }
3547
3548 /*********************************************************************
3549 *
3550 * This routine executes in interrupt context. It replenishes
3551 * the mbufs in the descriptor and sends data which has been
3552 * dma'ed into host memory to upper layer.
3553 *
3554 * We loop at most count times if count is > 0, or until done if
3555 * count < 0.
3556 *
3557 * For polling we also now return the number of cleaned packets
3558 *********************************************************************/
3559 static bool
lem_rxeof(struct adapter * adapter,int count,int * done)3560 lem_rxeof(struct adapter *adapter, int count, int *done)
3561 {
3562 if_t ifp = adapter->ifp;
3563 struct mbuf *mp;
3564 u8 status = 0, accept_frame = 0, eop = 0;
3565 u16 len, desc_len, prev_len_adj;
3566 int i, rx_sent = 0;
3567 struct e1000_rx_desc *current_desc;
3568
3569 #ifdef BATCH_DISPATCH
3570 struct mbuf *mh = NULL, *mt = NULL;
3571 #endif /* BATCH_DISPATCH */
3572 #ifdef NIC_PARAVIRT
3573 int retries = 0;
3574 struct paravirt_csb* csb = adapter->csb;
3575 int csb_mode = csb && csb->guest_csb_on;
3576
3577 //ND("clear guest_rxkick at %d", adapter->next_rx_desc_to_check);
3578 if (csb_mode && csb->guest_need_rxkick)
3579 csb->guest_need_rxkick = 0;
3580 #endif /* NIC_PARAVIRT */
3581 EM_RX_LOCK(adapter);
3582
3583 #ifdef BATCH_DISPATCH
3584 batch_again:
3585 #endif /* BATCH_DISPATCH */
3586 i = adapter->next_rx_desc_to_check;
3587 current_desc = &adapter->rx_desc_base[i];
3588 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
3589 BUS_DMASYNC_POSTREAD);
3590
3591 #ifdef DEV_NETMAP
3592 if (netmap_rx_irq(ifp, 0, &rx_sent)) {
3593 EM_RX_UNLOCK(adapter);
3594 return (FALSE);
3595 }
3596 #endif /* DEV_NETMAP */
3597
3598 #if 1 // XXX optimization ?
3599 if (!((current_desc->status) & E1000_RXD_STAT_DD)) {
3600 if (done != NULL)
3601 *done = rx_sent;
3602 EM_RX_UNLOCK(adapter);
3603 return (FALSE);
3604 }
3605 #endif /* 0 */
3606
3607 while (count != 0 && if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
3608 struct mbuf *m = NULL;
3609
3610 status = current_desc->status;
3611 if ((status & E1000_RXD_STAT_DD) == 0) {
3612 #ifdef NIC_PARAVIRT
3613 if (csb_mode) {
3614 /* buffer not ready yet. Retry a few times before giving up */
3615 if (++retries <= adapter->rx_retries) {
3616 continue;
3617 }
3618 if (csb->guest_need_rxkick == 0) {
3619 // ND("set guest_rxkick at %d", adapter->next_rx_desc_to_check);
3620 csb->guest_need_rxkick = 1;
3621 // XXX memory barrier, status volatile ?
3622 continue; /* double check */
3623 }
3624 }
3625 /* no buffer ready, give up */
3626 #endif /* NIC_PARAVIRT */
3627 break;
3628 }
3629 #ifdef NIC_PARAVIRT
3630 if (csb_mode) {
3631 if (csb->guest_need_rxkick)
3632 // ND("clear again guest_rxkick at %d", adapter->next_rx_desc_to_check);
3633 csb->guest_need_rxkick = 0;
3634 retries = 0;
3635 }
3636 #endif /* NIC_PARAVIRT */
3637
3638 mp = adapter->rx_buffer_area[i].m_head;
3639 /*
3640 * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT
3641 * needs to access the last received byte in the mbuf.
3642 */
3643 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map,
3644 BUS_DMASYNC_POSTREAD);
3645
3646 accept_frame = 1;
3647 prev_len_adj = 0;
3648 desc_len = le16toh(current_desc->length);
3649 if (status & E1000_RXD_STAT_EOP) {
3650 count--;
3651 eop = 1;
3652 if (desc_len < ETHER_CRC_LEN) {
3653 len = 0;
3654 prev_len_adj = ETHER_CRC_LEN - desc_len;
3655 } else
3656 len = desc_len - ETHER_CRC_LEN;
3657 } else {
3658 eop = 0;
3659 len = desc_len;
3660 }
3661
3662 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
3663 u8 last_byte;
3664 u32 pkt_len = desc_len;
3665
3666 if (adapter->fmp != NULL)
3667 pkt_len += adapter->fmp->m_pkthdr.len;
3668
3669 last_byte = *(mtod(mp, caddr_t) + desc_len - 1);
3670 if (TBI_ACCEPT(&adapter->hw, status,
3671 current_desc->errors, pkt_len, last_byte,
3672 adapter->min_frame_size, adapter->max_frame_size)) {
3673 e1000_tbi_adjust_stats_82543(&adapter->hw,
3674 &adapter->stats, pkt_len,
3675 adapter->hw.mac.addr,
3676 adapter->max_frame_size);
3677 if (len > 0)
3678 len--;
3679 } else
3680 accept_frame = 0;
3681 }
3682
3683 if (accept_frame) {
3684 if (lem_get_buf(adapter, i) != 0) {
3685 if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1);
3686 goto discard;
3687 }
3688
3689 /* Assign correct length to the current fragment */
3690 mp->m_len = len;
3691
3692 if (adapter->fmp == NULL) {
3693 mp->m_pkthdr.len = len;
3694 adapter->fmp = mp; /* Store the first mbuf */
3695 adapter->lmp = mp;
3696 } else {
3697 /* Chain mbuf's together */
3698 mp->m_flags &= ~M_PKTHDR;
3699 /*
3700 * Adjust length of previous mbuf in chain if
3701 * we received less than 4 bytes in the last
3702 * descriptor.
3703 */
3704 if (prev_len_adj > 0) {
3705 adapter->lmp->m_len -= prev_len_adj;
3706 adapter->fmp->m_pkthdr.len -=
3707 prev_len_adj;
3708 }
3709 adapter->lmp->m_next = mp;
3710 adapter->lmp = adapter->lmp->m_next;
3711 adapter->fmp->m_pkthdr.len += len;
3712 }
3713
3714 if (eop) {
3715 if_setrcvif(adapter->fmp, ifp);
3716 if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
3717 lem_receive_checksum(adapter, current_desc,
3718 adapter->fmp);
3719 #ifndef __NO_STRICT_ALIGNMENT
3720 if (adapter->max_frame_size >
3721 (MCLBYTES - ETHER_ALIGN) &&
3722 lem_fixup_rx(adapter) != 0)
3723 goto skip;
3724 #endif
3725 if (status & E1000_RXD_STAT_VP) {
3726 adapter->fmp->m_pkthdr.ether_vtag =
3727 le16toh(current_desc->special);
3728 adapter->fmp->m_flags |= M_VLANTAG;
3729 }
3730 #ifndef __NO_STRICT_ALIGNMENT
3731 skip:
3732 #endif
3733 m = adapter->fmp;
3734 adapter->fmp = NULL;
3735 adapter->lmp = NULL;
3736 }
3737 } else {
3738 adapter->dropped_pkts++;
3739 discard:
3740 /* Reuse loaded DMA map and just update mbuf chain */
3741 mp = adapter->rx_buffer_area[i].m_head;
3742 mp->m_len = mp->m_pkthdr.len = MCLBYTES;
3743 mp->m_data = mp->m_ext.ext_buf;
3744 mp->m_next = NULL;
3745 if (adapter->max_frame_size <=
3746 (MCLBYTES - ETHER_ALIGN))
3747 m_adj(mp, ETHER_ALIGN);
3748 if (adapter->fmp != NULL) {
3749 m_freem(adapter->fmp);
3750 adapter->fmp = NULL;
3751 adapter->lmp = NULL;
3752 }
3753 m = NULL;
3754 }
3755
3756 /* Zero out the receive descriptors status. */
3757 current_desc->status = 0;
3758 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
3759 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3760
3761 #ifdef NIC_PARAVIRT
3762 if (csb_mode) {
3763 /* the buffer at i has been already replaced by lem_get_buf()
3764 * so it is safe to set guest_rdt = i and possibly send a kick.
3765 * XXX see if we can optimize it later.
3766 */
3767 csb->guest_rdt = i;
3768 // XXX memory barrier
3769 if (i == csb->host_rxkick_at)
3770 E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i);
3771 }
3772 #endif /* NIC_PARAVIRT */
3773 /* Advance our pointers to the next descriptor. */
3774 if (++i == adapter->num_rx_desc)
3775 i = 0;
3776 /* Call into the stack */
3777 if (m != NULL) {
3778 #ifdef BATCH_DISPATCH
3779 if (adapter->batch_enable) {
3780 if (mh == NULL)
3781 mh = mt = m;
3782 else
3783 mt->m_nextpkt = m;
3784 mt = m;
3785 m->m_nextpkt = NULL;
3786 rx_sent++;
3787 current_desc = &adapter->rx_desc_base[i];
3788 continue;
3789 }
3790 #endif /* BATCH_DISPATCH */
3791 adapter->next_rx_desc_to_check = i;
3792 EM_RX_UNLOCK(adapter);
3793 if_input(ifp, m);
3794 EM_RX_LOCK(adapter);
3795 rx_sent++;
3796 i = adapter->next_rx_desc_to_check;
3797 }
3798 current_desc = &adapter->rx_desc_base[i];
3799 }
3800 adapter->next_rx_desc_to_check = i;
3801 #ifdef BATCH_DISPATCH
3802 if (mh) {
3803 EM_RX_UNLOCK(adapter);
3804 while ( (mt = mh) != NULL) {
3805 mh = mh->m_nextpkt;
3806 mt->m_nextpkt = NULL;
3807 if_input(ifp, mt);
3808 }
3809 EM_RX_LOCK(adapter);
3810 i = adapter->next_rx_desc_to_check; /* in case of interrupts */
3811 if (count > 0)
3812 goto batch_again;
3813 }
3814 #endif /* BATCH_DISPATCH */
3815
3816 /* Advance the E1000's Receive Queue #0 "Tail Pointer". */
3817 if (--i < 0)
3818 i = adapter->num_rx_desc - 1;
3819 #ifdef NIC_PARAVIRT
3820 if (!csb_mode) /* filter out writes */
3821 #endif /* NIC_PARAVIRT */
3822 E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i);
3823 if (done != NULL)
3824 *done = rx_sent;
3825 EM_RX_UNLOCK(adapter);
3826 return ((status & E1000_RXD_STAT_DD) ? TRUE : FALSE);
3827 }
3828
3829 #ifndef __NO_STRICT_ALIGNMENT
3830 /*
3831 * When jumbo frames are enabled we should realign entire payload on
3832 * architecures with strict alignment. This is serious design mistake of 8254x
3833 * as it nullifies DMA operations. 8254x just allows RX buffer size to be
3834 * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its
3835 * payload. On architecures without strict alignment restrictions 8254x still
3836 * performs unaligned memory access which would reduce the performance too.
3837 * To avoid copying over an entire frame to align, we allocate a new mbuf and
3838 * copy ethernet header to the new mbuf. The new mbuf is prepended into the
3839 * existing mbuf chain.
3840 *
3841 * Be aware, best performance of the 8254x is achived only when jumbo frame is
3842 * not used at all on architectures with strict alignment.
3843 */
3844 static int
lem_fixup_rx(struct adapter * adapter)3845 lem_fixup_rx(struct adapter *adapter)
3846 {
3847 struct mbuf *m, *n;
3848 int error;
3849
3850 error = 0;
3851 m = adapter->fmp;
3852 if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
3853 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
3854 m->m_data += ETHER_HDR_LEN;
3855 } else {
3856 MGETHDR(n, M_NOWAIT, MT_DATA);
3857 if (n != NULL) {
3858 bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
3859 m->m_data += ETHER_HDR_LEN;
3860 m->m_len -= ETHER_HDR_LEN;
3861 n->m_len = ETHER_HDR_LEN;
3862 M_MOVE_PKTHDR(n, m);
3863 n->m_next = m;
3864 adapter->fmp = n;
3865 } else {
3866 adapter->dropped_pkts++;
3867 m_freem(adapter->fmp);
3868 adapter->fmp = NULL;
3869 error = ENOMEM;
3870 }
3871 }
3872
3873 return (error);
3874 }
3875 #endif
3876
3877 /*********************************************************************
3878 *
3879 * Verify that the hardware indicated that the checksum is valid.
3880 * Inform the stack about the status of checksum so that stack
3881 * doesn't spend time verifying the checksum.
3882 *
3883 *********************************************************************/
3884 static void
lem_receive_checksum(struct adapter * adapter,struct e1000_rx_desc * rx_desc,struct mbuf * mp)3885 lem_receive_checksum(struct adapter *adapter,
3886 struct e1000_rx_desc *rx_desc, struct mbuf *mp)
3887 {
3888 /* 82543 or newer only */
3889 if ((adapter->hw.mac.type < e1000_82543) ||
3890 /* Ignore Checksum bit is set */
3891 (rx_desc->status & E1000_RXD_STAT_IXSM)) {
3892 mp->m_pkthdr.csum_flags = 0;
3893 return;
3894 }
3895
3896 if (rx_desc->status & E1000_RXD_STAT_IPCS) {
3897 /* Did it pass? */
3898 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
3899 /* IP Checksum Good */
3900 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
3901 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
3902
3903 } else {
3904 mp->m_pkthdr.csum_flags = 0;
3905 }
3906 }
3907
3908 if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
3909 /* Did it pass? */
3910 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
3911 mp->m_pkthdr.csum_flags |=
3912 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
3913 mp->m_pkthdr.csum_data = htons(0xffff);
3914 }
3915 }
3916 }
3917
3918 /*
3919 * This routine is run via an vlan
3920 * config EVENT
3921 */
3922 static void
lem_register_vlan(void * arg,if_t ifp,u16 vtag)3923 lem_register_vlan(void *arg, if_t ifp, u16 vtag)
3924 {
3925 struct adapter *adapter = if_getsoftc(ifp);
3926 u32 index, bit;
3927
3928 if (if_getsoftc(ifp) != arg) /* Not our event */
3929 return;
3930
3931 if ((vtag == 0) || (vtag > 4095)) /* Invalid ID */
3932 return;
3933
3934 EM_CORE_LOCK(adapter);
3935 index = (vtag >> 5) & 0x7F;
3936 bit = vtag & 0x1F;
3937 adapter->shadow_vfta[index] |= (1 << bit);
3938 ++adapter->num_vlans;
3939 /* Re-init to load the changes */
3940 if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
3941 lem_init_locked(adapter);
3942 EM_CORE_UNLOCK(adapter);
3943 }
3944
3945 /*
3946 * This routine is run via an vlan
3947 * unconfig EVENT
3948 */
3949 static void
lem_unregister_vlan(void * arg,if_t ifp,u16 vtag)3950 lem_unregister_vlan(void *arg, if_t ifp, u16 vtag)
3951 {
3952 struct adapter *adapter = if_getsoftc(ifp);
3953 u32 index, bit;
3954
3955 if (if_getsoftc(ifp) != arg)
3956 return;
3957
3958 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
3959 return;
3960
3961 EM_CORE_LOCK(adapter);
3962 index = (vtag >> 5) & 0x7F;
3963 bit = vtag & 0x1F;
3964 adapter->shadow_vfta[index] &= ~(1 << bit);
3965 --adapter->num_vlans;
3966 /* Re-init to load the changes */
3967 if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
3968 lem_init_locked(adapter);
3969 EM_CORE_UNLOCK(adapter);
3970 }
3971
3972 static void
lem_setup_vlan_hw_support(struct adapter * adapter)3973 lem_setup_vlan_hw_support(struct adapter *adapter)
3974 {
3975 struct e1000_hw *hw = &adapter->hw;
3976 u32 reg;
3977
3978 /*
3979 ** We get here thru init_locked, meaning
3980 ** a soft reset, this has already cleared
3981 ** the VFTA and other state, so if there
3982 ** have been no vlan's registered do nothing.
3983 */
3984 if (adapter->num_vlans == 0)
3985 return;
3986
3987 /*
3988 ** A soft reset zero's out the VFTA, so
3989 ** we need to repopulate it now.
3990 */
3991 for (int i = 0; i < EM_VFTA_SIZE; i++)
3992 if (adapter->shadow_vfta[i] != 0)
3993 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
3994 i, adapter->shadow_vfta[i]);
3995
3996 reg = E1000_READ_REG(hw, E1000_CTRL);
3997 reg |= E1000_CTRL_VME;
3998 E1000_WRITE_REG(hw, E1000_CTRL, reg);
3999
4000 /* Enable the Filter Table */
4001 reg = E1000_READ_REG(hw, E1000_RCTL);
4002 reg &= ~E1000_RCTL_CFIEN;
4003 reg |= E1000_RCTL_VFE;
4004 E1000_WRITE_REG(hw, E1000_RCTL, reg);
4005 }
4006
4007 static void
lem_enable_intr(struct adapter * adapter)4008 lem_enable_intr(struct adapter *adapter)
4009 {
4010 struct e1000_hw *hw = &adapter->hw;
4011 u32 ims_mask = IMS_ENABLE_MASK;
4012
4013 E1000_WRITE_REG(hw, E1000_IMS, ims_mask);
4014 }
4015
4016 static void
lem_disable_intr(struct adapter * adapter)4017 lem_disable_intr(struct adapter *adapter)
4018 {
4019 struct e1000_hw *hw = &adapter->hw;
4020
4021 E1000_WRITE_REG(hw, E1000_IMC, 0xffffffff);
4022 }
4023
4024 /*
4025 * Bit of a misnomer, what this really means is
4026 * to enable OS management of the system... aka
4027 * to disable special hardware management features
4028 */
4029 static void
lem_init_manageability(struct adapter * adapter)4030 lem_init_manageability(struct adapter *adapter)
4031 {
4032 /* A shared code workaround */
4033 if (adapter->has_manage) {
4034 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4035 /* disable hardware interception of ARP */
4036 manc &= ~(E1000_MANC_ARP_EN);
4037 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4038 }
4039 }
4040
4041 /*
4042 * Give control back to hardware management
4043 * controller if there is one.
4044 */
4045 static void
lem_release_manageability(struct adapter * adapter)4046 lem_release_manageability(struct adapter *adapter)
4047 {
4048 if (adapter->has_manage) {
4049 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4050
4051 /* re-enable hardware interception of ARP */
4052 manc |= E1000_MANC_ARP_EN;
4053 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4054 }
4055 }
4056
4057 /*
4058 * lem_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit.
4059 * For ASF and Pass Through versions of f/w this means
4060 * that the driver is loaded. For AMT version type f/w
4061 * this means that the network i/f is open.
4062 */
4063 static void
lem_get_hw_control(struct adapter * adapter)4064 lem_get_hw_control(struct adapter *adapter)
4065 {
4066 u32 ctrl_ext;
4067
4068 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4069 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4070 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
4071 return;
4072 }
4073
4074 /*
4075 * lem_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
4076 * For ASF and Pass Through versions of f/w this means that
4077 * the driver is no longer loaded. For AMT versions of the
4078 * f/w this means that the network i/f is closed.
4079 */
4080 static void
lem_release_hw_control(struct adapter * adapter)4081 lem_release_hw_control(struct adapter *adapter)
4082 {
4083 u32 ctrl_ext;
4084
4085 if (!adapter->has_manage)
4086 return;
4087
4088 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4089 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4090 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
4091 return;
4092 }
4093
4094 static int
lem_is_valid_ether_addr(u8 * addr)4095 lem_is_valid_ether_addr(u8 *addr)
4096 {
4097 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
4098
4099 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
4100 return (FALSE);
4101 }
4102
4103 return (TRUE);
4104 }
4105
4106 /*
4107 ** Parse the interface capabilities with regard
4108 ** to both system management and wake-on-lan for
4109 ** later use.
4110 */
4111 static void
lem_get_wakeup(device_t dev)4112 lem_get_wakeup(device_t dev)
4113 {
4114 struct adapter *adapter = device_get_softc(dev);
4115 u16 eeprom_data = 0, device_id, apme_mask;
4116
4117 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
4118 apme_mask = EM_EEPROM_APME;
4119
4120 switch (adapter->hw.mac.type) {
4121 case e1000_82542:
4122 case e1000_82543:
4123 break;
4124 case e1000_82544:
4125 e1000_read_nvm(&adapter->hw,
4126 NVM_INIT_CONTROL2_REG, 1, &eeprom_data);
4127 apme_mask = EM_82544_APME;
4128 break;
4129 case e1000_82546:
4130 case e1000_82546_rev_3:
4131 if (adapter->hw.bus.func == 1) {
4132 e1000_read_nvm(&adapter->hw,
4133 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
4134 break;
4135 } else
4136 e1000_read_nvm(&adapter->hw,
4137 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
4138 break;
4139 default:
4140 e1000_read_nvm(&adapter->hw,
4141 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
4142 break;
4143 }
4144 if (eeprom_data & apme_mask)
4145 adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC);
4146 /*
4147 * We have the eeprom settings, now apply the special cases
4148 * where the eeprom may be wrong or the board won't support
4149 * wake on lan on a particular port
4150 */
4151 device_id = pci_get_device(dev);
4152 switch (device_id) {
4153 case E1000_DEV_ID_82546GB_PCIE:
4154 adapter->wol = 0;
4155 break;
4156 case E1000_DEV_ID_82546EB_FIBER:
4157 case E1000_DEV_ID_82546GB_FIBER:
4158 /* Wake events only supported on port A for dual fiber
4159 * regardless of eeprom setting */
4160 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
4161 E1000_STATUS_FUNC_1)
4162 adapter->wol = 0;
4163 break;
4164 case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
4165 /* if quad port adapter, disable WoL on all but port A */
4166 if (global_quad_port_a != 0)
4167 adapter->wol = 0;
4168 /* Reset for multiple quad port adapters */
4169 if (++global_quad_port_a == 4)
4170 global_quad_port_a = 0;
4171 break;
4172 }
4173 return;
4174 }
4175
4176
4177 /*
4178 * Enable PCI Wake On Lan capability
4179 */
4180 static void
lem_enable_wakeup(device_t dev)4181 lem_enable_wakeup(device_t dev)
4182 {
4183 struct adapter *adapter = device_get_softc(dev);
4184 if_t ifp = adapter->ifp;
4185 u32 pmc, ctrl, ctrl_ext, rctl;
4186 u16 status;
4187
4188 if ((pci_find_cap(dev, PCIY_PMG, &pmc) != 0))
4189 return;
4190
4191 /* Advertise the wakeup capability */
4192 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
4193 ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3);
4194 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
4195 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
4196
4197 /* Keep the laser running on Fiber adapters */
4198 if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
4199 adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
4200 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4201 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA;
4202 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext);
4203 }
4204
4205 /*
4206 ** Determine type of Wakeup: note that wol
4207 ** is set with all bits on by default.
4208 */
4209 if ((if_getcapenable(ifp) & IFCAP_WOL_MAGIC) == 0)
4210 adapter->wol &= ~E1000_WUFC_MAG;
4211
4212 if ((if_getcapenable(ifp) & IFCAP_WOL_MCAST) == 0)
4213 adapter->wol &= ~E1000_WUFC_MC;
4214 else {
4215 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
4216 rctl |= E1000_RCTL_MPE;
4217 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
4218 }
4219
4220 if (adapter->hw.mac.type == e1000_pchlan) {
4221 if (lem_enable_phy_wakeup(adapter))
4222 return;
4223 } else {
4224 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
4225 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
4226 }
4227
4228
4229 /* Request PME */
4230 status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2);
4231 status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE);
4232 if (if_getcapenable(ifp) & IFCAP_WOL)
4233 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
4234 pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2);
4235
4236 return;
4237 }
4238
4239 /*
4240 ** WOL in the newer chipset interfaces (pchlan)
4241 ** require thing to be copied into the phy
4242 */
4243 static int
lem_enable_phy_wakeup(struct adapter * adapter)4244 lem_enable_phy_wakeup(struct adapter *adapter)
4245 {
4246 struct e1000_hw *hw = &adapter->hw;
4247 u32 mreg, ret = 0;
4248 u16 preg;
4249
4250 /* copy MAC RARs to PHY RARs */
4251 for (int i = 0; i < adapter->hw.mac.rar_entry_count; i++) {
4252 mreg = E1000_READ_REG(hw, E1000_RAL(i));
4253 e1000_write_phy_reg(hw, BM_RAR_L(i), (u16)(mreg & 0xFFFF));
4254 e1000_write_phy_reg(hw, BM_RAR_M(i),
4255 (u16)((mreg >> 16) & 0xFFFF));
4256 mreg = E1000_READ_REG(hw, E1000_RAH(i));
4257 e1000_write_phy_reg(hw, BM_RAR_H(i), (u16)(mreg & 0xFFFF));
4258 e1000_write_phy_reg(hw, BM_RAR_CTRL(i),
4259 (u16)((mreg >> 16) & 0xFFFF));
4260 }
4261
4262 /* copy MAC MTA to PHY MTA */
4263 for (int i = 0; i < adapter->hw.mac.mta_reg_count; i++) {
4264 mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i);
4265 e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF));
4266 e1000_write_phy_reg(hw, BM_MTA(i) + 1,
4267 (u16)((mreg >> 16) & 0xFFFF));
4268 }
4269
4270 /* configure PHY Rx Control register */
4271 e1000_read_phy_reg(&adapter->hw, BM_RCTL, &preg);
4272 mreg = E1000_READ_REG(hw, E1000_RCTL);
4273 if (mreg & E1000_RCTL_UPE)
4274 preg |= BM_RCTL_UPE;
4275 if (mreg & E1000_RCTL_MPE)
4276 preg |= BM_RCTL_MPE;
4277 preg &= ~(BM_RCTL_MO_MASK);
4278 if (mreg & E1000_RCTL_MO_3)
4279 preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT)
4280 << BM_RCTL_MO_SHIFT);
4281 if (mreg & E1000_RCTL_BAM)
4282 preg |= BM_RCTL_BAM;
4283 if (mreg & E1000_RCTL_PMCF)
4284 preg |= BM_RCTL_PMCF;
4285 mreg = E1000_READ_REG(hw, E1000_CTRL);
4286 if (mreg & E1000_CTRL_RFCE)
4287 preg |= BM_RCTL_RFCE;
4288 e1000_write_phy_reg(&adapter->hw, BM_RCTL, preg);
4289
4290 /* enable PHY wakeup in MAC register */
4291 E1000_WRITE_REG(hw, E1000_WUC,
4292 E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN);
4293 E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol);
4294
4295 /* configure and enable PHY wakeup in PHY registers */
4296 e1000_write_phy_reg(&adapter->hw, BM_WUFC, adapter->wol);
4297 e1000_write_phy_reg(&adapter->hw, BM_WUC, E1000_WUC_PME_EN);
4298
4299 /* activate PHY wakeup */
4300 ret = hw->phy.ops.acquire(hw);
4301 if (ret) {
4302 printf("Could not acquire PHY\n");
4303 return ret;
4304 }
4305 e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT,
4306 (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT));
4307 ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg);
4308 if (ret) {
4309 printf("Could not read PHY page 769\n");
4310 goto out;
4311 }
4312 preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT;
4313 ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg);
4314 if (ret)
4315 printf("Could not set PHY Host Wakeup bit\n");
4316 out:
4317 hw->phy.ops.release(hw);
4318
4319 return ret;
4320 }
4321
4322 static void
lem_led_func(void * arg,int onoff)4323 lem_led_func(void *arg, int onoff)
4324 {
4325 struct adapter *adapter = arg;
4326
4327 EM_CORE_LOCK(adapter);
4328 if (onoff) {
4329 e1000_setup_led(&adapter->hw);
4330 e1000_led_on(&adapter->hw);
4331 } else {
4332 e1000_led_off(&adapter->hw);
4333 e1000_cleanup_led(&adapter->hw);
4334 }
4335 EM_CORE_UNLOCK(adapter);
4336 }
4337
4338 /*********************************************************************
4339 * 82544 Coexistence issue workaround.
4340 * There are 2 issues.
4341 * 1. Transmit Hang issue.
4342 * To detect this issue, following equation can be used...
4343 * SIZE[3:0] + ADDR[2:0] = SUM[3:0].
4344 * If SUM[3:0] is in between 1 to 4, we will have this issue.
4345 *
4346 * 2. DAC issue.
4347 * To detect this issue, following equation can be used...
4348 * SIZE[3:0] + ADDR[2:0] = SUM[3:0].
4349 * If SUM[3:0] is in between 9 to c, we will have this issue.
4350 *
4351 *
4352 * WORKAROUND:
4353 * Make sure we do not have ending address
4354 * as 1,2,3,4(Hang) or 9,a,b,c (DAC)
4355 *
4356 *************************************************************************/
4357 static u32
lem_fill_descriptors(bus_addr_t address,u32 length,PDESC_ARRAY desc_array)4358 lem_fill_descriptors (bus_addr_t address, u32 length,
4359 PDESC_ARRAY desc_array)
4360 {
4361 u32 safe_terminator;
4362
4363 /* Since issue is sensitive to length and address.*/
4364 /* Let us first check the address...*/
4365 if (length <= 4) {
4366 desc_array->descriptor[0].address = address;
4367 desc_array->descriptor[0].length = length;
4368 desc_array->elements = 1;
4369 return (desc_array->elements);
4370 }
4371 safe_terminator = (u32)((((u32)address & 0x7) +
4372 (length & 0xF)) & 0xF);
4373 /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
4374 if (safe_terminator == 0 ||
4375 (safe_terminator > 4 &&
4376 safe_terminator < 9) ||
4377 (safe_terminator > 0xC &&
4378 safe_terminator <= 0xF)) {
4379 desc_array->descriptor[0].address = address;
4380 desc_array->descriptor[0].length = length;
4381 desc_array->elements = 1;
4382 return (desc_array->elements);
4383 }
4384
4385 desc_array->descriptor[0].address = address;
4386 desc_array->descriptor[0].length = length - 4;
4387 desc_array->descriptor[1].address = address + (length - 4);
4388 desc_array->descriptor[1].length = 4;
4389 desc_array->elements = 2;
4390 return (desc_array->elements);
4391 }
4392
4393 /**********************************************************************
4394 *
4395 * Update the board statistics counters.
4396 *
4397 **********************************************************************/
4398 static void
lem_update_stats_counters(struct adapter * adapter)4399 lem_update_stats_counters(struct adapter *adapter)
4400 {
4401
4402 if(adapter->hw.phy.media_type == e1000_media_type_copper ||
4403 (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
4404 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
4405 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
4406 }
4407 adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
4408 adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
4409 adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
4410 adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
4411
4412 adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
4413 adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
4414 adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
4415 adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
4416 adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
4417 adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
4418 adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
4419 adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
4420 adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
4421 adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
4422 adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
4423 adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
4424 adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
4425 adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
4426 adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
4427 adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
4428 adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
4429 adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
4430 adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
4431 adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
4432
4433 /* For the 64-bit byte counters the low dword must be read first. */
4434 /* Both registers clear on the read of the high dword */
4435
4436 adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCL) +
4437 ((u64)E1000_READ_REG(&adapter->hw, E1000_GORCH) << 32);
4438 adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCL) +
4439 ((u64)E1000_READ_REG(&adapter->hw, E1000_GOTCH) << 32);
4440
4441 adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
4442 adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
4443 adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
4444 adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
4445 adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
4446
4447 adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
4448 adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
4449
4450 adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
4451 adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
4452 adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
4453 adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
4454 adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
4455 adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
4456 adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
4457 adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
4458 adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
4459 adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
4460
4461 if (adapter->hw.mac.type >= e1000_82543) {
4462 adapter->stats.algnerrc +=
4463 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
4464 adapter->stats.rxerrc +=
4465 E1000_READ_REG(&adapter->hw, E1000_RXERRC);
4466 adapter->stats.tncrs +=
4467 E1000_READ_REG(&adapter->hw, E1000_TNCRS);
4468 adapter->stats.cexterr +=
4469 E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
4470 adapter->stats.tsctc +=
4471 E1000_READ_REG(&adapter->hw, E1000_TSCTC);
4472 adapter->stats.tsctfc +=
4473 E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
4474 }
4475 }
4476
4477 static uint64_t
lem_get_counter(if_t ifp,ift_counter cnt)4478 lem_get_counter(if_t ifp, ift_counter cnt)
4479 {
4480 struct adapter *adapter;
4481
4482 adapter = if_getsoftc(ifp);
4483
4484 switch (cnt) {
4485 case IFCOUNTER_COLLISIONS:
4486 return (adapter->stats.colc);
4487 case IFCOUNTER_IERRORS:
4488 return (adapter->dropped_pkts + adapter->stats.rxerrc +
4489 adapter->stats.crcerrs + adapter->stats.algnerrc +
4490 adapter->stats.ruc + adapter->stats.roc +
4491 adapter->stats.mpc + adapter->stats.cexterr);
4492 case IFCOUNTER_OERRORS:
4493 return (adapter->stats.ecol + adapter->stats.latecol +
4494 adapter->watchdog_events);
4495 default:
4496 return (if_get_counter_default(ifp, cnt));
4497 }
4498 }
4499
4500 /* Export a single 32-bit register via a read-only sysctl. */
4501 static int
lem_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)4502 lem_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
4503 {
4504 struct adapter *adapter;
4505 u_int val;
4506
4507 adapter = oidp->oid_arg1;
4508 val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2);
4509 return (sysctl_handle_int(oidp, &val, 0, req));
4510 }
4511
4512 /*
4513 * Add sysctl variables, one per statistic, to the system.
4514 */
4515 static void
lem_add_hw_stats(struct adapter * adapter)4516 lem_add_hw_stats(struct adapter *adapter)
4517 {
4518 device_t dev = adapter->dev;
4519
4520 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
4521 struct sysctl_oid *tree = device_get_sysctl_tree(dev);
4522 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
4523 struct e1000_hw_stats *stats = &adapter->stats;
4524
4525 struct sysctl_oid *stat_node;
4526 struct sysctl_oid_list *stat_list;
4527
4528 /* Driver Statistics */
4529 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_alloc_fail",
4530 CTLFLAG_RD, &adapter->mbuf_alloc_failed,
4531 "Std mbuf failed");
4532 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "cluster_alloc_fail",
4533 CTLFLAG_RD, &adapter->mbuf_cluster_failed,
4534 "Std mbuf cluster failed");
4535 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
4536 CTLFLAG_RD, &adapter->dropped_pkts,
4537 "Driver dropped packets");
4538 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail",
4539 CTLFLAG_RD, &adapter->no_tx_dma_setup,
4540 "Driver tx dma failure in xmit");
4541 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_desc_fail1",
4542 CTLFLAG_RD, &adapter->no_tx_desc_avail1,
4543 "Not enough tx descriptors failure in xmit");
4544 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_desc_fail2",
4545 CTLFLAG_RD, &adapter->no_tx_desc_avail2,
4546 "Not enough tx descriptors failure in xmit");
4547 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
4548 CTLFLAG_RD, &adapter->rx_overruns,
4549 "RX overruns");
4550 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts",
4551 CTLFLAG_RD, &adapter->watchdog_events,
4552 "Watchdog timeouts");
4553
4554 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control",
4555 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_CTRL,
4556 lem_sysctl_reg_handler, "IU",
4557 "Device Control Register");
4558 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control",
4559 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RCTL,
4560 lem_sysctl_reg_handler, "IU",
4561 "Receiver Control Register");
4562 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
4563 CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
4564 "Flow Control High Watermark");
4565 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water",
4566 CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
4567 "Flow Control Low Watermark");
4568 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "fifo_workaround",
4569 CTLFLAG_RD, &adapter->tx_fifo_wrk_cnt,
4570 "TX FIFO workaround events");
4571 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "fifo_reset",
4572 CTLFLAG_RD, &adapter->tx_fifo_reset_cnt,
4573 "TX FIFO resets");
4574
4575 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "txd_head",
4576 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDH(0),
4577 lem_sysctl_reg_handler, "IU",
4578 "Transmit Descriptor Head");
4579 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "txd_tail",
4580 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDT(0),
4581 lem_sysctl_reg_handler, "IU",
4582 "Transmit Descriptor Tail");
4583 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rxd_head",
4584 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDH(0),
4585 lem_sysctl_reg_handler, "IU",
4586 "Receive Descriptor Head");
4587 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rxd_tail",
4588 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDT(0),
4589 lem_sysctl_reg_handler, "IU",
4590 "Receive Descriptor Tail");
4591
4592
4593 /* MAC stats get their own sub node */
4594
4595 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
4596 CTLFLAG_RD, NULL, "Statistics");
4597 stat_list = SYSCTL_CHILDREN(stat_node);
4598
4599 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "excess_coll",
4600 CTLFLAG_RD, &stats->ecol,
4601 "Excessive collisions");
4602 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "single_coll",
4603 CTLFLAG_RD, &stats->scc,
4604 "Single collisions");
4605 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "multiple_coll",
4606 CTLFLAG_RD, &stats->mcc,
4607 "Multiple collisions");
4608 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "late_coll",
4609 CTLFLAG_RD, &stats->latecol,
4610 "Late collisions");
4611 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "collision_count",
4612 CTLFLAG_RD, &stats->colc,
4613 "Collision Count");
4614 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
4615 CTLFLAG_RD, &adapter->stats.symerrs,
4616 "Symbol Errors");
4617 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
4618 CTLFLAG_RD, &adapter->stats.sec,
4619 "Sequence Errors");
4620 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "defer_count",
4621 CTLFLAG_RD, &adapter->stats.dc,
4622 "Defer Count");
4623 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "missed_packets",
4624 CTLFLAG_RD, &adapter->stats.mpc,
4625 "Missed Packets");
4626 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
4627 CTLFLAG_RD, &adapter->stats.rnbc,
4628 "Receive No Buffers");
4629 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
4630 CTLFLAG_RD, &adapter->stats.ruc,
4631 "Receive Undersize");
4632 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
4633 CTLFLAG_RD, &adapter->stats.rfc,
4634 "Fragmented Packets Received ");
4635 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
4636 CTLFLAG_RD, &adapter->stats.roc,
4637 "Oversized Packets Received");
4638 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
4639 CTLFLAG_RD, &adapter->stats.rjc,
4640 "Recevied Jabber");
4641 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_errs",
4642 CTLFLAG_RD, &adapter->stats.rxerrc,
4643 "Receive Errors");
4644 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
4645 CTLFLAG_RD, &adapter->stats.crcerrs,
4646 "CRC errors");
4647 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
4648 CTLFLAG_RD, &adapter->stats.algnerrc,
4649 "Alignment Errors");
4650 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs",
4651 CTLFLAG_RD, &adapter->stats.cexterr,
4652 "Collision/Carrier extension errors");
4653 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
4654 CTLFLAG_RD, &adapter->stats.xonrxc,
4655 "XON Received");
4656 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
4657 CTLFLAG_RD, &adapter->stats.xontxc,
4658 "XON Transmitted");
4659 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
4660 CTLFLAG_RD, &adapter->stats.xoffrxc,
4661 "XOFF Received");
4662 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
4663 CTLFLAG_RD, &adapter->stats.xofftxc,
4664 "XOFF Transmitted");
4665
4666 /* Packet Reception Stats */
4667 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
4668 CTLFLAG_RD, &adapter->stats.tpr,
4669 "Total Packets Received ");
4670 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
4671 CTLFLAG_RD, &adapter->stats.gprc,
4672 "Good Packets Received");
4673 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
4674 CTLFLAG_RD, &adapter->stats.bprc,
4675 "Broadcast Packets Received");
4676 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
4677 CTLFLAG_RD, &adapter->stats.mprc,
4678 "Multicast Packets Received");
4679 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
4680 CTLFLAG_RD, &adapter->stats.prc64,
4681 "64 byte frames received ");
4682 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
4683 CTLFLAG_RD, &adapter->stats.prc127,
4684 "65-127 byte frames received");
4685 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
4686 CTLFLAG_RD, &adapter->stats.prc255,
4687 "128-255 byte frames received");
4688 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
4689 CTLFLAG_RD, &adapter->stats.prc511,
4690 "256-511 byte frames received");
4691 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
4692 CTLFLAG_RD, &adapter->stats.prc1023,
4693 "512-1023 byte frames received");
4694 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
4695 CTLFLAG_RD, &adapter->stats.prc1522,
4696 "1023-1522 byte frames received");
4697 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
4698 CTLFLAG_RD, &adapter->stats.gorc,
4699 "Good Octets Received");
4700
4701 /* Packet Transmission Stats */
4702 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
4703 CTLFLAG_RD, &adapter->stats.gotc,
4704 "Good Octets Transmitted");
4705 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
4706 CTLFLAG_RD, &adapter->stats.tpt,
4707 "Total Packets Transmitted");
4708 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
4709 CTLFLAG_RD, &adapter->stats.gptc,
4710 "Good Packets Transmitted");
4711 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
4712 CTLFLAG_RD, &adapter->stats.bptc,
4713 "Broadcast Packets Transmitted");
4714 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
4715 CTLFLAG_RD, &adapter->stats.mptc,
4716 "Multicast Packets Transmitted");
4717 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
4718 CTLFLAG_RD, &adapter->stats.ptc64,
4719 "64 byte frames transmitted ");
4720 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
4721 CTLFLAG_RD, &adapter->stats.ptc127,
4722 "65-127 byte frames transmitted");
4723 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
4724 CTLFLAG_RD, &adapter->stats.ptc255,
4725 "128-255 byte frames transmitted");
4726 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
4727 CTLFLAG_RD, &adapter->stats.ptc511,
4728 "256-511 byte frames transmitted");
4729 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
4730 CTLFLAG_RD, &adapter->stats.ptc1023,
4731 "512-1023 byte frames transmitted");
4732 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
4733 CTLFLAG_RD, &adapter->stats.ptc1522,
4734 "1024-1522 byte frames transmitted");
4735 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_txd",
4736 CTLFLAG_RD, &adapter->stats.tsctc,
4737 "TSO Contexts Transmitted");
4738 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail",
4739 CTLFLAG_RD, &adapter->stats.tsctfc,
4740 "TSO Contexts Failed");
4741 }
4742
4743 /**********************************************************************
4744 *
4745 * This routine provides a way to dump out the adapter eeprom,
4746 * often a useful debug/service tool. This only dumps the first
4747 * 32 words, stuff that matters is in that extent.
4748 *
4749 **********************************************************************/
4750
4751 static int
lem_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)4752 lem_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
4753 {
4754 struct adapter *adapter;
4755 int error;
4756 int result;
4757
4758 result = -1;
4759 error = sysctl_handle_int(oidp, &result, 0, req);
4760
4761 if (error || !req->newptr)
4762 return (error);
4763
4764 /*
4765 * This value will cause a hex dump of the
4766 * first 32 16-bit words of the EEPROM to
4767 * the screen.
4768 */
4769 if (result == 1) {
4770 adapter = (struct adapter *)arg1;
4771 lem_print_nvm_info(adapter);
4772 }
4773
4774 return (error);
4775 }
4776
4777 static void
lem_print_nvm_info(struct adapter * adapter)4778 lem_print_nvm_info(struct adapter *adapter)
4779 {
4780 u16 eeprom_data;
4781 int i, j, row = 0;
4782
4783 /* Its a bit crude, but it gets the job done */
4784 printf("\nInterface EEPROM Dump:\n");
4785 printf("Offset\n0x0000 ");
4786 for (i = 0, j = 0; i < 32; i++, j++) {
4787 if (j == 8) { /* Make the offset block */
4788 j = 0; ++row;
4789 printf("\n0x00%x0 ",row);
4790 }
4791 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
4792 printf("%04x ", eeprom_data);
4793 }
4794 printf("\n");
4795 }
4796
4797 static int
lem_sysctl_int_delay(SYSCTL_HANDLER_ARGS)4798 lem_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
4799 {
4800 struct em_int_delay_info *info;
4801 struct adapter *adapter;
4802 u32 regval;
4803 int error;
4804 int usecs;
4805 int ticks;
4806
4807 info = (struct em_int_delay_info *)arg1;
4808 usecs = info->value;
4809 error = sysctl_handle_int(oidp, &usecs, 0, req);
4810 if (error != 0 || req->newptr == NULL)
4811 return (error);
4812 if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535))
4813 return (EINVAL);
4814 info->value = usecs;
4815 ticks = EM_USECS_TO_TICKS(usecs);
4816 if (info->offset == E1000_ITR) /* units are 256ns here */
4817 ticks *= 4;
4818
4819 adapter = info->adapter;
4820
4821 EM_CORE_LOCK(adapter);
4822 regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
4823 regval = (regval & ~0xffff) | (ticks & 0xffff);
4824 /* Handle a few special cases. */
4825 switch (info->offset) {
4826 case E1000_RDTR:
4827 break;
4828 case E1000_TIDV:
4829 if (ticks == 0) {
4830 adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
4831 /* Don't write 0 into the TIDV register. */
4832 regval++;
4833 } else
4834 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
4835 break;
4836 }
4837 E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
4838 EM_CORE_UNLOCK(adapter);
4839 return (0);
4840 }
4841
4842 static void
lem_add_int_delay_sysctl(struct adapter * adapter,const char * name,const char * description,struct em_int_delay_info * info,int offset,int value)4843 lem_add_int_delay_sysctl(struct adapter *adapter, const char *name,
4844 const char *description, struct em_int_delay_info *info,
4845 int offset, int value)
4846 {
4847 info->adapter = adapter;
4848 info->offset = offset;
4849 info->value = value;
4850 SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev),
4851 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
4852 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
4853 info, 0, lem_sysctl_int_delay, "I", description);
4854 }
4855
4856 static void
lem_set_flow_cntrl(struct adapter * adapter,const char * name,const char * description,int * limit,int value)4857 lem_set_flow_cntrl(struct adapter *adapter, const char *name,
4858 const char *description, int *limit, int value)
4859 {
4860 *limit = value;
4861 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
4862 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
4863 OID_AUTO, name, CTLFLAG_RW, limit, value, description);
4864 }
4865
4866 static void
lem_add_rx_process_limit(struct adapter * adapter,const char * name,const char * description,int * limit,int value)4867 lem_add_rx_process_limit(struct adapter *adapter, const char *name,
4868 const char *description, int *limit, int value)
4869 {
4870 *limit = value;
4871 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
4872 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
4873 OID_AUTO, name, CTLFLAG_RW, limit, value, description);
4874 }
4875