1 /*
2 * Copyright (c) 2004 Joerg Sonnenberger <joerg@bec.de>. All rights reserved.
3 *
4 * Copyright (c) 2001-2015, Intel Corporation
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright notice,
11 * this list of conditions and the following disclaimer.
12 *
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * 3. Neither the name of the Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived from
19 * this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE.
32 *
33 *
34 * Copyright (c) 2005 The DragonFly Project. All rights reserved.
35 *
36 * This code is derived from software contributed to The DragonFly Project
37 * by Matthew Dillon <dillon@backplane.com>
38 *
39 * Redistribution and use in source and binary forms, with or without
40 * modification, are permitted provided that the following conditions
41 * are met:
42 *
43 * 1. Redistributions of source code must retain the above copyright
44 * notice, this list of conditions and the following disclaimer.
45 * 2. Redistributions in binary form must reproduce the above copyright
46 * notice, this list of conditions and the following disclaimer in
47 * the documentation and/or other materials provided with the
48 * distribution.
49 * 3. Neither the name of The DragonFly Project nor the names of its
50 * contributors may be used to endorse or promote products derived
51 * from this software without specific, prior written permission.
52 *
53 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
54 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
55 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
56 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
57 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
58 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
59 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
60 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
61 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
62 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
63 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
64 * SUCH DAMAGE.
65 *
66 */
67 /*
68 * SERIALIZATION API RULES:
69 *
70 * - We must call lwkt_serialize_handler_enable() prior to enabling the
71 * hardware interrupt and lwkt_serialize_handler_disable() after disabling
72 * the hardware interrupt in order to avoid handler execution races from
73 * scheduled interrupt threads.
74 */
75
76 #include "opt_ifpoll.h"
77
78 #include <sys/param.h>
79 #include <sys/bus.h>
80 #include <sys/endian.h>
81 #include <sys/interrupt.h>
82 #include <sys/kernel.h>
83 #include <sys/ktr.h>
84 #include <sys/malloc.h>
85 #include <sys/mbuf.h>
86 #include <sys/proc.h>
87 #include <sys/rman.h>
88 #include <sys/serialize.h>
89 #include <sys/socket.h>
90 #include <sys/sockio.h>
91 #include <sys/sysctl.h>
92 #include <sys/systm.h>
93
94 #include <net/bpf.h>
95 #include <net/ethernet.h>
96 #include <net/if.h>
97 #include <net/if_arp.h>
98 #include <net/if_dl.h>
99 #include <net/if_media.h>
100 #include <net/if_poll.h>
101 #include <net/ifq_var.h>
102 #include <net/vlan/if_vlan_var.h>
103 #include <net/vlan/if_vlan_ether.h>
104
105 #include <netinet/ip.h>
106 #include <netinet/tcp.h>
107 #include <netinet/udp.h>
108
109 #include <bus/pci/pcivar.h>
110 #include <bus/pci/pcireg.h>
111
112 #include <dev/netif/ig_hal/e1000_api.h>
113 #include <dev/netif/ig_hal/e1000_82571.h>
114 #include <dev/netif/ig_hal/e1000_dragonfly.h>
115 #include <dev/netif/em/if_em.h>
116
117 #define DEBUG_HW 0
118
119 #define EM_NAME "Intel(R) PRO/1000 Network Connection "
120 #define EM_VER " 7.6.2"
121
122 #define _EM_DEVICE(id, ret) \
123 { EM_VENDOR_ID, E1000_DEV_ID_##id, ret, EM_NAME #id EM_VER }
124 #define EM_EMX_DEVICE(id) _EM_DEVICE(id, -100)
125 #define EM_DEVICE(id) _EM_DEVICE(id, 0)
126 #define EM_DEVICE_NULL { 0, 0, 0, NULL }
127
128 static const struct em_vendor_info em_vendor_info_array[] = {
129 EM_DEVICE(82540EM),
130 EM_DEVICE(82540EM_LOM),
131 EM_DEVICE(82540EP),
132 EM_DEVICE(82540EP_LOM),
133 EM_DEVICE(82540EP_LP),
134
135 EM_DEVICE(82541EI),
136 EM_DEVICE(82541ER),
137 EM_DEVICE(82541ER_LOM),
138 EM_DEVICE(82541EI_MOBILE),
139 EM_DEVICE(82541GI),
140 EM_DEVICE(82541GI_LF),
141 EM_DEVICE(82541GI_MOBILE),
142
143 EM_DEVICE(82542),
144
145 EM_DEVICE(82543GC_FIBER),
146 EM_DEVICE(82543GC_COPPER),
147
148 EM_DEVICE(82544EI_COPPER),
149 EM_DEVICE(82544EI_FIBER),
150 EM_DEVICE(82544GC_COPPER),
151 EM_DEVICE(82544GC_LOM),
152
153 EM_DEVICE(82545EM_COPPER),
154 EM_DEVICE(82545EM_FIBER),
155 EM_DEVICE(82545GM_COPPER),
156 EM_DEVICE(82545GM_FIBER),
157 EM_DEVICE(82545GM_SERDES),
158
159 EM_DEVICE(82546EB_COPPER),
160 EM_DEVICE(82546EB_FIBER),
161 EM_DEVICE(82546EB_QUAD_COPPER),
162 EM_DEVICE(82546GB_COPPER),
163 EM_DEVICE(82546GB_FIBER),
164 EM_DEVICE(82546GB_SERDES),
165 EM_DEVICE(82546GB_PCIE),
166 EM_DEVICE(82546GB_QUAD_COPPER),
167 EM_DEVICE(82546GB_QUAD_COPPER_KSP3),
168
169 EM_DEVICE(82547EI),
170 EM_DEVICE(82547EI_MOBILE),
171 EM_DEVICE(82547GI),
172
173 EM_EMX_DEVICE(82571EB_COPPER),
174 EM_EMX_DEVICE(82571EB_FIBER),
175 EM_EMX_DEVICE(82571EB_SERDES),
176 EM_EMX_DEVICE(82571EB_SERDES_DUAL),
177 EM_EMX_DEVICE(82571EB_SERDES_QUAD),
178 EM_EMX_DEVICE(82571EB_QUAD_COPPER),
179 EM_EMX_DEVICE(82571EB_QUAD_COPPER_BP),
180 EM_EMX_DEVICE(82571EB_QUAD_COPPER_LP),
181 EM_EMX_DEVICE(82571EB_QUAD_FIBER),
182 EM_EMX_DEVICE(82571PT_QUAD_COPPER),
183
184 EM_EMX_DEVICE(82572EI_COPPER),
185 EM_EMX_DEVICE(82572EI_FIBER),
186 EM_EMX_DEVICE(82572EI_SERDES),
187 EM_EMX_DEVICE(82572EI),
188
189 EM_EMX_DEVICE(82573E),
190 EM_EMX_DEVICE(82573E_IAMT),
191 EM_EMX_DEVICE(82573L),
192
193 EM_DEVICE(82583V),
194
195 EM_EMX_DEVICE(80003ES2LAN_COPPER_SPT),
196 EM_EMX_DEVICE(80003ES2LAN_SERDES_SPT),
197 EM_EMX_DEVICE(80003ES2LAN_COPPER_DPT),
198 EM_EMX_DEVICE(80003ES2LAN_SERDES_DPT),
199
200 EM_DEVICE(ICH8_IGP_M_AMT),
201 EM_DEVICE(ICH8_IGP_AMT),
202 EM_DEVICE(ICH8_IGP_C),
203 EM_DEVICE(ICH8_IFE),
204 EM_DEVICE(ICH8_IFE_GT),
205 EM_DEVICE(ICH8_IFE_G),
206 EM_DEVICE(ICH8_IGP_M),
207 EM_DEVICE(ICH8_82567V_3),
208
209 EM_DEVICE(ICH9_IGP_M_AMT),
210 EM_DEVICE(ICH9_IGP_AMT),
211 EM_DEVICE(ICH9_IGP_C),
212 EM_DEVICE(ICH9_IGP_M),
213 EM_DEVICE(ICH9_IGP_M_V),
214 EM_DEVICE(ICH9_IFE),
215 EM_DEVICE(ICH9_IFE_GT),
216 EM_DEVICE(ICH9_IFE_G),
217 EM_DEVICE(ICH9_BM),
218
219 EM_EMX_DEVICE(82574L),
220 EM_EMX_DEVICE(82574LA),
221
222 EM_DEVICE(ICH10_R_BM_LM),
223 EM_DEVICE(ICH10_R_BM_LF),
224 EM_DEVICE(ICH10_R_BM_V),
225 EM_DEVICE(ICH10_D_BM_LM),
226 EM_DEVICE(ICH10_D_BM_LF),
227 EM_DEVICE(ICH10_D_BM_V),
228
229 EM_DEVICE(PCH_M_HV_LM),
230 EM_DEVICE(PCH_M_HV_LC),
231 EM_DEVICE(PCH_D_HV_DM),
232 EM_DEVICE(PCH_D_HV_DC),
233
234 EM_DEVICE(PCH2_LV_LM),
235 EM_DEVICE(PCH2_LV_V),
236
237 EM_EMX_DEVICE(PCH_LPT_I217_LM),
238 EM_EMX_DEVICE(PCH_LPT_I217_V),
239 EM_EMX_DEVICE(PCH_LPTLP_I218_LM),
240 EM_EMX_DEVICE(PCH_LPTLP_I218_V),
241 EM_EMX_DEVICE(PCH_I218_LM2),
242 EM_EMX_DEVICE(PCH_I218_V2),
243 EM_EMX_DEVICE(PCH_I218_LM3),
244 EM_EMX_DEVICE(PCH_I218_V3),
245 EM_EMX_DEVICE(PCH_SPT_I219_LM),
246 EM_EMX_DEVICE(PCH_SPT_I219_V),
247 EM_EMX_DEVICE(PCH_SPT_I219_LM2),
248 EM_EMX_DEVICE(PCH_SPT_I219_V2),
249 EM_EMX_DEVICE(PCH_LBG_I219_LM3),
250 EM_EMX_DEVICE(PCH_SPT_I219_LM4),
251 EM_EMX_DEVICE(PCH_SPT_I219_V4),
252 EM_EMX_DEVICE(PCH_SPT_I219_LM5),
253 EM_EMX_DEVICE(PCH_SPT_I219_V5),
254 EM_EMX_DEVICE(PCH_CNP_I219_LM6),
255 EM_EMX_DEVICE(PCH_CNP_I219_V6),
256 EM_EMX_DEVICE(PCH_CNP_I219_LM7),
257 EM_EMX_DEVICE(PCH_CNP_I219_V7),
258 EM_EMX_DEVICE(PCH_ICP_I219_LM8),
259 EM_EMX_DEVICE(PCH_ICP_I219_V8),
260 EM_EMX_DEVICE(PCH_ICP_I219_LM9),
261 EM_EMX_DEVICE(PCH_ICP_I219_V9),
262 EM_EMX_DEVICE(PCH_CMP_I219_LM10),
263 EM_EMX_DEVICE(PCH_CMP_I219_V10),
264 EM_EMX_DEVICE(PCH_CMP_I219_LM11),
265 EM_EMX_DEVICE(PCH_CMP_I219_V11),
266 EM_EMX_DEVICE(PCH_CMP_I219_LM12),
267 EM_EMX_DEVICE(PCH_CMP_I219_V12),
268 EM_EMX_DEVICE(PCH_TGP_I219_LM13),
269 EM_EMX_DEVICE(PCH_TGP_I219_V13),
270 EM_EMX_DEVICE(PCH_TGP_I219_LM14),
271 EM_EMX_DEVICE(PCH_TGP_I219_V14),
272 EM_EMX_DEVICE(PCH_TGP_I219_LM15),
273 EM_EMX_DEVICE(PCH_TGP_I219_V15),
274 EM_EMX_DEVICE(PCH_ADP_I219_LM16),
275 EM_EMX_DEVICE(PCH_ADP_I219_V16),
276 EM_EMX_DEVICE(PCH_ADP_I219_LM17),
277 EM_EMX_DEVICE(PCH_ADP_I219_V17),
278 EM_EMX_DEVICE(PCH_MTP_I219_LM18),
279 EM_EMX_DEVICE(PCH_MTP_I219_V18),
280 EM_EMX_DEVICE(PCH_MTP_I219_LM19),
281 EM_EMX_DEVICE(PCH_MTP_I219_V19),
282
283 /* required last entry */
284 EM_DEVICE_NULL
285 };
286
287 static int em_probe(device_t);
288 static int em_attach(device_t);
289 static int em_detach(device_t);
290 static int em_shutdown(device_t);
291 static int em_suspend(device_t);
292 static int em_resume(device_t);
293
294 static void em_init(void *);
295 static void em_stop(struct adapter *);
296 static int em_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
297 static void em_start(struct ifnet *, struct ifaltq_subque *);
298 #ifdef IFPOLL_ENABLE
299 static void em_npoll(struct ifnet *, struct ifpoll_info *);
300 static void em_npoll_compat(struct ifnet *, void *, int);
301 #endif
302 static void em_watchdog(struct ifnet *);
303 static void em_media_status(struct ifnet *, struct ifmediareq *);
304 static int em_media_change(struct ifnet *);
305 static void em_timer(void *);
306
307 static void em_intr(void *);
308 static void em_intr_mask(void *);
309 static void em_intr_body(struct adapter *, boolean_t);
310 static void em_rxeof(struct adapter *, int);
311 static void em_txeof(struct adapter *);
312 static void em_tx_collect(struct adapter *, boolean_t);
313 static void em_tx_purge(struct adapter *);
314 static void em_txgc_timer(void *);
315 static void em_enable_intr(struct adapter *);
316 static void em_disable_intr(struct adapter *);
317
318 static int em_dma_malloc(struct adapter *, bus_size_t,
319 struct em_dma_alloc *);
320 static void em_dma_free(struct adapter *, struct em_dma_alloc *);
321 static void em_init_tx_ring(struct adapter *);
322 static int em_init_rx_ring(struct adapter *);
323 static int em_create_tx_ring(struct adapter *);
324 static int em_create_rx_ring(struct adapter *);
325 static void em_destroy_tx_ring(struct adapter *, int);
326 static void em_destroy_rx_ring(struct adapter *, int);
327 static int em_newbuf(struct adapter *, int, int);
328 static int em_encap(struct adapter *, struct mbuf **, int *, int *);
329 static void em_rxcsum(struct adapter *, struct e1000_rx_desc *,
330 struct mbuf *);
331 static int em_txcsum(struct adapter *, struct mbuf *,
332 uint32_t *, uint32_t *);
333 static int em_tso_pullup(struct adapter *, struct mbuf **);
334 static int em_tso_setup(struct adapter *, struct mbuf *,
335 uint32_t *, uint32_t *);
336
337 static int em_get_hw_info(struct adapter *);
338 static int em_is_valid_eaddr(const uint8_t *);
339 static int em_alloc_pci_res(struct adapter *);
340 static void em_free_pci_res(struct adapter *);
341 static int em_reset(struct adapter *);
342 static void em_setup_ifp(struct adapter *);
343 static void em_init_tx_unit(struct adapter *);
344 static void em_init_rx_unit(struct adapter *);
345 static void em_update_stats(struct adapter *);
346 static void em_set_promisc(struct adapter *);
347 static void em_disable_promisc(struct adapter *);
348 static void em_set_multi(struct adapter *);
349 static void em_update_link_status(struct adapter *);
350 static void em_smartspeed(struct adapter *);
351 static void em_set_itr(struct adapter *, uint32_t);
352 static void em_disable_aspm(struct adapter *);
353 static void em_flush_tx_ring(struct adapter *);
354 static void em_flush_rx_ring(struct adapter *);
355 static void em_flush_txrx_ring(struct adapter *);
356
357 /* Hardware workarounds */
358 static int em_82547_fifo_workaround(struct adapter *, int);
359 static void em_82547_update_fifo_head(struct adapter *, int);
360 static int em_82547_tx_fifo_reset(struct adapter *);
361 static void em_82547_move_tail(void *);
362 static void em_82547_move_tail_serialized(struct adapter *);
363 static uint32_t em_82544_fill_desc(bus_addr_t, uint32_t, PDESC_ARRAY);
364
365 static void em_print_debug_info(struct adapter *);
366 static void em_print_nvm_info(struct adapter *);
367 static void em_print_hw_stats(struct adapter *);
368
369 static int em_sysctl_stats(SYSCTL_HANDLER_ARGS);
370 static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
371 static int em_sysctl_int_throttle(SYSCTL_HANDLER_ARGS);
372 static int em_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS);
373 static void em_add_sysctl(struct adapter *adapter);
374
375 /* Management and WOL Support */
376 static void em_get_mgmt(struct adapter *);
377 static void em_rel_mgmt(struct adapter *);
378 static void em_get_hw_control(struct adapter *);
379 static void em_rel_hw_control(struct adapter *);
380 static void em_enable_wol(device_t);
381
382 static device_method_t em_methods[] = {
383 /* Device interface */
384 DEVMETHOD(device_probe, em_probe),
385 DEVMETHOD(device_attach, em_attach),
386 DEVMETHOD(device_detach, em_detach),
387 DEVMETHOD(device_shutdown, em_shutdown),
388 DEVMETHOD(device_suspend, em_suspend),
389 DEVMETHOD(device_resume, em_resume),
390 DEVMETHOD_END
391 };
392
393 static driver_t em_driver = {
394 "em",
395 em_methods,
396 sizeof(struct adapter),
397 };
398
399 static devclass_t em_devclass;
400
401 DECLARE_DUMMY_MODULE(if_em);
402 MODULE_DEPEND(em, ig_hal, 1, 1, 1);
403 DRIVER_MODULE(if_em, pci, em_driver, em_devclass, NULL, NULL);
404
405 /*
406 * Tunables
407 */
408 static int em_int_throttle_ceil = EM_DEFAULT_ITR;
409 static int em_rxd = EM_DEFAULT_RXD;
410 static int em_txd = EM_DEFAULT_TXD;
411 static int em_smart_pwr_down = 0;
412
413 /* Controls whether promiscuous also shows bad packets */
414 static int em_debug_sbp = FALSE;
415
416 static int em_82573_workaround = 1;
417 static int em_msi_enable = 1;
418
419 static char em_flowctrl[IFM_ETH_FC_STRLEN] = IFM_ETH_FC_NONE;
420
421 TUNABLE_INT("hw.em.int_throttle_ceil", &em_int_throttle_ceil);
422 TUNABLE_INT("hw.em.rxd", &em_rxd);
423 TUNABLE_INT("hw.em.txd", &em_txd);
424 TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down);
425 TUNABLE_INT("hw.em.sbp", &em_debug_sbp);
426 TUNABLE_INT("hw.em.82573_workaround", &em_82573_workaround);
427 TUNABLE_INT("hw.em.msi.enable", &em_msi_enable);
428 TUNABLE_STR("hw.em.flow_ctrl", em_flowctrl, sizeof(em_flowctrl));
429
430 /* Global used in WOL setup with multiport cards */
431 static int em_global_quad_port_a = 0;
432
433 /* Set this to one to display debug statistics */
434 static int em_display_debug_stats = 0;
435
436 #if !defined(KTR_IF_EM)
437 #define KTR_IF_EM KTR_ALL
438 #endif
439 KTR_INFO_MASTER(if_em);
440 KTR_INFO(KTR_IF_EM, if_em, intr_beg, 0, "intr begin");
441 KTR_INFO(KTR_IF_EM, if_em, intr_end, 1, "intr end");
442 KTR_INFO(KTR_IF_EM, if_em, pkt_receive, 4, "rx packet");
443 KTR_INFO(KTR_IF_EM, if_em, pkt_txqueue, 5, "tx packet");
444 KTR_INFO(KTR_IF_EM, if_em, pkt_txclean, 6, "tx clean");
445 #define logif(name) KTR_LOG(if_em_ ## name)
446
447 static __inline void
em_tx_intr(struct adapter * adapter)448 em_tx_intr(struct adapter *adapter)
449 {
450 struct ifnet *ifp = &adapter->arpcom.ac_if;
451
452 em_txeof(adapter);
453 if (!ifq_is_empty(&ifp->if_snd))
454 if_devstart(ifp);
455 }
456
457 static __inline void
em_free_txbuffer(struct adapter * adapter,struct em_buffer * tx_buffer)458 em_free_txbuffer(struct adapter *adapter, struct em_buffer *tx_buffer)
459 {
460
461 KKASSERT(tx_buffer->m_head != NULL);
462 KKASSERT(adapter->tx_nmbuf > 0);
463 adapter->tx_nmbuf--;
464
465 bus_dmamap_unload(adapter->txtag, tx_buffer->map);
466 m_freem(tx_buffer->m_head);
467 tx_buffer->m_head = NULL;
468 }
469
470 static __inline void
em_try_txgc(struct adapter * adapter,int dec)471 em_try_txgc(struct adapter *adapter, int dec)
472 {
473
474 if (adapter->tx_running > 0) {
475 adapter->tx_running -= dec;
476 if (adapter->tx_running <= 0 && adapter->tx_nmbuf &&
477 adapter->num_tx_desc_avail < adapter->num_tx_desc &&
478 adapter->num_tx_desc_avail + adapter->tx_int_nsegs >
479 adapter->num_tx_desc)
480 em_tx_collect(adapter, TRUE);
481 }
482 }
483
484 static void
em_txgc_timer(void * xadapter)485 em_txgc_timer(void *xadapter)
486 {
487 struct adapter *adapter = xadapter;
488 struct ifnet *ifp = &adapter->arpcom.ac_if;
489
490 if ((ifp->if_flags & (IFF_RUNNING | IFF_UP | IFF_NPOLLING)) !=
491 (IFF_RUNNING | IFF_UP))
492 return;
493
494 if (!lwkt_serialize_try(ifp->if_serializer))
495 goto done;
496
497 if ((ifp->if_flags & (IFF_RUNNING | IFF_UP | IFF_NPOLLING)) !=
498 (IFF_RUNNING | IFF_UP)) {
499 lwkt_serialize_exit(ifp->if_serializer);
500 return;
501 }
502 em_try_txgc(adapter, EM_TX_RUNNING_DEC);
503
504 lwkt_serialize_exit(ifp->if_serializer);
505 done:
506 callout_reset(&adapter->tx_gc_timer, 1, em_txgc_timer, adapter);
507 }
508
509 static int
em_probe(device_t dev)510 em_probe(device_t dev)
511 {
512 const struct em_vendor_info *ent;
513 uint16_t vid, did;
514
515 vid = pci_get_vendor(dev);
516 did = pci_get_device(dev);
517
518 for (ent = em_vendor_info_array; ent->desc != NULL; ++ent) {
519 if (vid == ent->vendor_id && did == ent->device_id) {
520 device_set_desc(dev, ent->desc);
521 device_set_async_attach(dev, TRUE);
522 return (ent->ret);
523 }
524 }
525 return (ENXIO);
526 }
527
528 static int
em_attach(device_t dev)529 em_attach(device_t dev)
530 {
531 struct adapter *adapter = device_get_softc(dev);
532 struct ifnet *ifp = &adapter->arpcom.ac_if;
533 int tsize, rsize;
534 int error = 0;
535 int cap;
536 uint16_t eeprom_data, device_id, apme_mask;
537 driver_intr_t *intr_func;
538 char flowctrl[IFM_ETH_FC_STRLEN];
539
540 adapter->dev = adapter->osdep.dev = dev;
541
542 /*
543 * Some versions of I219 only have PCI AF.
544 */
545 if (pci_is_pcie(dev) || pci_find_extcap(dev, PCIY_PCIAF, &cap) == 0)
546 adapter->flags |= EM_FLAG_GEN2;
547
548 callout_init_mp(&adapter->timer);
549 callout_init_mp(&adapter->tx_fifo_timer);
550 callout_init_mp(&adapter->tx_gc_timer);
551
552 ifmedia_init(&adapter->media, IFM_IMASK | IFM_ETH_FCMASK,
553 em_media_change, em_media_status);
554
555 /* Determine hardware and mac info */
556 error = em_get_hw_info(adapter);
557 if (error) {
558 device_printf(dev, "Identify hardware failed\n");
559 goto fail;
560 }
561
562 /* Setup PCI resources */
563 error = em_alloc_pci_res(adapter);
564 if (error) {
565 device_printf(dev, "Allocation of PCI resources failed\n");
566 goto fail;
567 }
568
569 /*
570 * For ICH8 and family we need to map the flash memory,
571 * and this must happen after the MAC is identified.
572 *
573 * (SPT does not map the flash with a separate BAR)
574 */
575 if (adapter->hw.mac.type == e1000_ich8lan ||
576 adapter->hw.mac.type == e1000_ich9lan ||
577 adapter->hw.mac.type == e1000_ich10lan ||
578 adapter->hw.mac.type == e1000_pchlan ||
579 adapter->hw.mac.type == e1000_pch2lan ||
580 adapter->hw.mac.type == e1000_pch_lpt) {
581 adapter->flash_rid = EM_BAR_FLASH;
582
583 adapter->flash = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
584 &adapter->flash_rid, RF_ACTIVE);
585 if (adapter->flash == NULL) {
586 device_printf(dev, "Mapping of Flash failed\n");
587 error = ENXIO;
588 goto fail;
589 }
590 adapter->osdep.flash_bus_space_tag =
591 rman_get_bustag(adapter->flash);
592 adapter->osdep.flash_bus_space_handle =
593 rman_get_bushandle(adapter->flash);
594
595 /*
596 * This is used in the shared code
597 * XXX this goof is actually not used.
598 */
599 adapter->hw.flash_address = (uint8_t *)adapter->flash;
600 } else if (adapter->hw.mac.type >= e1000_pch_spt) {
601 /*
602 * In the new SPT device flash is not a seperate BAR,
603 * rather it is also in BAR0, so use the same tag and
604 * an offset handle for the FLASH read/write macros
605 * in the shared code.
606 */
607 adapter->osdep.flash_bus_space_tag =
608 adapter->osdep.mem_bus_space_tag;
609 adapter->osdep.flash_bus_space_handle =
610 adapter->osdep.mem_bus_space_handle + E1000_FLASH_BASE_ADDR;
611 }
612
613 switch (adapter->hw.mac.type) {
614 case e1000_82571:
615 case e1000_82572:
616 case e1000_pch_lpt:
617 case e1000_pch_spt:
618 case e1000_pch_cnp:
619 /*
620 * Pullup extra 4bytes into the first data segment for
621 * TSO, see:
622 * 82571/82572 specification update errata #7
623 *
624 * Same applies to I217 (and maybe I218 and I219).
625 *
626 * NOTE:
627 * 4bytes instead of 2bytes, which are mentioned in the
628 * errata, are pulled; mainly to keep rest of the data
629 * properly aligned.
630 */
631 adapter->flags |= EM_FLAG_TSO_PULLEX;
632 /* FALL THROUGH */
633
634 default:
635 if (adapter->flags & EM_FLAG_GEN2)
636 adapter->flags |= EM_FLAG_TSO;
637 break;
638 }
639
640 /* Do Shared Code initialization */
641 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
642 device_printf(dev, "Setup of Shared code failed\n");
643 error = ENXIO;
644 goto fail;
645 }
646
647 e1000_get_bus_info(&adapter->hw);
648
649 /*
650 * Validate number of transmit and receive descriptors. It
651 * must not exceed hardware maximum, and must be multiple
652 * of E1000_DBA_ALIGN.
653 */
654 if ((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN != 0 ||
655 (adapter->hw.mac.type >= e1000_82544 && em_txd > EM_MAX_TXD) ||
656 (adapter->hw.mac.type < e1000_82544 && em_txd > EM_MAX_TXD_82543) ||
657 em_txd < EM_MIN_TXD) {
658 if (adapter->hw.mac.type < e1000_82544)
659 adapter->num_tx_desc = EM_MAX_TXD_82543;
660 else
661 adapter->num_tx_desc = EM_DEFAULT_TXD;
662 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
663 adapter->num_tx_desc, em_txd);
664 } else {
665 adapter->num_tx_desc = em_txd;
666 }
667 if ((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN != 0 ||
668 (adapter->hw.mac.type >= e1000_82544 && em_rxd > EM_MAX_RXD) ||
669 (adapter->hw.mac.type < e1000_82544 && em_rxd > EM_MAX_RXD_82543) ||
670 em_rxd < EM_MIN_RXD) {
671 if (adapter->hw.mac.type < e1000_82544)
672 adapter->num_rx_desc = EM_MAX_RXD_82543;
673 else
674 adapter->num_rx_desc = EM_DEFAULT_RXD;
675 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
676 adapter->num_rx_desc, em_rxd);
677 } else {
678 adapter->num_rx_desc = em_rxd;
679 }
680
681 adapter->hw.mac.autoneg = DO_AUTO_NEG;
682 adapter->hw.phy.autoneg_wait_to_complete = FALSE;
683 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
684 adapter->rx_buffer_len = MCLBYTES;
685
686 /*
687 * Interrupt throttle rate
688 */
689 if (em_int_throttle_ceil == 0) {
690 adapter->int_throttle_ceil = 0;
691 } else {
692 int throttle = em_int_throttle_ceil;
693
694 if (throttle < 0)
695 throttle = EM_DEFAULT_ITR;
696
697 /* Recalculate the tunable value to get the exact frequency. */
698 throttle = 1000000000 / 256 / throttle;
699
700 /* Upper 16bits of ITR is reserved and should be zero */
701 if (throttle & 0xffff0000)
702 throttle = 1000000000 / 256 / EM_DEFAULT_ITR;
703
704 adapter->int_throttle_ceil = 1000000000 / 256 / throttle;
705 }
706
707 e1000_init_script_state_82541(&adapter->hw, TRUE);
708 e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE);
709
710 /* Copper options */
711 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
712 adapter->hw.phy.mdix = AUTO_ALL_MODES;
713 adapter->hw.phy.disable_polarity_correction = FALSE;
714 adapter->hw.phy.ms_type = EM_MASTER_SLAVE;
715 }
716
717 /* Set the frame limits assuming standard ethernet sized frames. */
718 adapter->hw.mac.max_frame_size =
719 ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN;
720 adapter->min_frame_size = ETH_ZLEN + ETHER_CRC_LEN;
721
722 /* This controls when hardware reports transmit completion status. */
723 adapter->hw.mac.report_tx_early = 1;
724
725 /*
726 * Create top level busdma tag
727 */
728 error = bus_dma_tag_create(NULL, 1, 0,
729 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
730 BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT,
731 0, &adapter->parent_dtag);
732 if (error) {
733 device_printf(dev, "could not create top level DMA tag\n");
734 goto fail;
735 }
736
737 /*
738 * Allocate Transmit Descriptor ring
739 */
740 tsize = roundup2(adapter->num_tx_desc * sizeof(struct e1000_tx_desc),
741 EM_DBA_ALIGN);
742 error = em_dma_malloc(adapter, tsize, &adapter->txdma);
743 if (error) {
744 device_printf(dev, "Unable to allocate tx_desc memory\n");
745 goto fail;
746 }
747 adapter->tx_desc_base = adapter->txdma.dma_vaddr;
748
749 /*
750 * Allocate Receive Descriptor ring
751 */
752 rsize = roundup2(adapter->num_rx_desc * sizeof(struct e1000_rx_desc),
753 EM_DBA_ALIGN);
754 error = em_dma_malloc(adapter, rsize, &adapter->rxdma);
755 if (error) {
756 device_printf(dev, "Unable to allocate rx_desc memory\n");
757 goto fail;
758 }
759 adapter->rx_desc_base = adapter->rxdma.dma_vaddr;
760
761 /* Allocate multicast array memory. */
762 adapter->mta = kmalloc(ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES,
763 M_DEVBUF, M_WAITOK);
764
765 /* Indicate SOL/IDER usage */
766 if (e1000_check_reset_block(&adapter->hw)) {
767 device_printf(dev,
768 "PHY reset is blocked due to SOL/IDER session.\n");
769 }
770
771 /* Disable EEE */
772 adapter->hw.dev_spec.ich8lan.eee_disable = 1;
773
774 /*
775 * Start from a known state, this is important in reading the
776 * nvm and mac from that.
777 */
778 e1000_reset_hw(&adapter->hw);
779
780 /* Make sure we have a good EEPROM before we read from it */
781 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
782 /*
783 * Some PCI-E parts fail the first check due to
784 * the link being in sleep state, call it again,
785 * if it fails a second time its a real issue.
786 */
787 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
788 device_printf(dev,
789 "The EEPROM Checksum Is Not Valid\n");
790 error = EIO;
791 goto fail;
792 }
793 }
794
795 /* Copy the permanent MAC address out of the EEPROM */
796 if (e1000_read_mac_addr(&adapter->hw) < 0) {
797 device_printf(dev, "EEPROM read error while reading MAC"
798 " address\n");
799 error = EIO;
800 goto fail;
801 }
802 if (!em_is_valid_eaddr(adapter->hw.mac.addr)) {
803 device_printf(dev, "Invalid MAC address\n");
804 error = EIO;
805 goto fail;
806 }
807
808 /* Disable ULP support */
809 e1000_disable_ulp_lpt_lp(&adapter->hw, TRUE);
810
811 /* Allocate transmit descriptors and buffers */
812 error = em_create_tx_ring(adapter);
813 if (error) {
814 device_printf(dev, "Could not setup transmit structures\n");
815 goto fail;
816 }
817
818 /* Allocate receive descriptors and buffers */
819 error = em_create_rx_ring(adapter);
820 if (error) {
821 device_printf(dev, "Could not setup receive structures\n");
822 goto fail;
823 }
824
825 /* Manually turn off all interrupts */
826 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
827
828 /* Determine if we have to control management hardware */
829 if (e1000_enable_mng_pass_thru(&adapter->hw))
830 adapter->flags |= EM_FLAG_HAS_MGMT;
831
832 /*
833 * Setup Wake-on-Lan
834 */
835 apme_mask = EM_EEPROM_APME;
836 eeprom_data = 0;
837 switch (adapter->hw.mac.type) {
838 case e1000_82542:
839 case e1000_82543:
840 break;
841
842 case e1000_82573:
843 case e1000_82583:
844 adapter->flags |= EM_FLAG_HAS_AMT;
845 /* FALL THROUGH */
846
847 case e1000_82546:
848 case e1000_82546_rev_3:
849 case e1000_82571:
850 case e1000_82572:
851 case e1000_80003es2lan:
852 if (adapter->hw.bus.func == 1) {
853 e1000_read_nvm(&adapter->hw,
854 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
855 } else {
856 e1000_read_nvm(&adapter->hw,
857 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
858 }
859 break;
860
861 case e1000_ich8lan:
862 case e1000_ich9lan:
863 case e1000_ich10lan:
864 case e1000_pchlan:
865 case e1000_pch2lan:
866 case e1000_pch_lpt:
867 case e1000_pch_spt:
868 case e1000_pch_cnp:
869 apme_mask = E1000_WUC_APME;
870 adapter->flags |= EM_FLAG_HAS_AMT;
871 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC);
872 break;
873
874 default:
875 e1000_read_nvm(&adapter->hw,
876 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
877 break;
878 }
879 if (eeprom_data & apme_mask)
880 adapter->wol = E1000_WUFC_MAG | E1000_WUFC_MC;
881
882 /*
883 * We have the eeprom settings, now apply the special cases
884 * where the eeprom may be wrong or the board won't support
885 * wake on lan on a particular port
886 */
887 device_id = pci_get_device(dev);
888 switch (device_id) {
889 case E1000_DEV_ID_82546GB_PCIE:
890 adapter->wol = 0;
891 break;
892
893 case E1000_DEV_ID_82546EB_FIBER:
894 case E1000_DEV_ID_82546GB_FIBER:
895 case E1000_DEV_ID_82571EB_FIBER:
896 /*
897 * Wake events only supported on port A for dual fiber
898 * regardless of eeprom setting
899 */
900 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
901 E1000_STATUS_FUNC_1)
902 adapter->wol = 0;
903 break;
904
905 case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
906 case E1000_DEV_ID_82571EB_QUAD_COPPER:
907 case E1000_DEV_ID_82571EB_QUAD_FIBER:
908 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP:
909 /* if quad port adapter, disable WoL on all but port A */
910 if (em_global_quad_port_a != 0)
911 adapter->wol = 0;
912 /* Reset for multiple quad port adapters */
913 if (++em_global_quad_port_a == 4)
914 em_global_quad_port_a = 0;
915 break;
916 }
917
918 /* XXX disable wol */
919 adapter->wol = 0;
920
921 /* Setup flow control. */
922 device_getenv_string(dev, "flow_ctrl", flowctrl, sizeof(flowctrl),
923 em_flowctrl);
924 adapter->ifm_flowctrl = ifmedia_str2ethfc(flowctrl);
925 if (adapter->hw.mac.type == e1000_pchlan) {
926 /* Only PAUSE reception is supported on PCH */
927 adapter->ifm_flowctrl &= ~IFM_ETH_TXPAUSE;
928 }
929
930 /* Setup OS specific network interface */
931 em_setup_ifp(adapter);
932
933 /* Add sysctl tree, must after em_setup_ifp() */
934 em_add_sysctl(adapter);
935
936 #ifdef IFPOLL_ENABLE
937 /* Polling setup */
938 ifpoll_compat_setup(&adapter->npoll,
939 device_get_sysctl_ctx(dev), device_get_sysctl_tree(dev),
940 device_get_unit(dev), ifp->if_serializer);
941 #endif
942
943 /* Reset the hardware */
944 error = em_reset(adapter);
945 if (error) {
946 /*
947 * Some 82573 parts fail the first reset, call it again,
948 * if it fails a second time its a real issue.
949 */
950 error = em_reset(adapter);
951 if (error) {
952 device_printf(dev, "Unable to reset the hardware\n");
953 ether_ifdetach(ifp);
954 goto fail;
955 }
956 }
957
958 /* Initialize statistics */
959 em_update_stats(adapter);
960
961 adapter->hw.mac.get_link_status = 1;
962 em_update_link_status(adapter);
963
964 /* Do we need workaround for 82544 PCI-X adapter? */
965 if (adapter->hw.bus.type == e1000_bus_type_pcix &&
966 adapter->hw.mac.type == e1000_82544)
967 adapter->pcix_82544 = TRUE;
968 else
969 adapter->pcix_82544 = FALSE;
970
971 if (adapter->pcix_82544) {
972 /*
973 * 82544 on PCI-X may split one TX segment
974 * into two TX descs, so we double its number
975 * of spare TX desc here.
976 */
977 adapter->spare_tx_desc = 2 * EM_TX_SPARE;
978 } else {
979 adapter->spare_tx_desc = EM_TX_SPARE;
980 }
981 if (adapter->flags & EM_FLAG_TSO)
982 adapter->spare_tx_desc = EM_TX_SPARE_TSO;
983 adapter->tx_wreg_nsegs = EM_DEFAULT_TXWREG;
984
985 /*
986 * Keep following relationship between spare_tx_desc, oact_tx_desc
987 * and tx_int_nsegs:
988 * (spare_tx_desc + EM_TX_RESERVED) <=
989 * oact_tx_desc <= EM_TX_OACTIVE_MAX <= tx_int_nsegs
990 */
991 adapter->oact_tx_desc = adapter->num_tx_desc / 8;
992 if (adapter->oact_tx_desc > EM_TX_OACTIVE_MAX)
993 adapter->oact_tx_desc = EM_TX_OACTIVE_MAX;
994 if (adapter->oact_tx_desc < adapter->spare_tx_desc + EM_TX_RESERVED)
995 adapter->oact_tx_desc = adapter->spare_tx_desc + EM_TX_RESERVED;
996
997 adapter->tx_int_nsegs = adapter->num_tx_desc / 16;
998 if (adapter->tx_int_nsegs < adapter->oact_tx_desc)
999 adapter->tx_int_nsegs = adapter->oact_tx_desc;
1000
1001 /* Non-AMT based hardware can now take control from firmware */
1002 if ((adapter->flags & (EM_FLAG_HAS_MGMT | EM_FLAG_HAS_AMT)) ==
1003 EM_FLAG_HAS_MGMT && adapter->hw.mac.type >= e1000_82571)
1004 em_get_hw_control(adapter);
1005
1006 ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(adapter->intr_res));
1007
1008 /*
1009 * Missing Interrupt Following ICR read:
1010 *
1011 * 82571/82572 specification update errata #76
1012 * 82573 specification update errata #31
1013 * 82574 specification update errata #12
1014 * 82583 specification update errata #4
1015 */
1016 intr_func = em_intr;
1017 if ((adapter->flags & EM_FLAG_SHARED_INTR) &&
1018 (adapter->hw.mac.type == e1000_82571 ||
1019 adapter->hw.mac.type == e1000_82572 ||
1020 adapter->hw.mac.type == e1000_82573 ||
1021 adapter->hw.mac.type == e1000_82574 ||
1022 adapter->hw.mac.type == e1000_82583))
1023 intr_func = em_intr_mask;
1024
1025 error = bus_setup_intr(dev, adapter->intr_res, INTR_MPSAFE,
1026 intr_func, adapter, &adapter->intr_tag,
1027 ifp->if_serializer);
1028 if (error) {
1029 device_printf(dev, "Failed to register interrupt handler");
1030 ether_ifdetach(ifp);
1031 goto fail;
1032 }
1033 return (0);
1034 fail:
1035 em_detach(dev);
1036 return (error);
1037 }
1038
1039 static int
em_detach(device_t dev)1040 em_detach(device_t dev)
1041 {
1042 struct adapter *adapter = device_get_softc(dev);
1043
1044 if (device_is_attached(dev)) {
1045 struct ifnet *ifp = &adapter->arpcom.ac_if;
1046
1047 lwkt_serialize_enter(ifp->if_serializer);
1048
1049 em_stop(adapter);
1050
1051 e1000_phy_hw_reset(&adapter->hw);
1052
1053 em_rel_mgmt(adapter);
1054 em_rel_hw_control(adapter);
1055
1056 if (adapter->wol) {
1057 E1000_WRITE_REG(&adapter->hw, E1000_WUC,
1058 E1000_WUC_PME_EN);
1059 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
1060 em_enable_wol(dev);
1061 }
1062
1063 bus_teardown_intr(dev, adapter->intr_res, adapter->intr_tag);
1064
1065 lwkt_serialize_exit(ifp->if_serializer);
1066
1067 ether_ifdetach(ifp);
1068 } else if (adapter->memory != NULL) {
1069 em_rel_hw_control(adapter);
1070 }
1071
1072 ifmedia_removeall(&adapter->media);
1073 bus_generic_detach(dev);
1074
1075 em_free_pci_res(adapter);
1076
1077 em_destroy_tx_ring(adapter, adapter->num_tx_desc);
1078 em_destroy_rx_ring(adapter, adapter->num_rx_desc);
1079
1080 /* Free Transmit Descriptor ring */
1081 if (adapter->tx_desc_base)
1082 em_dma_free(adapter, &adapter->txdma);
1083
1084 /* Free Receive Descriptor ring */
1085 if (adapter->rx_desc_base)
1086 em_dma_free(adapter, &adapter->rxdma);
1087
1088 /* Free top level busdma tag */
1089 if (adapter->parent_dtag != NULL)
1090 bus_dma_tag_destroy(adapter->parent_dtag);
1091
1092 if (adapter->mta != NULL)
1093 kfree(adapter->mta, M_DEVBUF);
1094
1095 return (0);
1096 }
1097
1098 static int
em_shutdown(device_t dev)1099 em_shutdown(device_t dev)
1100 {
1101 return em_suspend(dev);
1102 }
1103
1104 static int
em_suspend(device_t dev)1105 em_suspend(device_t dev)
1106 {
1107 struct adapter *adapter = device_get_softc(dev);
1108 struct ifnet *ifp = &adapter->arpcom.ac_if;
1109
1110 lwkt_serialize_enter(ifp->if_serializer);
1111
1112 em_stop(adapter);
1113
1114 em_rel_mgmt(adapter);
1115 em_rel_hw_control(adapter);
1116
1117 if (adapter->wol) {
1118 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
1119 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
1120 em_enable_wol(dev);
1121 }
1122
1123 lwkt_serialize_exit(ifp->if_serializer);
1124
1125 return bus_generic_suspend(dev);
1126 }
1127
1128 static int
em_resume(device_t dev)1129 em_resume(device_t dev)
1130 {
1131 struct adapter *adapter = device_get_softc(dev);
1132 struct ifnet *ifp = &adapter->arpcom.ac_if;
1133
1134 lwkt_serialize_enter(ifp->if_serializer);
1135
1136 if (adapter->hw.mac.type == e1000_pch2lan)
1137 e1000_resume_workarounds_pchlan(&adapter->hw);
1138
1139 em_init(adapter);
1140 em_get_mgmt(adapter);
1141 if_devstart(ifp);
1142
1143 lwkt_serialize_exit(ifp->if_serializer);
1144
1145 return bus_generic_resume(dev);
1146 }
1147
1148 static void
em_start(struct ifnet * ifp,struct ifaltq_subque * ifsq)1149 em_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
1150 {
1151 struct adapter *adapter = ifp->if_softc;
1152 struct mbuf *m_head;
1153 int idx = -1, nsegs = 0;
1154
1155 ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq);
1156 ASSERT_SERIALIZED(ifp->if_serializer);
1157
1158 if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd))
1159 return;
1160
1161 if (!adapter->link_active) {
1162 ifq_purge(&ifp->if_snd);
1163 return;
1164 }
1165
1166 while (!ifq_is_empty(&ifp->if_snd)) {
1167 /* Now do we at least have a minimal? */
1168 if (EM_IS_OACTIVE(adapter)) {
1169 em_tx_collect(adapter, FALSE);
1170 if (EM_IS_OACTIVE(adapter)) {
1171 ifq_set_oactive(&ifp->if_snd);
1172 adapter->no_tx_desc_avail1++;
1173 break;
1174 }
1175 }
1176
1177 logif(pkt_txqueue);
1178 m_head = ifq_dequeue(&ifp->if_snd);
1179 if (m_head == NULL)
1180 break;
1181
1182 if (em_encap(adapter, &m_head, &nsegs, &idx)) {
1183 IFNET_STAT_INC(ifp, oerrors, 1);
1184 em_tx_collect(adapter, FALSE);
1185 continue;
1186 }
1187
1188 /*
1189 * TX interrupt are aggressively aggregated, so increasing
1190 * opackets at TX interrupt time will make the opackets
1191 * statistics vastly inaccurate; we do the opackets increment
1192 * now.
1193 */
1194 IFNET_STAT_INC(ifp, opackets, 1);
1195
1196 if (nsegs >= adapter->tx_wreg_nsegs && idx >= 0) {
1197 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), idx);
1198 nsegs = 0;
1199 idx = -1;
1200 }
1201
1202 /* Send a copy of the frame to the BPF listener */
1203 ETHER_BPF_MTAP(ifp, m_head);
1204
1205 /* Set timeout in case hardware has problems transmitting. */
1206 ifp->if_timer = EM_TX_TIMEOUT;
1207 }
1208 if (idx >= 0)
1209 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), idx);
1210 adapter->tx_running = EM_TX_RUNNING;
1211 }
1212
1213 static int
em_ioctl(struct ifnet * ifp,u_long command,caddr_t data,struct ucred * cr)1214 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
1215 {
1216 struct adapter *adapter = ifp->if_softc;
1217 struct ifreq *ifr = (struct ifreq *)data;
1218 uint16_t eeprom_data = 0;
1219 int max_frame_size, mask, reinit;
1220 int error = 0;
1221
1222 ASSERT_SERIALIZED(ifp->if_serializer);
1223
1224 switch (command) {
1225 case SIOCSIFMTU:
1226 switch (adapter->hw.mac.type) {
1227 case e1000_82573:
1228 /*
1229 * 82573 only supports jumbo frames
1230 * if ASPM is disabled.
1231 */
1232 e1000_read_nvm(&adapter->hw,
1233 NVM_INIT_3GIO_3, 1, &eeprom_data);
1234 if (eeprom_data & NVM_WORD1A_ASPM_MASK) {
1235 max_frame_size = ETHER_MAX_LEN;
1236 break;
1237 }
1238 /* FALL THROUGH */
1239
1240 /* Limit Jumbo Frame size */
1241 case e1000_82571:
1242 case e1000_82572:
1243 case e1000_ich9lan:
1244 case e1000_ich10lan:
1245 case e1000_pch2lan:
1246 case e1000_pch_lpt:
1247 case e1000_pch_spt:
1248 case e1000_pch_cnp:
1249 case e1000_82574:
1250 case e1000_82583:
1251 case e1000_80003es2lan:
1252 max_frame_size = 9234;
1253 break;
1254
1255 case e1000_pchlan:
1256 max_frame_size = 4096;
1257 break;
1258
1259 /* Adapters that do not support jumbo frames */
1260 case e1000_82542:
1261 case e1000_ich8lan:
1262 max_frame_size = ETHER_MAX_LEN;
1263 break;
1264
1265 default:
1266 max_frame_size = MAX_JUMBO_FRAME_SIZE;
1267 break;
1268 }
1269 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1270 ETHER_CRC_LEN) {
1271 error = EINVAL;
1272 break;
1273 }
1274
1275 ifp->if_mtu = ifr->ifr_mtu;
1276 adapter->hw.mac.max_frame_size =
1277 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1278
1279 if (ifp->if_flags & IFF_RUNNING)
1280 em_init(adapter);
1281 break;
1282
1283 case SIOCSIFFLAGS:
1284 if (ifp->if_flags & IFF_UP) {
1285 if ((ifp->if_flags & IFF_RUNNING)) {
1286 if ((ifp->if_flags ^ adapter->if_flags) &
1287 (IFF_PROMISC | IFF_ALLMULTI)) {
1288 em_disable_promisc(adapter);
1289 em_set_promisc(adapter);
1290 }
1291 } else {
1292 em_init(adapter);
1293 }
1294 } else if (ifp->if_flags & IFF_RUNNING) {
1295 em_stop(adapter);
1296 }
1297 adapter->if_flags = ifp->if_flags;
1298 break;
1299
1300 case SIOCADDMULTI:
1301 case SIOCDELMULTI:
1302 if (ifp->if_flags & IFF_RUNNING) {
1303 em_disable_intr(adapter);
1304 em_set_multi(adapter);
1305 if (adapter->hw.mac.type == e1000_82542 &&
1306 adapter->hw.revision_id == E1000_REVISION_2)
1307 em_init_rx_unit(adapter);
1308 #ifdef IFPOLL_ENABLE
1309 if (!(ifp->if_flags & IFF_NPOLLING))
1310 #endif
1311 em_enable_intr(adapter);
1312 }
1313 break;
1314
1315 case SIOCSIFMEDIA:
1316 /* Check SOL/IDER usage */
1317 if (e1000_check_reset_block(&adapter->hw)) {
1318 device_printf(adapter->dev, "Media change is"
1319 " blocked due to SOL/IDER session.\n");
1320 break;
1321 }
1322 /* FALL THROUGH */
1323
1324 case SIOCGIFMEDIA:
1325 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1326 break;
1327
1328 case SIOCSIFCAP:
1329 reinit = 0;
1330 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1331 if (mask & IFCAP_RXCSUM) {
1332 ifp->if_capenable ^= IFCAP_RXCSUM;
1333 reinit = 1;
1334 }
1335 if (mask & IFCAP_TXCSUM) {
1336 ifp->if_capenable ^= IFCAP_TXCSUM;
1337 if (ifp->if_capenable & IFCAP_TXCSUM)
1338 ifp->if_hwassist |= EM_CSUM_FEATURES;
1339 else
1340 ifp->if_hwassist &= ~EM_CSUM_FEATURES;
1341 }
1342 if (mask & IFCAP_TSO) {
1343 ifp->if_capenable ^= IFCAP_TSO;
1344 if (ifp->if_capenable & IFCAP_TSO)
1345 ifp->if_hwassist |= CSUM_TSO;
1346 else
1347 ifp->if_hwassist &= ~CSUM_TSO;
1348 }
1349 if (mask & IFCAP_VLAN_HWTAGGING) {
1350 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1351 reinit = 1;
1352 }
1353 if (reinit && (ifp->if_flags & IFF_RUNNING))
1354 em_init(adapter);
1355 break;
1356
1357 default:
1358 error = ether_ioctl(ifp, command, data);
1359 break;
1360 }
1361 return (error);
1362 }
1363
1364 static void
em_watchdog(struct ifnet * ifp)1365 em_watchdog(struct ifnet *ifp)
1366 {
1367 struct adapter *adapter = ifp->if_softc;
1368
1369 ASSERT_SERIALIZED(ifp->if_serializer);
1370
1371 /*
1372 * The timer is set to 5 every time start queues a packet.
1373 * Then txeof keeps resetting it as long as it cleans at
1374 * least one descriptor.
1375 * Finally, anytime all descriptors are clean the timer is
1376 * set to 0.
1377 */
1378
1379 if (E1000_READ_REG(&adapter->hw, E1000_TDT(0)) ==
1380 E1000_READ_REG(&adapter->hw, E1000_TDH(0))) {
1381 /*
1382 * If we reach here, all TX jobs are completed and
1383 * the TX engine should have been idled for some time.
1384 * We don't need to call if_devstart() here.
1385 */
1386 ifq_clr_oactive(&ifp->if_snd);
1387 ifp->if_timer = 0;
1388 return;
1389 }
1390
1391 /*
1392 * If we are in this routine because of pause frames, then
1393 * don't reset the hardware.
1394 */
1395 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
1396 E1000_STATUS_TXOFF) {
1397 ifp->if_timer = EM_TX_TIMEOUT;
1398 return;
1399 }
1400
1401 if (e1000_check_for_link(&adapter->hw) == 0)
1402 if_printf(ifp, "watchdog timeout -- resetting\n");
1403
1404 IFNET_STAT_INC(ifp, oerrors, 1);
1405 adapter->watchdog_events++;
1406
1407 em_init(adapter);
1408
1409 if (!ifq_is_empty(&ifp->if_snd))
1410 if_devstart(ifp);
1411 }
1412
1413 static void
em_init(void * xsc)1414 em_init(void *xsc)
1415 {
1416 struct adapter *adapter = xsc;
1417 struct ifnet *ifp = &adapter->arpcom.ac_if;
1418 device_t dev = adapter->dev;
1419
1420 ASSERT_SERIALIZED(ifp->if_serializer);
1421
1422 em_stop(adapter);
1423
1424 /* Get the latest mac address, User can use a LAA */
1425 bcopy(IF_LLADDR(ifp), adapter->hw.mac.addr, ETHER_ADDR_LEN);
1426
1427 /* Put the address into the Receive Address Array */
1428 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1429
1430 /*
1431 * With the 82571 adapter, RAR[0] may be overwritten
1432 * when the other port is reset, we make a duplicate
1433 * in RAR[14] for that eventuality, this assures
1434 * the interface continues to function.
1435 */
1436 if (adapter->hw.mac.type == e1000_82571) {
1437 e1000_set_laa_state_82571(&adapter->hw, TRUE);
1438 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr,
1439 E1000_RAR_ENTRIES - 1);
1440 }
1441
1442 /* Reset the hardware */
1443 if (em_reset(adapter)) {
1444 device_printf(dev, "Unable to reset the hardware\n");
1445 /* XXX em_stop()? */
1446 return;
1447 }
1448 em_update_link_status(adapter);
1449
1450 /* Setup VLAN support, basic and offload if available */
1451 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1452
1453 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1454 uint32_t ctrl;
1455
1456 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
1457 ctrl |= E1000_CTRL_VME;
1458 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
1459 }
1460
1461 /* Configure for OS presence */
1462 em_get_mgmt(adapter);
1463
1464 /* Prepare transmit descriptors and buffers */
1465 em_init_tx_ring(adapter);
1466 em_init_tx_unit(adapter);
1467
1468 /* Setup Multicast table */
1469 em_set_multi(adapter);
1470
1471 /* Prepare receive descriptors and buffers */
1472 if (em_init_rx_ring(adapter)) {
1473 device_printf(dev, "Could not setup receive structures\n");
1474 em_stop(adapter);
1475 return;
1476 }
1477 em_init_rx_unit(adapter);
1478
1479 /* Don't lose promiscuous settings */
1480 em_set_promisc(adapter);
1481
1482 /* Reset hardware counters */
1483 e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1484
1485 /* MSI/X configuration for 82574 */
1486 if (adapter->hw.mac.type == e1000_82574) {
1487 int tmp;
1488
1489 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
1490 tmp |= E1000_CTRL_EXT_PBA_CLR;
1491 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp);
1492 /*
1493 * XXX MSIX
1494 * Set the IVAR - interrupt vector routing.
1495 * Each nibble represents a vector, high bit
1496 * is enable, other 3 bits are the MSIX table
1497 * entry, we map RXQ0 to 0, TXQ0 to 1, and
1498 * Link (other) to 2, hence the magic number.
1499 */
1500 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, 0x800A0908);
1501 }
1502
1503 #ifdef IFPOLL_ENABLE
1504 /*
1505 * Only enable interrupts if we are not polling, make sure
1506 * they are off otherwise.
1507 */
1508 if (ifp->if_flags & IFF_NPOLLING)
1509 em_disable_intr(adapter);
1510 else
1511 #endif /* IFPOLL_ENABLE */
1512 em_enable_intr(adapter);
1513
1514 /* AMT based hardware can now take control from firmware */
1515 if ((adapter->flags & (EM_FLAG_HAS_MGMT | EM_FLAG_HAS_AMT)) ==
1516 (EM_FLAG_HAS_MGMT | EM_FLAG_HAS_AMT) &&
1517 adapter->hw.mac.type >= e1000_82571)
1518 em_get_hw_control(adapter);
1519
1520 ifp->if_flags |= IFF_RUNNING;
1521 ifq_clr_oactive(&ifp->if_snd);
1522
1523 #ifdef IFPOLL_ENABLE
1524 if ((ifp->if_flags & IFF_NPOLLING) == 0)
1525 #endif
1526 {
1527 callout_reset_bycpu(&adapter->tx_gc_timer, 1,
1528 em_txgc_timer, adapter,
1529 rman_get_cpuid(adapter->intr_res));
1530 }
1531 callout_reset(&adapter->timer, hz, em_timer, adapter);
1532 }
1533
1534 #ifdef IFPOLL_ENABLE
1535
1536 static void
em_npoll_compat(struct ifnet * ifp,void * arg __unused,int count)1537 em_npoll_compat(struct ifnet *ifp, void *arg __unused, int count)
1538 {
1539 struct adapter *adapter = ifp->if_softc;
1540
1541 ASSERT_SERIALIZED(ifp->if_serializer);
1542
1543 if (adapter->npoll.ifpc_stcount-- == 0) {
1544 uint32_t reg_icr;
1545
1546 adapter->npoll.ifpc_stcount = adapter->npoll.ifpc_stfrac;
1547
1548 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1549 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1550 callout_stop(&adapter->timer);
1551 adapter->hw.mac.get_link_status = 1;
1552 em_update_link_status(adapter);
1553 callout_reset(&adapter->timer, hz, em_timer, adapter);
1554 }
1555 }
1556
1557 em_rxeof(adapter, count);
1558
1559 em_tx_intr(adapter);
1560 em_try_txgc(adapter, 1);
1561 }
1562
1563 static void
em_npoll(struct ifnet * ifp,struct ifpoll_info * info)1564 em_npoll(struct ifnet *ifp, struct ifpoll_info *info)
1565 {
1566 struct adapter *adapter = ifp->if_softc;
1567
1568 ASSERT_SERIALIZED(ifp->if_serializer);
1569
1570 if (info != NULL) {
1571 int cpuid = adapter->npoll.ifpc_cpuid;
1572
1573 info->ifpi_rx[cpuid].poll_func = em_npoll_compat;
1574 info->ifpi_rx[cpuid].arg = NULL;
1575 info->ifpi_rx[cpuid].serializer = ifp->if_serializer;
1576
1577 ifq_set_cpuid(&ifp->if_snd, cpuid);
1578 } else {
1579 ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(adapter->intr_res));
1580 }
1581 if (ifp->if_flags & IFF_RUNNING)
1582 em_init(adapter);
1583 }
1584
1585 #endif /* IFPOLL_ENABLE */
1586
1587 static void
em_intr(void * xsc)1588 em_intr(void *xsc)
1589 {
1590 em_intr_body(xsc, TRUE);
1591 }
1592
1593 static void
em_intr_body(struct adapter * adapter,boolean_t chk_asserted)1594 em_intr_body(struct adapter *adapter, boolean_t chk_asserted)
1595 {
1596 struct ifnet *ifp = &adapter->arpcom.ac_if;
1597 uint32_t reg_icr;
1598
1599 logif(intr_beg);
1600 ASSERT_SERIALIZED(ifp->if_serializer);
1601
1602 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1603
1604 if (chk_asserted &&
1605 ((adapter->hw.mac.type >= e1000_82571 &&
1606 (reg_icr & E1000_ICR_INT_ASSERTED) == 0) ||
1607 reg_icr == 0)) {
1608 logif(intr_end);
1609 return;
1610 }
1611
1612 /*
1613 * XXX: some laptops trigger several spurious interrupts
1614 * on em(4) when in the resume cycle. The ICR register
1615 * reports all-ones value in this case. Processing such
1616 * interrupts would lead to a freeze. I don't know why.
1617 */
1618 if (reg_icr == 0xffffffff) {
1619 logif(intr_end);
1620 return;
1621 }
1622
1623 if (ifp->if_flags & IFF_RUNNING) {
1624 if (reg_icr &
1625 (E1000_ICR_RXT0 | E1000_ICR_RXDMT0 | E1000_ICR_RXO))
1626 em_rxeof(adapter, -1);
1627 if (reg_icr & E1000_ICR_TXDW)
1628 em_tx_intr(adapter);
1629 }
1630
1631 /* Link status change */
1632 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1633 callout_stop(&adapter->timer);
1634 adapter->hw.mac.get_link_status = 1;
1635 em_update_link_status(adapter);
1636
1637 /* Deal with TX cruft when link lost */
1638 em_tx_purge(adapter);
1639
1640 callout_reset(&adapter->timer, hz, em_timer, adapter);
1641 }
1642
1643 if (reg_icr & E1000_ICR_RXO)
1644 adapter->rx_overruns++;
1645
1646 logif(intr_end);
1647 }
1648
1649 static void
em_intr_mask(void * xsc)1650 em_intr_mask(void *xsc)
1651 {
1652 struct adapter *adapter = xsc;
1653
1654 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
1655 /*
1656 * NOTE:
1657 * ICR.INT_ASSERTED bit will never be set if IMS is 0,
1658 * so don't check it.
1659 */
1660 em_intr_body(adapter, FALSE);
1661 E1000_WRITE_REG(&adapter->hw, E1000_IMS, IMS_ENABLE_MASK);
1662 }
1663
1664 static void
em_media_status(struct ifnet * ifp,struct ifmediareq * ifmr)1665 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1666 {
1667 struct adapter *adapter = ifp->if_softc;
1668
1669 ASSERT_SERIALIZED(ifp->if_serializer);
1670
1671 em_update_link_status(adapter);
1672
1673 ifmr->ifm_status = IFM_AVALID;
1674 ifmr->ifm_active = IFM_ETHER;
1675
1676 if (!adapter->link_active) {
1677 if (adapter->hw.mac.autoneg)
1678 ifmr->ifm_active |= IFM_NONE;
1679 else
1680 ifmr->ifm_active = adapter->media.ifm_media;
1681 return;
1682 }
1683
1684 ifmr->ifm_status |= IFM_ACTIVE;
1685 if (adapter->ifm_flowctrl & IFM_ETH_FORCEPAUSE)
1686 ifmr->ifm_active |= adapter->ifm_flowctrl;
1687
1688 if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
1689 adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
1690 u_char fiber_type = IFM_1000_SX;
1691
1692 if (adapter->hw.mac.type == e1000_82545)
1693 fiber_type = IFM_1000_LX;
1694 ifmr->ifm_active |= fiber_type | IFM_FDX;
1695 } else {
1696 switch (adapter->link_speed) {
1697 case 10:
1698 ifmr->ifm_active |= IFM_10_T;
1699 break;
1700 case 100:
1701 ifmr->ifm_active |= IFM_100_TX;
1702 break;
1703
1704 case 1000:
1705 ifmr->ifm_active |= IFM_1000_T;
1706 break;
1707 }
1708 if (adapter->link_duplex == FULL_DUPLEX)
1709 ifmr->ifm_active |= IFM_FDX;
1710 else
1711 ifmr->ifm_active |= IFM_HDX;
1712 }
1713 if (ifmr->ifm_active & IFM_FDX) {
1714 ifmr->ifm_active |=
1715 e1000_fc2ifmedia(adapter->hw.fc.current_mode);
1716 }
1717 }
1718
1719 static int
em_media_change(struct ifnet * ifp)1720 em_media_change(struct ifnet *ifp)
1721 {
1722 struct adapter *adapter = ifp->if_softc;
1723 struct ifmedia *ifm = &adapter->media;
1724
1725 ASSERT_SERIALIZED(ifp->if_serializer);
1726
1727 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1728 return (EINVAL);
1729
1730 if (adapter->hw.mac.type == e1000_pchlan &&
1731 (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_TXPAUSE)) {
1732 if (bootverbose)
1733 if_printf(ifp, "TX PAUSE is not supported on PCH\n");
1734 return EINVAL;
1735 }
1736
1737 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1738 case IFM_AUTO:
1739 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1740 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1741 break;
1742
1743 case IFM_1000_LX:
1744 case IFM_1000_SX:
1745 case IFM_1000_T:
1746 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1747 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1748 break;
1749
1750 case IFM_100_TX:
1751 if (IFM_OPTIONS(ifm->ifm_media) & IFM_FDX) {
1752 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1753 } else {
1754 if (IFM_OPTIONS(ifm->ifm_media) &
1755 (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE)) {
1756 if (bootverbose) {
1757 if_printf(ifp, "Flow control is not "
1758 "allowed for half-duplex\n");
1759 }
1760 return EINVAL;
1761 }
1762 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1763 }
1764 adapter->hw.mac.autoneg = FALSE;
1765 adapter->hw.phy.autoneg_advertised = 0;
1766 break;
1767
1768 case IFM_10_T:
1769 if (IFM_OPTIONS(ifm->ifm_media) & IFM_FDX) {
1770 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1771 } else {
1772 if (IFM_OPTIONS(ifm->ifm_media) &
1773 (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE)) {
1774 if (bootverbose) {
1775 if_printf(ifp, "Flow control is not "
1776 "allowed for half-duplex\n");
1777 }
1778 return EINVAL;
1779 }
1780 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1781 }
1782 adapter->hw.mac.autoneg = FALSE;
1783 adapter->hw.phy.autoneg_advertised = 0;
1784 break;
1785
1786 default:
1787 if (bootverbose) {
1788 if_printf(ifp, "Unsupported media type %d\n",
1789 IFM_SUBTYPE(ifm->ifm_media));
1790 }
1791 return EINVAL;
1792 }
1793 adapter->ifm_flowctrl = ifm->ifm_media & IFM_ETH_FCMASK;
1794
1795 if (ifp->if_flags & IFF_RUNNING)
1796 em_init(adapter);
1797
1798 return (0);
1799 }
1800
1801 static int
em_encap(struct adapter * adapter,struct mbuf ** m_headp,int * segs_used,int * idx)1802 em_encap(struct adapter *adapter, struct mbuf **m_headp,
1803 int *segs_used, int *idx)
1804 {
1805 bus_dma_segment_t segs[EM_MAX_SCATTER];
1806 bus_dmamap_t map;
1807 struct em_buffer *tx_buffer, *tx_buffer_mapped;
1808 struct e1000_tx_desc *ctxd = NULL;
1809 struct mbuf *m_head = *m_headp;
1810 uint32_t txd_upper, txd_lower, txd_used, cmd = 0;
1811 int maxsegs, nsegs, i, j, first, last = 0, error;
1812
1813 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1814 error = em_tso_pullup(adapter, m_headp);
1815 if (error)
1816 return error;
1817 m_head = *m_headp;
1818 }
1819
1820 txd_upper = txd_lower = 0;
1821 txd_used = 0;
1822
1823 /*
1824 * Capture the first descriptor index, this descriptor
1825 * will have the index of the EOP which is the only one
1826 * that now gets a DONE bit writeback.
1827 */
1828 first = adapter->next_avail_tx_desc;
1829 tx_buffer = &adapter->tx_buffer_area[first];
1830 tx_buffer_mapped = tx_buffer;
1831 map = tx_buffer->map;
1832
1833 maxsegs = adapter->num_tx_desc_avail - EM_TX_RESERVED;
1834 KASSERT(maxsegs >= adapter->spare_tx_desc,
1835 ("not enough spare TX desc"));
1836 if (adapter->pcix_82544) {
1837 /* Half it; see the comment in em_attach() */
1838 maxsegs >>= 1;
1839 }
1840 if (maxsegs > EM_MAX_SCATTER)
1841 maxsegs = EM_MAX_SCATTER;
1842
1843 error = bus_dmamap_load_mbuf_defrag(adapter->txtag, map, m_headp,
1844 segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
1845 if (error) {
1846 if (error == ENOBUFS)
1847 adapter->mbuf_alloc_failed++;
1848 else
1849 adapter->no_tx_dma_setup++;
1850
1851 m_freem(*m_headp);
1852 *m_headp = NULL;
1853 return error;
1854 }
1855 bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE);
1856
1857 m_head = *m_headp;
1858 adapter->tx_nsegs += nsegs;
1859 *segs_used += nsegs;
1860
1861 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1862 /* TSO will consume one TX desc */
1863 i = em_tso_setup(adapter, m_head, &txd_upper, &txd_lower);
1864 adapter->tx_nsegs += i;
1865 *segs_used += i;
1866 } else if (m_head->m_pkthdr.csum_flags & EM_CSUM_FEATURES) {
1867 /* TX csum offloading will consume one TX desc */
1868 i = em_txcsum(adapter, m_head, &txd_upper, &txd_lower);
1869 adapter->tx_nsegs += i;
1870 *segs_used += i;
1871 }
1872
1873 /* Handle VLAN tag */
1874 if (m_head->m_flags & M_VLANTAG) {
1875 /* Set the vlan id. */
1876 txd_upper |= (htole16(m_head->m_pkthdr.ether_vlantag) << 16);
1877 /* Tell hardware to add tag */
1878 txd_lower |= htole32(E1000_TXD_CMD_VLE);
1879 }
1880
1881 i = adapter->next_avail_tx_desc;
1882
1883 /* Set up our transmit descriptors */
1884 for (j = 0; j < nsegs; j++) {
1885 /* If adapter is 82544 and on PCIX bus */
1886 if(adapter->pcix_82544) {
1887 DESC_ARRAY desc_array;
1888 uint32_t array_elements, counter;
1889
1890 /*
1891 * Check the Address and Length combination and
1892 * split the data accordingly
1893 */
1894 array_elements = em_82544_fill_desc(segs[j].ds_addr,
1895 segs[j].ds_len, &desc_array);
1896 for (counter = 0; counter < array_elements; counter++) {
1897 KKASSERT(txd_used < adapter->num_tx_desc_avail);
1898
1899 tx_buffer = &adapter->tx_buffer_area[i];
1900 ctxd = &adapter->tx_desc_base[i];
1901
1902 ctxd->buffer_addr = htole64(
1903 desc_array.descriptor[counter].address);
1904 ctxd->lower.data = htole32(
1905 E1000_TXD_CMD_IFCS | txd_lower |
1906 desc_array.descriptor[counter].length);
1907 ctxd->upper.data = htole32(txd_upper);
1908
1909 last = i;
1910 if (++i == adapter->num_tx_desc)
1911 i = 0;
1912
1913 txd_used++;
1914 }
1915 } else {
1916 tx_buffer = &adapter->tx_buffer_area[i];
1917 ctxd = &adapter->tx_desc_base[i];
1918
1919 ctxd->buffer_addr = htole64(segs[j].ds_addr);
1920 ctxd->lower.data = htole32(E1000_TXD_CMD_IFCS |
1921 txd_lower | segs[j].ds_len);
1922 ctxd->upper.data = htole32(txd_upper);
1923
1924 last = i;
1925 if (++i == adapter->num_tx_desc)
1926 i = 0;
1927 }
1928 }
1929
1930 adapter->next_avail_tx_desc = i;
1931 if (adapter->pcix_82544) {
1932 KKASSERT(adapter->num_tx_desc_avail > txd_used);
1933 adapter->num_tx_desc_avail -= txd_used;
1934 } else {
1935 KKASSERT(adapter->num_tx_desc_avail > nsegs);
1936 adapter->num_tx_desc_avail -= nsegs;
1937 }
1938 adapter->tx_nmbuf++;
1939
1940 tx_buffer->m_head = m_head;
1941 tx_buffer_mapped->map = tx_buffer->map;
1942 tx_buffer->map = map;
1943
1944 if (adapter->tx_nsegs >= adapter->tx_int_nsegs) {
1945 adapter->tx_nsegs = 0;
1946
1947 /*
1948 * Report Status (RS) is turned on
1949 * every tx_int_nsegs descriptors.
1950 */
1951 cmd = E1000_TXD_CMD_RS;
1952
1953 /*
1954 * Keep track of the descriptor, which will
1955 * be written back by hardware.
1956 */
1957 adapter->tx_dd[adapter->tx_dd_tail] = last;
1958 EM_INC_TXDD_IDX(adapter->tx_dd_tail);
1959 KKASSERT(adapter->tx_dd_tail != adapter->tx_dd_head);
1960 }
1961
1962 /*
1963 * Last Descriptor of Packet needs End Of Packet (EOP)
1964 */
1965 ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | cmd);
1966
1967 if (adapter->hw.mac.type == e1000_82547) {
1968 /*
1969 * Advance the Transmit Descriptor Tail (TDT), this tells the
1970 * E1000 that this frame is available to transmit.
1971 */
1972 if (adapter->link_duplex == HALF_DUPLEX) {
1973 em_82547_move_tail_serialized(adapter);
1974 } else {
1975 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i);
1976 em_82547_update_fifo_head(adapter,
1977 m_head->m_pkthdr.len);
1978 }
1979 } else {
1980 /*
1981 * Defer TDT updating, until enough descriptors are setup
1982 */
1983 *idx = i;
1984 }
1985 return (0);
1986 }
1987
1988 /*
1989 * 82547 workaround to avoid controller hang in half-duplex environment.
1990 * The workaround is to avoid queuing a large packet that would span
1991 * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
1992 * in this case. We do that only when FIFO is quiescent.
1993 */
1994 static void
em_82547_move_tail_serialized(struct adapter * adapter)1995 em_82547_move_tail_serialized(struct adapter *adapter)
1996 {
1997 struct e1000_tx_desc *tx_desc;
1998 uint16_t hw_tdt, sw_tdt, length = 0;
1999 bool eop = 0;
2000
2001 ASSERT_SERIALIZED(adapter->arpcom.ac_if.if_serializer);
2002
2003 hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0));
2004 sw_tdt = adapter->next_avail_tx_desc;
2005
2006 while (hw_tdt != sw_tdt) {
2007 tx_desc = &adapter->tx_desc_base[hw_tdt];
2008 length += tx_desc->lower.flags.length;
2009 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
2010 if (++hw_tdt == adapter->num_tx_desc)
2011 hw_tdt = 0;
2012
2013 if (eop) {
2014 if (em_82547_fifo_workaround(adapter, length)) {
2015 adapter->tx_fifo_wrk_cnt++;
2016 callout_reset(&adapter->tx_fifo_timer, 1,
2017 em_82547_move_tail, adapter);
2018 break;
2019 }
2020 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt);
2021 em_82547_update_fifo_head(adapter, length);
2022 length = 0;
2023 }
2024 }
2025 }
2026
2027 static void
em_82547_move_tail(void * xsc)2028 em_82547_move_tail(void *xsc)
2029 {
2030 struct adapter *adapter = xsc;
2031 struct ifnet *ifp = &adapter->arpcom.ac_if;
2032
2033 lwkt_serialize_enter(ifp->if_serializer);
2034 em_82547_move_tail_serialized(adapter);
2035 lwkt_serialize_exit(ifp->if_serializer);
2036 }
2037
2038 static int
em_82547_fifo_workaround(struct adapter * adapter,int len)2039 em_82547_fifo_workaround(struct adapter *adapter, int len)
2040 {
2041 int fifo_space, fifo_pkt_len;
2042
2043 fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
2044
2045 if (adapter->link_duplex == HALF_DUPLEX) {
2046 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
2047
2048 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
2049 if (em_82547_tx_fifo_reset(adapter))
2050 return (0);
2051 else
2052 return (1);
2053 }
2054 }
2055 return (0);
2056 }
2057
2058 static void
em_82547_update_fifo_head(struct adapter * adapter,int len)2059 em_82547_update_fifo_head(struct adapter *adapter, int len)
2060 {
2061 int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
2062
2063 /* tx_fifo_head is always 16 byte aligned */
2064 adapter->tx_fifo_head += fifo_pkt_len;
2065 if (adapter->tx_fifo_head >= adapter->tx_fifo_size)
2066 adapter->tx_fifo_head -= adapter->tx_fifo_size;
2067 }
2068
2069 static int
em_82547_tx_fifo_reset(struct adapter * adapter)2070 em_82547_tx_fifo_reset(struct adapter *adapter)
2071 {
2072 uint32_t tctl;
2073
2074 if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) ==
2075 E1000_READ_REG(&adapter->hw, E1000_TDH(0))) &&
2076 (E1000_READ_REG(&adapter->hw, E1000_TDFT) ==
2077 E1000_READ_REG(&adapter->hw, E1000_TDFH)) &&
2078 (E1000_READ_REG(&adapter->hw, E1000_TDFTS) ==
2079 E1000_READ_REG(&adapter->hw, E1000_TDFHS)) &&
2080 (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) {
2081 /* Disable TX unit */
2082 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
2083 E1000_WRITE_REG(&adapter->hw, E1000_TCTL,
2084 tctl & ~E1000_TCTL_EN);
2085
2086 /* Reset FIFO pointers */
2087 E1000_WRITE_REG(&adapter->hw, E1000_TDFT,
2088 adapter->tx_head_addr);
2089 E1000_WRITE_REG(&adapter->hw, E1000_TDFH,
2090 adapter->tx_head_addr);
2091 E1000_WRITE_REG(&adapter->hw, E1000_TDFTS,
2092 adapter->tx_head_addr);
2093 E1000_WRITE_REG(&adapter->hw, E1000_TDFHS,
2094 adapter->tx_head_addr);
2095
2096 /* Re-enable TX unit */
2097 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
2098 E1000_WRITE_FLUSH(&adapter->hw);
2099
2100 adapter->tx_fifo_head = 0;
2101 adapter->tx_fifo_reset_cnt++;
2102
2103 return (TRUE);
2104 } else {
2105 return (FALSE);
2106 }
2107 }
2108
2109 static void
em_set_promisc(struct adapter * adapter)2110 em_set_promisc(struct adapter *adapter)
2111 {
2112 struct ifnet *ifp = &adapter->arpcom.ac_if;
2113 uint32_t reg_rctl;
2114
2115 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2116
2117 if (ifp->if_flags & IFF_PROMISC) {
2118 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2119 /* Turn this on if you want to see bad packets */
2120 if (em_debug_sbp)
2121 reg_rctl |= E1000_RCTL_SBP;
2122 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2123 } else if (ifp->if_flags & IFF_ALLMULTI) {
2124 reg_rctl |= E1000_RCTL_MPE;
2125 reg_rctl &= ~E1000_RCTL_UPE;
2126 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2127 }
2128 }
2129
2130 static void
em_disable_promisc(struct adapter * adapter)2131 em_disable_promisc(struct adapter *adapter)
2132 {
2133 struct ifnet *ifp = &adapter->arpcom.ac_if;
2134 uint32_t reg_rctl;
2135 int mcnt = 0;
2136
2137 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2138 reg_rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_SBP);
2139
2140 if (ifp->if_flags & IFF_ALLMULTI) {
2141 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
2142 } else {
2143 const struct ifmultiaddr *ifma;
2144
2145 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2146 if (ifma->ifma_addr->sa_family != AF_LINK)
2147 continue;
2148 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2149 break;
2150 mcnt++;
2151 }
2152 }
2153 /* Don't disable if in MAX groups */
2154 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
2155 reg_rctl &= ~E1000_RCTL_MPE;
2156
2157 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2158 }
2159
2160 static void
em_set_multi(struct adapter * adapter)2161 em_set_multi(struct adapter *adapter)
2162 {
2163 struct ifnet *ifp = &adapter->arpcom.ac_if;
2164 struct ifmultiaddr *ifma;
2165 uint32_t reg_rctl = 0;
2166 uint8_t *mta;
2167 int mcnt = 0;
2168
2169 mta = adapter->mta;
2170 bzero(mta, ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES);
2171
2172 if (adapter->hw.mac.type == e1000_82542 &&
2173 adapter->hw.revision_id == E1000_REVISION_2) {
2174 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2175 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2176 e1000_pci_clear_mwi(&adapter->hw);
2177 reg_rctl |= E1000_RCTL_RST;
2178 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2179 msec_delay(5);
2180 }
2181
2182 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2183 if (ifma->ifma_addr->sa_family != AF_LINK)
2184 continue;
2185
2186 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2187 break;
2188
2189 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2190 &mta[mcnt * ETHER_ADDR_LEN], ETHER_ADDR_LEN);
2191 mcnt++;
2192 }
2193
2194 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
2195 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2196 reg_rctl |= E1000_RCTL_MPE;
2197 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2198 } else {
2199 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
2200 }
2201
2202 if (adapter->hw.mac.type == e1000_82542 &&
2203 adapter->hw.revision_id == E1000_REVISION_2) {
2204 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2205 reg_rctl &= ~E1000_RCTL_RST;
2206 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2207 msec_delay(5);
2208 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2209 e1000_pci_set_mwi(&adapter->hw);
2210 }
2211 }
2212
2213 /*
2214 * This routine checks for link status and updates statistics.
2215 */
2216 static void
em_timer(void * xsc)2217 em_timer(void *xsc)
2218 {
2219 struct adapter *adapter = xsc;
2220 struct ifnet *ifp = &adapter->arpcom.ac_if;
2221
2222 lwkt_serialize_enter(ifp->if_serializer);
2223
2224 em_update_link_status(adapter);
2225 em_update_stats(adapter);
2226
2227 /* Reset LAA into RAR[0] on 82571 */
2228 if (e1000_get_laa_state_82571(&adapter->hw) == TRUE)
2229 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
2230
2231 if (em_display_debug_stats && (ifp->if_flags & IFF_RUNNING))
2232 em_print_hw_stats(adapter);
2233
2234 em_smartspeed(adapter);
2235
2236 callout_reset(&adapter->timer, hz, em_timer, adapter);
2237
2238 lwkt_serialize_exit(ifp->if_serializer);
2239 }
2240
2241 static void
em_update_link_status(struct adapter * adapter)2242 em_update_link_status(struct adapter *adapter)
2243 {
2244 struct e1000_hw *hw = &adapter->hw;
2245 struct ifnet *ifp = &adapter->arpcom.ac_if;
2246 device_t dev = adapter->dev;
2247 uint32_t link_check = 0;
2248
2249 /* Get the cached link value or read phy for real */
2250 switch (hw->phy.media_type) {
2251 case e1000_media_type_copper:
2252 if (hw->mac.get_link_status) {
2253 if (hw->mac.type >= e1000_pch_spt)
2254 msec_delay(50);
2255 /* Do the work to read phy */
2256 e1000_check_for_link(hw);
2257 link_check = !hw->mac.get_link_status;
2258 if (link_check) /* ESB2 fix */
2259 e1000_cfg_on_link_up(hw);
2260 } else {
2261 link_check = TRUE;
2262 }
2263 break;
2264
2265 case e1000_media_type_fiber:
2266 e1000_check_for_link(hw);
2267 link_check =
2268 E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU;
2269 break;
2270
2271 case e1000_media_type_internal_serdes:
2272 e1000_check_for_link(hw);
2273 link_check = adapter->hw.mac.serdes_has_link;
2274 break;
2275
2276 case e1000_media_type_unknown:
2277 default:
2278 break;
2279 }
2280
2281 /* Now check for a transition */
2282 if (link_check && adapter->link_active == 0) {
2283 e1000_get_speed_and_duplex(hw, &adapter->link_speed,
2284 &adapter->link_duplex);
2285
2286 /*
2287 * Check if we should enable/disable SPEED_MODE bit on
2288 * 82571/82572
2289 */
2290 if (adapter->link_speed != SPEED_1000 &&
2291 (hw->mac.type == e1000_82571 ||
2292 hw->mac.type == e1000_82572)) {
2293 int tarc0;
2294
2295 tarc0 = E1000_READ_REG(hw, E1000_TARC(0));
2296 tarc0 &= ~TARC_SPEED_MODE_BIT;
2297 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0);
2298 }
2299 if (bootverbose) {
2300 char flowctrl[IFM_ETH_FC_STRLEN];
2301
2302 e1000_fc2str(hw->fc.current_mode, flowctrl,
2303 sizeof(flowctrl));
2304 device_printf(dev, "Link is up %d Mbps %s, "
2305 "Flow control: %s\n",
2306 adapter->link_speed,
2307 (adapter->link_duplex == FULL_DUPLEX) ?
2308 "Full Duplex" : "Half Duplex",
2309 flowctrl);
2310 }
2311 if (adapter->ifm_flowctrl & IFM_ETH_FORCEPAUSE)
2312 e1000_force_flowctrl(hw, adapter->ifm_flowctrl);
2313 adapter->link_active = 1;
2314 adapter->smartspeed = 0;
2315 ifp->if_baudrate = adapter->link_speed * 1000000;
2316 ifp->if_link_state = LINK_STATE_UP;
2317 if_link_state_change(ifp);
2318 } else if (!link_check && adapter->link_active == 1) {
2319 ifp->if_baudrate = adapter->link_speed = 0;
2320 adapter->link_duplex = 0;
2321 if (bootverbose)
2322 device_printf(dev, "Link is Down\n");
2323 adapter->link_active = 0;
2324 #if 0
2325 /* Link down, disable watchdog */
2326 if->if_timer = 0;
2327 #endif
2328 ifp->if_link_state = LINK_STATE_DOWN;
2329 if_link_state_change(ifp);
2330 }
2331 }
2332
2333 static void
em_stop(struct adapter * adapter)2334 em_stop(struct adapter *adapter)
2335 {
2336 struct ifnet *ifp = &adapter->arpcom.ac_if;
2337 int i;
2338
2339 ASSERT_SERIALIZED(ifp->if_serializer);
2340
2341 em_disable_intr(adapter);
2342
2343 callout_stop(&adapter->timer);
2344 callout_stop(&adapter->tx_fifo_timer);
2345
2346 ifp->if_flags &= ~IFF_RUNNING;
2347 ifq_clr_oactive(&ifp->if_snd);
2348 ifp->if_timer = 0;
2349 adapter->tx_running = 0;
2350 callout_stop(&adapter->tx_gc_timer);
2351
2352 /* I219 needs some special flushing to avoid hangs */
2353 if (adapter->hw.mac.type >= e1000_pch_spt)
2354 em_flush_txrx_ring(adapter);
2355
2356 e1000_reset_hw(&adapter->hw);
2357 if (adapter->hw.mac.type >= e1000_82544)
2358 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2359
2360 for (i = 0; i < adapter->num_tx_desc; i++) {
2361 struct em_buffer *tx_buffer = &adapter->tx_buffer_area[i];
2362
2363 if (tx_buffer->m_head != NULL)
2364 em_free_txbuffer(adapter, tx_buffer);
2365 }
2366
2367 for (i = 0; i < adapter->num_rx_desc; i++) {
2368 struct em_buffer *rx_buffer = &adapter->rx_buffer_area[i];
2369
2370 if (rx_buffer->m_head != NULL) {
2371 bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
2372 m_freem(rx_buffer->m_head);
2373 rx_buffer->m_head = NULL;
2374 }
2375 }
2376
2377 if (adapter->fmp != NULL)
2378 m_freem(adapter->fmp);
2379 adapter->fmp = NULL;
2380 adapter->lmp = NULL;
2381
2382 adapter->csum_flags = 0;
2383 adapter->csum_lhlen = 0;
2384 adapter->csum_iphlen = 0;
2385 adapter->csum_thlen = 0;
2386 adapter->csum_mss = 0;
2387 adapter->csum_pktlen = 0;
2388
2389 adapter->tx_dd_head = 0;
2390 adapter->tx_dd_tail = 0;
2391 adapter->tx_nsegs = 0;
2392 }
2393
2394 static int
em_get_hw_info(struct adapter * adapter)2395 em_get_hw_info(struct adapter *adapter)
2396 {
2397 device_t dev = adapter->dev;
2398
2399 /* Save off the information about this board */
2400 adapter->hw.vendor_id = pci_get_vendor(dev);
2401 adapter->hw.device_id = pci_get_device(dev);
2402 adapter->hw.revision_id = pci_get_revid(dev);
2403 adapter->hw.subsystem_vendor_id = pci_get_subvendor(dev);
2404 adapter->hw.subsystem_device_id = pci_get_subdevice(dev);
2405
2406 /* Do Shared Code Init and Setup */
2407 if (e1000_set_mac_type(&adapter->hw))
2408 return ENXIO;
2409 return 0;
2410 }
2411
2412 static int
em_alloc_pci_res(struct adapter * adapter)2413 em_alloc_pci_res(struct adapter *adapter)
2414 {
2415 device_t dev = adapter->dev;
2416 u_int intr_flags;
2417 int val, rid, msi_enable;
2418
2419 /* Enable bus mastering */
2420 pci_enable_busmaster(dev);
2421
2422 adapter->memory_rid = EM_BAR_MEM;
2423 adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2424 &adapter->memory_rid, RF_ACTIVE);
2425 if (adapter->memory == NULL) {
2426 device_printf(dev, "Unable to allocate bus resource: memory\n");
2427 return (ENXIO);
2428 }
2429 adapter->osdep.mem_bus_space_tag =
2430 rman_get_bustag(adapter->memory);
2431 adapter->osdep.mem_bus_space_handle =
2432 rman_get_bushandle(adapter->memory);
2433
2434 /* XXX This is quite goofy, it is not actually used */
2435 adapter->hw.hw_addr = (uint8_t *)&adapter->osdep.mem_bus_space_handle;
2436
2437 /* Only older adapters use IO mapping */
2438 if (adapter->hw.mac.type > e1000_82543 &&
2439 adapter->hw.mac.type < e1000_82571) {
2440 /* Figure our where our IO BAR is ? */
2441 for (rid = PCIR_BAR(0); rid < PCIR_CARDBUSCIS;) {
2442 val = pci_read_config(dev, rid, 4);
2443 if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) {
2444 adapter->io_rid = rid;
2445 break;
2446 }
2447 rid += 4;
2448 /* check for 64bit BAR */
2449 if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT)
2450 rid += 4;
2451 }
2452 if (rid >= PCIR_CARDBUSCIS) {
2453 device_printf(dev, "Unable to locate IO BAR\n");
2454 return (ENXIO);
2455 }
2456 adapter->ioport = bus_alloc_resource_any(dev, SYS_RES_IOPORT,
2457 &adapter->io_rid, RF_ACTIVE);
2458 if (adapter->ioport == NULL) {
2459 device_printf(dev, "Unable to allocate bus resource: "
2460 "ioport\n");
2461 return (ENXIO);
2462 }
2463 adapter->hw.io_base = 0;
2464 adapter->osdep.io_bus_space_tag =
2465 rman_get_bustag(adapter->ioport);
2466 adapter->osdep.io_bus_space_handle =
2467 rman_get_bushandle(adapter->ioport);
2468 }
2469
2470 /*
2471 * Don't enable MSI-X on 82574, see:
2472 * 82574 specification update errata #15
2473 *
2474 * Don't enable MSI on PCI/PCI-X chips, see:
2475 * 82540 specification update errata #6
2476 * 82545 specification update errata #4
2477 *
2478 * Don't enable MSI on 82571/82572, see:
2479 * 82571/82572 specification update errata #63
2480 */
2481 msi_enable = em_msi_enable;
2482 if (msi_enable &&
2483 ((adapter->flags & EM_FLAG_GEN2) == 0 ||
2484 adapter->hw.mac.type == e1000_82571 ||
2485 adapter->hw.mac.type == e1000_82572))
2486 msi_enable = 0;
2487 again:
2488 adapter->intr_type = pci_alloc_1intr(dev, msi_enable,
2489 &adapter->intr_rid, &intr_flags);
2490
2491 if (adapter->intr_type == PCI_INTR_TYPE_LEGACY) {
2492 int unshared;
2493
2494 unshared = device_getenv_int(dev, "irq.unshared", 0);
2495 if (!unshared) {
2496 adapter->flags |= EM_FLAG_SHARED_INTR;
2497 if (bootverbose)
2498 device_printf(dev, "IRQ shared\n");
2499 } else {
2500 intr_flags &= ~RF_SHAREABLE;
2501 if (bootverbose)
2502 device_printf(dev, "IRQ unshared\n");
2503 }
2504 }
2505
2506 adapter->intr_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
2507 &adapter->intr_rid, intr_flags);
2508 if (adapter->intr_res == NULL) {
2509 device_printf(dev, "Unable to allocate bus resource: %s\n",
2510 adapter->intr_type == PCI_INTR_TYPE_MSI ?
2511 "MSI" : "legacy intr");
2512 if (!msi_enable) {
2513 /* Retry with MSI. */
2514 msi_enable = 1;
2515 adapter->flags &= ~EM_FLAG_SHARED_INTR;
2516 goto again;
2517 }
2518 return (ENXIO);
2519 }
2520
2521 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2522 adapter->hw.back = &adapter->osdep;
2523 return (0);
2524 }
2525
2526 static void
em_free_pci_res(struct adapter * adapter)2527 em_free_pci_res(struct adapter *adapter)
2528 {
2529 device_t dev = adapter->dev;
2530
2531 if (adapter->intr_res != NULL) {
2532 bus_release_resource(dev, SYS_RES_IRQ,
2533 adapter->intr_rid, adapter->intr_res);
2534 }
2535
2536 if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2537 pci_release_msi(dev);
2538
2539 if (adapter->memory != NULL) {
2540 bus_release_resource(dev, SYS_RES_MEMORY,
2541 adapter->memory_rid, adapter->memory);
2542 }
2543
2544 if (adapter->flash != NULL) {
2545 bus_release_resource(dev, SYS_RES_MEMORY,
2546 adapter->flash_rid, adapter->flash);
2547 }
2548
2549 if (adapter->ioport != NULL) {
2550 bus_release_resource(dev, SYS_RES_IOPORT,
2551 adapter->io_rid, adapter->ioport);
2552 }
2553 }
2554
2555 static int
em_reset(struct adapter * adapter)2556 em_reset(struct adapter *adapter)
2557 {
2558 device_t dev = adapter->dev;
2559 uint16_t rx_buffer_size;
2560 uint32_t pba;
2561
2562 /* When hardware is reset, fifo_head is also reset */
2563 adapter->tx_fifo_head = 0;
2564
2565 /* Set up smart power down as default off on newer adapters. */
2566 if (!em_smart_pwr_down &&
2567 (adapter->hw.mac.type == e1000_82571 ||
2568 adapter->hw.mac.type == e1000_82572)) {
2569 uint16_t phy_tmp = 0;
2570
2571 /* Speed up time to link by disabling smart power down. */
2572 e1000_read_phy_reg(&adapter->hw,
2573 IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
2574 phy_tmp &= ~IGP02E1000_PM_SPD;
2575 e1000_write_phy_reg(&adapter->hw,
2576 IGP02E1000_PHY_POWER_MGMT, phy_tmp);
2577 }
2578
2579 /*
2580 * Packet Buffer Allocation (PBA)
2581 * Writing PBA sets the receive portion of the buffer
2582 * the remainder is used for the transmit buffer.
2583 *
2584 * Devices before the 82547 had a Packet Buffer of 64K.
2585 * Default allocation: PBA=48K for Rx, leaving 16K for Tx.
2586 * After the 82547 the buffer was reduced to 40K.
2587 * Default allocation: PBA=30K for Rx, leaving 10K for Tx.
2588 * Note: default does not leave enough room for Jumbo Frame >10k.
2589 */
2590 switch (adapter->hw.mac.type) {
2591 case e1000_82547:
2592 case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */
2593 if (adapter->hw.mac.max_frame_size > 8192)
2594 pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */
2595 else
2596 pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */
2597 adapter->tx_fifo_head = 0;
2598 adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT;
2599 adapter->tx_fifo_size =
2600 (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT;
2601 break;
2602
2603 /* Total Packet Buffer on these is 48K */
2604 case e1000_82571:
2605 case e1000_82572:
2606 case e1000_80003es2lan:
2607 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
2608 break;
2609
2610 case e1000_82573: /* 82573: Total Packet Buffer is 32K */
2611 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
2612 break;
2613
2614 case e1000_82574:
2615 case e1000_82583:
2616 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */
2617 break;
2618
2619 case e1000_ich8lan:
2620 pba = E1000_PBA_8K;
2621 break;
2622
2623 case e1000_ich9lan:
2624 case e1000_ich10lan:
2625 #define E1000_PBA_10K 0x000A
2626 pba = E1000_PBA_10K;
2627 break;
2628
2629 case e1000_pchlan:
2630 case e1000_pch2lan:
2631 case e1000_pch_lpt:
2632 case e1000_pch_spt:
2633 case e1000_pch_cnp:
2634 pba = E1000_PBA_26K;
2635 break;
2636
2637 default:
2638 /* Devices before 82547 had a Packet Buffer of 64K. */
2639 if (adapter->hw.mac.max_frame_size > 8192)
2640 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
2641 else
2642 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
2643 }
2644 E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
2645
2646 /*
2647 * These parameters control the automatic generation (Tx) and
2648 * response (Rx) to Ethernet PAUSE frames.
2649 * - High water mark should allow for at least two frames to be
2650 * received after sending an XOFF.
2651 * - Low water mark works best when it is very near the high water mark.
2652 * This allows the receiver to restart by sending XON when it has
2653 * drained a bit. Here we use an arbitary value of 1500 which will
2654 * restart after one full frame is pulled from the buffer. There
2655 * could be several smaller frames in the buffer and if so they will
2656 * not trigger the XON until their total number reduces the buffer
2657 * by 1500.
2658 * - The pause time is fairly large at 1000 x 512ns = 512 usec.
2659 */
2660 rx_buffer_size =
2661 (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) << 10;
2662
2663 adapter->hw.fc.high_water = rx_buffer_size -
2664 roundup2(adapter->hw.mac.max_frame_size, 1024);
2665 adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500;
2666
2667 if (adapter->hw.mac.type == e1000_80003es2lan)
2668 adapter->hw.fc.pause_time = 0xFFFF;
2669 else
2670 adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME;
2671
2672 adapter->hw.fc.send_xon = TRUE;
2673
2674 adapter->hw.fc.requested_mode = e1000_ifmedia2fc(adapter->ifm_flowctrl);
2675
2676 /*
2677 * Device specific overrides/settings
2678 */
2679 switch (adapter->hw.mac.type) {
2680 case e1000_pchlan:
2681 KASSERT(adapter->hw.fc.requested_mode == e1000_fc_rx_pause ||
2682 adapter->hw.fc.requested_mode == e1000_fc_none,
2683 ("unsupported flow control on PCH %d",
2684 adapter->hw.fc.requested_mode));
2685 adapter->hw.fc.pause_time = 0xFFFF; /* override */
2686 if (adapter->arpcom.ac_if.if_mtu > ETHERMTU) {
2687 adapter->hw.fc.high_water = 0x3500;
2688 adapter->hw.fc.low_water = 0x1500;
2689 } else {
2690 adapter->hw.fc.high_water = 0x5000;
2691 adapter->hw.fc.low_water = 0x3000;
2692 }
2693 adapter->hw.fc.refresh_time = 0x1000;
2694 break;
2695
2696 case e1000_pch2lan:
2697 case e1000_pch_lpt:
2698 case e1000_pch_spt:
2699 case e1000_pch_cnp:
2700 adapter->hw.fc.high_water = 0x5C20;
2701 adapter->hw.fc.low_water = 0x5048;
2702 adapter->hw.fc.pause_time = 0x0650;
2703 adapter->hw.fc.refresh_time = 0x0400;
2704 /* Jumbos need adjusted PBA */
2705 if (adapter->arpcom.ac_if.if_mtu > ETHERMTU)
2706 E1000_WRITE_REG(&adapter->hw, E1000_PBA, 12);
2707 else
2708 E1000_WRITE_REG(&adapter->hw, E1000_PBA, 26);
2709 break;
2710
2711 case e1000_ich9lan:
2712 case e1000_ich10lan:
2713 if (adapter->arpcom.ac_if.if_mtu > ETHERMTU) {
2714 adapter->hw.fc.high_water = 0x2800;
2715 adapter->hw.fc.low_water =
2716 adapter->hw.fc.high_water - 8;
2717 break;
2718 }
2719 /* FALL THROUGH */
2720 default:
2721 if (adapter->hw.mac.type == e1000_80003es2lan)
2722 adapter->hw.fc.pause_time = 0xFFFF;
2723 break;
2724 }
2725
2726 /* I219 needs some special flushing to avoid hangs */
2727 if (adapter->hw.mac.type >= e1000_pch_spt)
2728 em_flush_txrx_ring(adapter);
2729
2730 /* Issue a global reset */
2731 e1000_reset_hw(&adapter->hw);
2732 if (adapter->hw.mac.type >= e1000_82544)
2733 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2734 em_disable_aspm(adapter);
2735
2736 if (e1000_init_hw(&adapter->hw) < 0) {
2737 device_printf(dev, "Hardware Initialization Failed\n");
2738 return (EIO);
2739 }
2740
2741 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
2742 e1000_get_phy_info(&adapter->hw);
2743 e1000_check_for_link(&adapter->hw);
2744
2745 return (0);
2746 }
2747
2748 static void
em_setup_ifp(struct adapter * adapter)2749 em_setup_ifp(struct adapter *adapter)
2750 {
2751 struct ifnet *ifp = &adapter->arpcom.ac_if;
2752
2753 if_initname(ifp, device_get_name(adapter->dev),
2754 device_get_unit(adapter->dev));
2755 ifp->if_softc = adapter;
2756 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2757 ifp->if_init = em_init;
2758 ifp->if_ioctl = em_ioctl;
2759 ifp->if_start = em_start;
2760 #ifdef IFPOLL_ENABLE
2761 ifp->if_npoll = em_npoll;
2762 #endif
2763 ifp->if_watchdog = em_watchdog;
2764 ifp->if_nmbclusters = adapter->num_rx_desc;
2765 ifq_set_maxlen(&ifp->if_snd, adapter->num_tx_desc - 1);
2766 ifq_set_ready(&ifp->if_snd);
2767
2768 ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2769
2770 ifp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2771 if (adapter->hw.mac.type >= e1000_82543)
2772 ifp->if_capabilities |= IFCAP_HWCSUM;
2773 if (adapter->flags & EM_FLAG_TSO)
2774 ifp->if_capabilities |= IFCAP_TSO;
2775 ifp->if_capenable = ifp->if_capabilities;
2776
2777 if (ifp->if_capenable & IFCAP_TXCSUM)
2778 ifp->if_hwassist |= EM_CSUM_FEATURES;
2779 if (ifp->if_capenable & IFCAP_TSO)
2780 ifp->if_hwassist |= CSUM_TSO;
2781
2782 /*
2783 * Tell the upper layer(s) we support long frames.
2784 */
2785 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2786
2787 /*
2788 * Specify the media types supported by this adapter and register
2789 * callbacks to update media and link information
2790 */
2791 if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
2792 adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
2793 u_char fiber_type = IFM_1000_SX; /* default type */
2794
2795 if (adapter->hw.mac.type == e1000_82545)
2796 fiber_type = IFM_1000_LX;
2797 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
2798 0, NULL);
2799 } else {
2800 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
2801 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
2802 0, NULL);
2803 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
2804 0, NULL);
2805 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
2806 0, NULL);
2807 if (adapter->hw.phy.type != e1000_phy_ife) {
2808 ifmedia_add(&adapter->media,
2809 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2810 }
2811 }
2812 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2813 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO |
2814 adapter->ifm_flowctrl);
2815 }
2816
2817
2818 /*
2819 * Workaround for SmartSpeed on 82541 and 82547 controllers
2820 */
2821 static void
em_smartspeed(struct adapter * adapter)2822 em_smartspeed(struct adapter *adapter)
2823 {
2824 uint16_t phy_tmp;
2825
2826 if (adapter->link_active || adapter->hw.phy.type != e1000_phy_igp ||
2827 adapter->hw.mac.autoneg == 0 ||
2828 (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0)
2829 return;
2830
2831 if (adapter->smartspeed == 0) {
2832 /*
2833 * If Master/Slave config fault is asserted twice,
2834 * we assume back-to-back
2835 */
2836 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
2837 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
2838 return;
2839 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
2840 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
2841 e1000_read_phy_reg(&adapter->hw,
2842 PHY_1000T_CTRL, &phy_tmp);
2843 if (phy_tmp & CR_1000T_MS_ENABLE) {
2844 phy_tmp &= ~CR_1000T_MS_ENABLE;
2845 e1000_write_phy_reg(&adapter->hw,
2846 PHY_1000T_CTRL, phy_tmp);
2847 adapter->smartspeed++;
2848 if (adapter->hw.mac.autoneg &&
2849 !e1000_phy_setup_autoneg(&adapter->hw) &&
2850 !e1000_read_phy_reg(&adapter->hw,
2851 PHY_CONTROL, &phy_tmp)) {
2852 phy_tmp |= MII_CR_AUTO_NEG_EN |
2853 MII_CR_RESTART_AUTO_NEG;
2854 e1000_write_phy_reg(&adapter->hw,
2855 PHY_CONTROL, phy_tmp);
2856 }
2857 }
2858 }
2859 return;
2860 } else if (adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
2861 /* If still no link, perhaps using 2/3 pair cable */
2862 e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
2863 phy_tmp |= CR_1000T_MS_ENABLE;
2864 e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
2865 if (adapter->hw.mac.autoneg &&
2866 !e1000_phy_setup_autoneg(&adapter->hw) &&
2867 !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) {
2868 phy_tmp |= MII_CR_AUTO_NEG_EN | MII_CR_RESTART_AUTO_NEG;
2869 e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp);
2870 }
2871 }
2872
2873 /* Restart process after EM_SMARTSPEED_MAX iterations */
2874 if (adapter->smartspeed++ == EM_SMARTSPEED_MAX)
2875 adapter->smartspeed = 0;
2876 }
2877
2878 static int
em_dma_malloc(struct adapter * adapter,bus_size_t size,struct em_dma_alloc * dma)2879 em_dma_malloc(struct adapter *adapter, bus_size_t size,
2880 struct em_dma_alloc *dma)
2881 {
2882 dma->dma_vaddr = bus_dmamem_coherent_any(adapter->parent_dtag,
2883 EM_DBA_ALIGN, size, BUS_DMA_WAITOK,
2884 &dma->dma_tag, &dma->dma_map,
2885 &dma->dma_paddr);
2886 if (dma->dma_vaddr == NULL)
2887 return ENOMEM;
2888 else
2889 return 0;
2890 }
2891
2892 static void
em_dma_free(struct adapter * adapter,struct em_dma_alloc * dma)2893 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
2894 {
2895 if (dma->dma_tag == NULL)
2896 return;
2897 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2898 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2899 bus_dma_tag_destroy(dma->dma_tag);
2900 }
2901
2902 static int
em_create_tx_ring(struct adapter * adapter)2903 em_create_tx_ring(struct adapter *adapter)
2904 {
2905 device_t dev = adapter->dev;
2906 struct em_buffer *tx_buffer;
2907 int error, i;
2908
2909 adapter->tx_buffer_area =
2910 kmalloc(sizeof(struct em_buffer) * adapter->num_tx_desc,
2911 M_DEVBUF, M_WAITOK | M_ZERO);
2912
2913 /*
2914 * Create DMA tags for tx buffers
2915 */
2916 error = bus_dma_tag_create(adapter->parent_dtag, /* parent */
2917 1, 0, /* alignment, bounds */
2918 BUS_SPACE_MAXADDR, /* lowaddr */
2919 BUS_SPACE_MAXADDR, /* highaddr */
2920 EM_TSO_SIZE, /* maxsize */
2921 EM_MAX_SCATTER, /* nsegments */
2922 PAGE_SIZE, /* maxsegsize */
2923 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW |
2924 BUS_DMA_ONEBPAGE, /* flags */
2925 &adapter->txtag);
2926 if (error) {
2927 device_printf(dev, "Unable to allocate TX DMA tag\n");
2928 kfree(adapter->tx_buffer_area, M_DEVBUF);
2929 adapter->tx_buffer_area = NULL;
2930 return error;
2931 }
2932
2933 /*
2934 * Create DMA maps for tx buffers
2935 */
2936 for (i = 0; i < adapter->num_tx_desc; i++) {
2937 tx_buffer = &adapter->tx_buffer_area[i];
2938
2939 error = bus_dmamap_create(adapter->txtag,
2940 BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
2941 &tx_buffer->map);
2942 if (error) {
2943 device_printf(dev, "Unable to create TX DMA map\n");
2944 em_destroy_tx_ring(adapter, i);
2945 return error;
2946 }
2947 }
2948 return (0);
2949 }
2950
2951 static void
em_init_tx_ring(struct adapter * adapter)2952 em_init_tx_ring(struct adapter *adapter)
2953 {
2954 /* Clear the old ring contents */
2955 bzero(adapter->tx_desc_base,
2956 (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc);
2957
2958 /* Reset state */
2959 adapter->next_avail_tx_desc = 0;
2960 adapter->next_tx_to_clean = 0;
2961 adapter->num_tx_desc_avail = adapter->num_tx_desc;
2962 adapter->tx_nmbuf = 0;
2963 adapter->tx_running = 0;
2964 }
2965
2966 static void
em_init_tx_unit(struct adapter * adapter)2967 em_init_tx_unit(struct adapter *adapter)
2968 {
2969 uint32_t tctl, tarc, tipg = 0;
2970 uint64_t bus_addr;
2971
2972 /* Setup the Base and Length of the Tx Descriptor Ring */
2973 bus_addr = adapter->txdma.dma_paddr;
2974 E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0),
2975 adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
2976 E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0),
2977 (uint32_t)(bus_addr >> 32));
2978 E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0),
2979 (uint32_t)bus_addr);
2980 /* Setup the HW Tx Head and Tail descriptor pointers */
2981 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0);
2982 E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0);
2983 if (adapter->flags & EM_FLAG_GEN2) {
2984 uint32_t txdctl = 0;
2985
2986 txdctl |= 0x1f; /* PTHRESH */
2987 txdctl |= 1 << 8; /* HTHRESH */
2988 txdctl |= 1 << 16; /* WTHRESH */
2989 txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */
2990 txdctl |= E1000_TXDCTL_GRAN;
2991 txdctl |= 1 << 25; /* LWTHRESH */
2992
2993 E1000_WRITE_REG(&adapter->hw, E1000_TXDCTL(0), txdctl);
2994 }
2995
2996 /* Set the default values for the Tx Inter Packet Gap timer */
2997 switch (adapter->hw.mac.type) {
2998 case e1000_82542:
2999 tipg = DEFAULT_82542_TIPG_IPGT;
3000 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3001 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3002 break;
3003
3004 case e1000_80003es2lan:
3005 tipg = DEFAULT_82543_TIPG_IPGR1;
3006 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 <<
3007 E1000_TIPG_IPGR2_SHIFT;
3008 break;
3009
3010 default:
3011 if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
3012 adapter->hw.phy.media_type ==
3013 e1000_media_type_internal_serdes)
3014 tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
3015 else
3016 tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
3017 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3018 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3019 break;
3020 }
3021
3022 E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
3023
3024 /* NOTE: 0 is not allowed for TIDV */
3025 E1000_WRITE_REG(&adapter->hw, E1000_TIDV, 1);
3026 if(adapter->hw.mac.type >= e1000_82540)
3027 E1000_WRITE_REG(&adapter->hw, E1000_TADV, 0);
3028
3029 if (adapter->hw.mac.type == e1000_82571 ||
3030 adapter->hw.mac.type == e1000_82572) {
3031 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3032 tarc |= TARC_SPEED_MODE_BIT;
3033 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3034 } else if (adapter->hw.mac.type == e1000_80003es2lan) {
3035 /* errata: program both queues to unweighted RR */
3036 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3037 tarc |= 1;
3038 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3039 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
3040 tarc |= 1;
3041 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
3042 } else if (adapter->hw.mac.type == e1000_82574) {
3043 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3044 tarc |= TARC_ERRATA_BIT;
3045 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3046 }
3047
3048 /* Program the Transmit Control Register */
3049 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
3050 tctl &= ~E1000_TCTL_CT;
3051 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3052 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
3053
3054 if (adapter->hw.mac.type >= e1000_82571)
3055 tctl |= E1000_TCTL_MULR;
3056
3057 /* This write will effectively turn on the transmit unit. */
3058 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
3059
3060 if (adapter->hw.mac.type == e1000_82571 ||
3061 adapter->hw.mac.type == e1000_82572 ||
3062 adapter->hw.mac.type == e1000_80003es2lan) {
3063 /* Bit 28 of TARC1 must be cleared when MULR is enabled */
3064 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
3065 tarc &= ~(1 << 28);
3066 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
3067 } else if (adapter->hw.mac.type >= e1000_pch_spt) {
3068 uint32_t reg;
3069
3070 reg = E1000_READ_REG(&adapter->hw, E1000_IOSFPC);
3071 reg |= E1000_RCTL_RDMTS_HEX;
3072 E1000_WRITE_REG(&adapter->hw, E1000_IOSFPC, reg);
3073 reg = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3074 reg |= E1000_TARC0_CB_MULTIQ_3_REQ;
3075 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), reg);
3076 }
3077 }
3078
3079 static void
em_destroy_tx_ring(struct adapter * adapter,int ndesc)3080 em_destroy_tx_ring(struct adapter *adapter, int ndesc)
3081 {
3082 struct em_buffer *tx_buffer;
3083 int i;
3084
3085 if (adapter->tx_buffer_area == NULL)
3086 return;
3087
3088 for (i = 0; i < ndesc; i++) {
3089 tx_buffer = &adapter->tx_buffer_area[i];
3090
3091 KKASSERT(tx_buffer->m_head == NULL);
3092 bus_dmamap_destroy(adapter->txtag, tx_buffer->map);
3093 }
3094 bus_dma_tag_destroy(adapter->txtag);
3095
3096 kfree(adapter->tx_buffer_area, M_DEVBUF);
3097 adapter->tx_buffer_area = NULL;
3098 }
3099
3100 /*
3101 * The offload context needs to be set when we transfer the first
3102 * packet of a particular protocol (TCP/UDP). This routine has been
3103 * enhanced to deal with inserted VLAN headers.
3104 *
3105 * If the new packet's ether header length, ip header length and
3106 * csum offloading type are same as the previous packet, we should
3107 * avoid allocating a new csum context descriptor; mainly to take
3108 * advantage of the pipeline effect of the TX data read request.
3109 *
3110 * This function returns number of TX descrptors allocated for
3111 * csum context.
3112 */
3113 static int
em_txcsum(struct adapter * adapter,struct mbuf * mp,uint32_t * txd_upper,uint32_t * txd_lower)3114 em_txcsum(struct adapter *adapter, struct mbuf *mp,
3115 uint32_t *txd_upper, uint32_t *txd_lower)
3116 {
3117 struct e1000_context_desc *TXD;
3118 int curr_txd, ehdrlen, csum_flags;
3119 uint32_t cmd, hdr_len, ip_hlen;
3120
3121 csum_flags = mp->m_pkthdr.csum_flags & EM_CSUM_FEATURES;
3122 ip_hlen = mp->m_pkthdr.csum_iphlen;
3123 ehdrlen = mp->m_pkthdr.csum_lhlen;
3124
3125 if (adapter->csum_lhlen == ehdrlen &&
3126 adapter->csum_iphlen == ip_hlen &&
3127 adapter->csum_flags == csum_flags) {
3128 /*
3129 * Same csum offload context as the previous packets;
3130 * just return.
3131 */
3132 *txd_upper = adapter->csum_txd_upper;
3133 *txd_lower = adapter->csum_txd_lower;
3134 return 0;
3135 }
3136
3137 /*
3138 * Setup a new csum offload context.
3139 */
3140
3141 curr_txd = adapter->next_avail_tx_desc;
3142 TXD = (struct e1000_context_desc *)&adapter->tx_desc_base[curr_txd];
3143
3144 cmd = 0;
3145
3146 /* Setup of IP header checksum. */
3147 if (csum_flags & CSUM_IP) {
3148 /*
3149 * Start offset for header checksum calculation.
3150 * End offset for header checksum calculation.
3151 * Offset of place to put the checksum.
3152 */
3153 TXD->lower_setup.ip_fields.ipcss = ehdrlen;
3154 TXD->lower_setup.ip_fields.ipcse =
3155 htole16(ehdrlen + ip_hlen - 1);
3156 TXD->lower_setup.ip_fields.ipcso =
3157 ehdrlen + offsetof(struct ip, ip_sum);
3158 cmd |= E1000_TXD_CMD_IP;
3159 *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
3160 }
3161 hdr_len = ehdrlen + ip_hlen;
3162
3163 if (csum_flags & CSUM_TCP) {
3164 /*
3165 * Start offset for payload checksum calculation.
3166 * End offset for payload checksum calculation.
3167 * Offset of place to put the checksum.
3168 */
3169 TXD->upper_setup.tcp_fields.tucss = hdr_len;
3170 TXD->upper_setup.tcp_fields.tucse = htole16(0);
3171 TXD->upper_setup.tcp_fields.tucso =
3172 hdr_len + offsetof(struct tcphdr, th_sum);
3173 cmd |= E1000_TXD_CMD_TCP;
3174 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3175 } else if (csum_flags & CSUM_UDP) {
3176 /*
3177 * Start offset for header checksum calculation.
3178 * End offset for header checksum calculation.
3179 * Offset of place to put the checksum.
3180 */
3181 TXD->upper_setup.tcp_fields.tucss = hdr_len;
3182 TXD->upper_setup.tcp_fields.tucse = htole16(0);
3183 TXD->upper_setup.tcp_fields.tucso =
3184 hdr_len + offsetof(struct udphdr, uh_sum);
3185 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3186 }
3187
3188 *txd_lower = E1000_TXD_CMD_DEXT | /* Extended descr type */
3189 E1000_TXD_DTYP_D; /* Data descr */
3190
3191 /* Save the information for this csum offloading context */
3192 adapter->csum_lhlen = ehdrlen;
3193 adapter->csum_iphlen = ip_hlen;
3194 adapter->csum_flags = csum_flags;
3195 adapter->csum_txd_upper = *txd_upper;
3196 adapter->csum_txd_lower = *txd_lower;
3197
3198 TXD->tcp_seg_setup.data = htole32(0);
3199 TXD->cmd_and_length =
3200 htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd);
3201
3202 if (++curr_txd == adapter->num_tx_desc)
3203 curr_txd = 0;
3204
3205 KKASSERT(adapter->num_tx_desc_avail > 0);
3206 adapter->num_tx_desc_avail--;
3207
3208 adapter->next_avail_tx_desc = curr_txd;
3209 return 1;
3210 }
3211
3212 static void
em_txeof(struct adapter * adapter)3213 em_txeof(struct adapter *adapter)
3214 {
3215 struct ifnet *ifp = &adapter->arpcom.ac_if;
3216 struct em_buffer *tx_buffer;
3217 int first, num_avail;
3218
3219 if (adapter->tx_dd_head == adapter->tx_dd_tail)
3220 return;
3221
3222 if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
3223 return;
3224
3225 num_avail = adapter->num_tx_desc_avail;
3226 first = adapter->next_tx_to_clean;
3227
3228 while (adapter->tx_dd_head != adapter->tx_dd_tail) {
3229 struct e1000_tx_desc *tx_desc;
3230 int dd_idx = adapter->tx_dd[adapter->tx_dd_head];
3231
3232 tx_desc = &adapter->tx_desc_base[dd_idx];
3233 if (tx_desc->upper.fields.status & E1000_TXD_STAT_DD) {
3234 EM_INC_TXDD_IDX(adapter->tx_dd_head);
3235
3236 if (++dd_idx == adapter->num_tx_desc)
3237 dd_idx = 0;
3238
3239 while (first != dd_idx) {
3240 logif(pkt_txclean);
3241
3242 KKASSERT(num_avail < adapter->num_tx_desc);
3243 num_avail++;
3244
3245 tx_buffer = &adapter->tx_buffer_area[first];
3246 if (tx_buffer->m_head != NULL)
3247 em_free_txbuffer(adapter, tx_buffer);
3248
3249 if (++first == adapter->num_tx_desc)
3250 first = 0;
3251 }
3252 } else {
3253 break;
3254 }
3255 }
3256 adapter->next_tx_to_clean = first;
3257 adapter->num_tx_desc_avail = num_avail;
3258
3259 if (adapter->tx_dd_head == adapter->tx_dd_tail) {
3260 adapter->tx_dd_head = 0;
3261 adapter->tx_dd_tail = 0;
3262 }
3263
3264 if (!EM_IS_OACTIVE(adapter)) {
3265 ifq_clr_oactive(&ifp->if_snd);
3266
3267 /* All clean, turn off the timer */
3268 if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
3269 ifp->if_timer = 0;
3270 }
3271 adapter->tx_running = EM_TX_RUNNING;
3272 }
3273
3274 static void
em_tx_collect(struct adapter * adapter,boolean_t gc)3275 em_tx_collect(struct adapter *adapter, boolean_t gc)
3276 {
3277 struct ifnet *ifp = &adapter->arpcom.ac_if;
3278 struct em_buffer *tx_buffer;
3279 int tdh, first, num_avail, dd_idx = -1;
3280
3281 if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
3282 return;
3283
3284 tdh = E1000_READ_REG(&adapter->hw, E1000_TDH(0));
3285 if (tdh == adapter->next_tx_to_clean) {
3286 if (gc && adapter->tx_nmbuf > 0)
3287 adapter->tx_running = EM_TX_RUNNING;
3288 return;
3289 }
3290 if (gc)
3291 adapter->tx_gc++;
3292
3293 if (adapter->tx_dd_head != adapter->tx_dd_tail)
3294 dd_idx = adapter->tx_dd[adapter->tx_dd_head];
3295
3296 num_avail = adapter->num_tx_desc_avail;
3297 first = adapter->next_tx_to_clean;
3298
3299 while (first != tdh) {
3300 logif(pkt_txclean);
3301
3302 KKASSERT(num_avail < adapter->num_tx_desc);
3303 num_avail++;
3304
3305 tx_buffer = &adapter->tx_buffer_area[first];
3306 if (tx_buffer->m_head != NULL)
3307 em_free_txbuffer(adapter, tx_buffer);
3308
3309 if (first == dd_idx) {
3310 EM_INC_TXDD_IDX(adapter->tx_dd_head);
3311 if (adapter->tx_dd_head == adapter->tx_dd_tail) {
3312 adapter->tx_dd_head = 0;
3313 adapter->tx_dd_tail = 0;
3314 dd_idx = -1;
3315 } else {
3316 dd_idx = adapter->tx_dd[adapter->tx_dd_head];
3317 }
3318 }
3319
3320 if (++first == adapter->num_tx_desc)
3321 first = 0;
3322 }
3323 adapter->next_tx_to_clean = first;
3324 adapter->num_tx_desc_avail = num_avail;
3325
3326 if (!EM_IS_OACTIVE(adapter)) {
3327 ifq_clr_oactive(&ifp->if_snd);
3328
3329 /* All clean, turn off the timer */
3330 if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
3331 ifp->if_timer = 0;
3332 }
3333 if (!gc || adapter->tx_nmbuf > 0)
3334 adapter->tx_running = EM_TX_RUNNING;
3335 }
3336
3337 /*
3338 * When Link is lost sometimes there is work still in the TX ring
3339 * which will result in a watchdog, rather than allow that do an
3340 * attempted cleanup and then reinit here. Note that this has been
3341 * seens mostly with fiber adapters.
3342 */
3343 static void
em_tx_purge(struct adapter * adapter)3344 em_tx_purge(struct adapter *adapter)
3345 {
3346 struct ifnet *ifp = &adapter->arpcom.ac_if;
3347
3348 if (!adapter->link_active && ifp->if_timer) {
3349 em_tx_collect(adapter, FALSE);
3350 if (ifp->if_timer) {
3351 if_printf(ifp, "Link lost, TX pending, reinit\n");
3352 ifp->if_timer = 0;
3353 em_init(adapter);
3354 }
3355 }
3356 }
3357
3358 static int
em_newbuf(struct adapter * adapter,int i,int init)3359 em_newbuf(struct adapter *adapter, int i, int init)
3360 {
3361 struct mbuf *m;
3362 bus_dma_segment_t seg;
3363 bus_dmamap_t map;
3364 struct em_buffer *rx_buffer;
3365 int error, nseg;
3366
3367 m = m_getcl(init ? M_WAITOK : M_NOWAIT, MT_DATA, M_PKTHDR);
3368 if (m == NULL) {
3369 adapter->mbuf_cluster_failed++;
3370 if (init) {
3371 if_printf(&adapter->arpcom.ac_if,
3372 "Unable to allocate RX mbuf\n");
3373 }
3374 return (ENOBUFS);
3375 }
3376 m->m_len = m->m_pkthdr.len = MCLBYTES;
3377
3378 if (adapter->hw.mac.max_frame_size <= MCLBYTES - ETHER_ALIGN)
3379 m_adj(m, ETHER_ALIGN);
3380
3381 error = bus_dmamap_load_mbuf_segment(adapter->rxtag,
3382 adapter->rx_sparemap, m,
3383 &seg, 1, &nseg, BUS_DMA_NOWAIT);
3384 if (error) {
3385 m_freem(m);
3386 if (init) {
3387 if_printf(&adapter->arpcom.ac_if,
3388 "Unable to load RX mbuf\n");
3389 }
3390 return (error);
3391 }
3392
3393 rx_buffer = &adapter->rx_buffer_area[i];
3394 if (rx_buffer->m_head != NULL)
3395 bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
3396
3397 map = rx_buffer->map;
3398 rx_buffer->map = adapter->rx_sparemap;
3399 adapter->rx_sparemap = map;
3400
3401 rx_buffer->m_head = m;
3402
3403 adapter->rx_desc_base[i].buffer_addr = htole64(seg.ds_addr);
3404 return (0);
3405 }
3406
3407 static int
em_create_rx_ring(struct adapter * adapter)3408 em_create_rx_ring(struct adapter *adapter)
3409 {
3410 device_t dev = adapter->dev;
3411 struct em_buffer *rx_buffer;
3412 int i, error;
3413
3414 adapter->rx_buffer_area =
3415 kmalloc(sizeof(struct em_buffer) * adapter->num_rx_desc,
3416 M_DEVBUF, M_WAITOK | M_ZERO);
3417
3418 /*
3419 * Create DMA tag for rx buffers
3420 */
3421 error = bus_dma_tag_create(adapter->parent_dtag, /* parent */
3422 1, 0, /* alignment, bounds */
3423 BUS_SPACE_MAXADDR, /* lowaddr */
3424 BUS_SPACE_MAXADDR, /* highaddr */
3425 MCLBYTES, /* maxsize */
3426 1, /* nsegments */
3427 MCLBYTES, /* maxsegsize */
3428 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, /* flags */
3429 &adapter->rxtag);
3430 if (error) {
3431 device_printf(dev, "Unable to allocate RX DMA tag\n");
3432 kfree(adapter->rx_buffer_area, M_DEVBUF);
3433 adapter->rx_buffer_area = NULL;
3434 return error;
3435 }
3436
3437 /*
3438 * Create spare DMA map for rx buffers
3439 */
3440 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_WAITOK,
3441 &adapter->rx_sparemap);
3442 if (error) {
3443 device_printf(dev, "Unable to create spare RX DMA map\n");
3444 bus_dma_tag_destroy(adapter->rxtag);
3445 kfree(adapter->rx_buffer_area, M_DEVBUF);
3446 adapter->rx_buffer_area = NULL;
3447 return error;
3448 }
3449
3450 /*
3451 * Create DMA maps for rx buffers
3452 */
3453 for (i = 0; i < adapter->num_rx_desc; i++) {
3454 rx_buffer = &adapter->rx_buffer_area[i];
3455
3456 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_WAITOK,
3457 &rx_buffer->map);
3458 if (error) {
3459 device_printf(dev, "Unable to create RX DMA map\n");
3460 em_destroy_rx_ring(adapter, i);
3461 return error;
3462 }
3463 }
3464 return (0);
3465 }
3466
3467 static int
em_init_rx_ring(struct adapter * adapter)3468 em_init_rx_ring(struct adapter *adapter)
3469 {
3470 int i, error;
3471
3472 /* Reset descriptor ring */
3473 bzero(adapter->rx_desc_base,
3474 (sizeof(struct e1000_rx_desc)) * adapter->num_rx_desc);
3475
3476 /* Allocate new ones. */
3477 for (i = 0; i < adapter->num_rx_desc; i++) {
3478 error = em_newbuf(adapter, i, 1);
3479 if (error)
3480 return (error);
3481 }
3482
3483 /* Setup our descriptor pointers */
3484 adapter->next_rx_desc_to_check = 0;
3485
3486 return (0);
3487 }
3488
3489 static void
em_init_rx_unit(struct adapter * adapter)3490 em_init_rx_unit(struct adapter *adapter)
3491 {
3492 struct ifnet *ifp = &adapter->arpcom.ac_if;
3493 uint64_t bus_addr;
3494 uint32_t rctl, rxcsum;
3495
3496 /*
3497 * Make sure receives are disabled while setting
3498 * up the descriptor ring
3499 */
3500 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
3501 /* Do not disable if ever enabled on this hardware */
3502 if (adapter->hw.mac.type != e1000_82574 &&
3503 adapter->hw.mac.type != e1000_82583)
3504 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
3505
3506 if (adapter->hw.mac.type >= e1000_82540) {
3507 uint32_t itr;
3508
3509 /*
3510 * Set the interrupt throttling rate. Value is calculated
3511 * as ITR = 1 / (INT_THROTTLE_CEIL * 256ns)
3512 */
3513 if (adapter->int_throttle_ceil)
3514 itr = 1000000000 / 256 / adapter->int_throttle_ceil;
3515 else
3516 itr = 0;
3517 em_set_itr(adapter, itr);
3518 }
3519
3520 /* Disable accelerated ackknowledge */
3521 if (adapter->hw.mac.type == e1000_82574) {
3522 uint32_t rfctl;
3523
3524 rfctl = E1000_READ_REG(&adapter->hw, E1000_RFCTL);
3525 rfctl |= E1000_RFCTL_ACK_DIS;
3526 E1000_WRITE_REG(&adapter->hw, E1000_RFCTL, rfctl);
3527 }
3528
3529 /* Receive Checksum Offload for IP and TCP/UDP */
3530 rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM);
3531 if (ifp->if_capenable & IFCAP_RXCSUM)
3532 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
3533 else
3534 rxcsum &= ~(E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
3535 E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum);
3536
3537 /*
3538 * XXX TEMPORARY WORKAROUND: on some systems with 82573
3539 * long latencies are observed, like Lenovo X60. This
3540 * change eliminates the problem, but since having positive
3541 * values in RDTR is a known source of problems on other
3542 * platforms another solution is being sought.
3543 */
3544 if (em_82573_workaround && adapter->hw.mac.type == e1000_82573) {
3545 E1000_WRITE_REG(&adapter->hw, E1000_RADV, EM_RADV_82573);
3546 E1000_WRITE_REG(&adapter->hw, E1000_RDTR, EM_RDTR_82573);
3547 }
3548
3549 /*
3550 * Setup the Base and Length of the Rx Descriptor Ring
3551 */
3552 bus_addr = adapter->rxdma.dma_paddr;
3553 E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(0),
3554 adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
3555 E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(0),
3556 (uint32_t)(bus_addr >> 32));
3557 E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(0),
3558 (uint32_t)bus_addr);
3559
3560 /*
3561 * Setup the HW Rx Head and Tail Descriptor Pointers
3562 */
3563 E1000_WRITE_REG(&adapter->hw, E1000_RDH(0), 0);
3564 E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), adapter->num_rx_desc - 1);
3565
3566 /* Set PTHRESH for improved jumbo performance */
3567 if (ifp->if_mtu > ETHERMTU) {
3568 uint32_t rxdctl;
3569
3570 if (adapter->hw.mac.type == e1000_ich9lan ||
3571 adapter->hw.mac.type == e1000_pch2lan ||
3572 adapter->hw.mac.type == e1000_ich10lan) {
3573 rxdctl = E1000_READ_REG(&adapter->hw, E1000_RXDCTL(0));
3574 E1000_WRITE_REG(&adapter->hw, E1000_RXDCTL(0),
3575 rxdctl | 3);
3576 } else if (adapter->hw.mac.type == e1000_82574) {
3577 rxdctl = E1000_READ_REG(&adapter->hw, E1000_RXDCTL(0));
3578 rxdctl |= 0x20; /* PTHRESH */
3579 rxdctl |= 4 << 8; /* HTHRESH */
3580 rxdctl |= 4 << 16; /* WTHRESH */
3581 rxdctl |= 1 << 24; /* Switch to granularity */
3582 E1000_WRITE_REG(&adapter->hw, E1000_RXDCTL(0), rxdctl);
3583 }
3584 }
3585
3586 if (adapter->hw.mac.type >= e1000_pch2lan) {
3587 if (ifp->if_mtu > ETHERMTU)
3588 e1000_lv_jumbo_workaround_ich8lan(&adapter->hw, TRUE);
3589 else
3590 e1000_lv_jumbo_workaround_ich8lan(&adapter->hw, FALSE);
3591 }
3592
3593 /* Setup the Receive Control Register */
3594 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
3595 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
3596 E1000_RCTL_RDMTS_HALF |
3597 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
3598
3599 /* Make sure VLAN Filters are off */
3600 rctl &= ~E1000_RCTL_VFE;
3601
3602 if (e1000_tbi_sbp_enabled_82543(&adapter->hw))
3603 rctl |= E1000_RCTL_SBP;
3604 else
3605 rctl &= ~E1000_RCTL_SBP;
3606
3607 switch (adapter->rx_buffer_len) {
3608 default:
3609 case 2048:
3610 rctl |= E1000_RCTL_SZ_2048;
3611 break;
3612
3613 case 4096:
3614 rctl |= E1000_RCTL_SZ_4096 |
3615 E1000_RCTL_BSEX | E1000_RCTL_LPE;
3616 break;
3617
3618 case 8192:
3619 rctl |= E1000_RCTL_SZ_8192 |
3620 E1000_RCTL_BSEX | E1000_RCTL_LPE;
3621 break;
3622
3623 case 16384:
3624 rctl |= E1000_RCTL_SZ_16384 |
3625 E1000_RCTL_BSEX | E1000_RCTL_LPE;
3626 break;
3627 }
3628
3629 if (ifp->if_mtu > ETHERMTU)
3630 rctl |= E1000_RCTL_LPE;
3631 else
3632 rctl &= ~E1000_RCTL_LPE;
3633
3634 /* Enable Receives */
3635 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
3636 }
3637
3638 static void
em_destroy_rx_ring(struct adapter * adapter,int ndesc)3639 em_destroy_rx_ring(struct adapter *adapter, int ndesc)
3640 {
3641 struct em_buffer *rx_buffer;
3642 int i;
3643
3644 if (adapter->rx_buffer_area == NULL)
3645 return;
3646
3647 for (i = 0; i < ndesc; i++) {
3648 rx_buffer = &adapter->rx_buffer_area[i];
3649
3650 KKASSERT(rx_buffer->m_head == NULL);
3651 bus_dmamap_destroy(adapter->rxtag, rx_buffer->map);
3652 }
3653 bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap);
3654 bus_dma_tag_destroy(adapter->rxtag);
3655
3656 kfree(adapter->rx_buffer_area, M_DEVBUF);
3657 adapter->rx_buffer_area = NULL;
3658 }
3659
3660 static void
em_rxeof(struct adapter * adapter,int count)3661 em_rxeof(struct adapter *adapter, int count)
3662 {
3663 struct ifnet *ifp = &adapter->arpcom.ac_if;
3664 uint8_t status, accept_frame = 0, eop = 0;
3665 uint16_t len, desc_len, prev_len_adj;
3666 struct e1000_rx_desc *current_desc;
3667 struct mbuf *mp;
3668 int i;
3669
3670 i = adapter->next_rx_desc_to_check;
3671 current_desc = &adapter->rx_desc_base[i];
3672
3673 if (!(current_desc->status & E1000_RXD_STAT_DD))
3674 return;
3675
3676 while ((current_desc->status & E1000_RXD_STAT_DD) && count != 0) {
3677 struct mbuf *m = NULL;
3678
3679 logif(pkt_receive);
3680
3681 mp = adapter->rx_buffer_area[i].m_head;
3682
3683 /*
3684 * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT
3685 * needs to access the last received byte in the mbuf.
3686 */
3687 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map,
3688 BUS_DMASYNC_POSTREAD);
3689
3690 accept_frame = 1;
3691 prev_len_adj = 0;
3692 desc_len = le16toh(current_desc->length);
3693 status = current_desc->status;
3694 if (status & E1000_RXD_STAT_EOP) {
3695 count--;
3696 eop = 1;
3697 if (desc_len < ETHER_CRC_LEN) {
3698 len = 0;
3699 prev_len_adj = ETHER_CRC_LEN - desc_len;
3700 } else {
3701 len = desc_len - ETHER_CRC_LEN;
3702 }
3703 } else {
3704 eop = 0;
3705 len = desc_len;
3706 }
3707
3708 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
3709 uint8_t last_byte;
3710 uint32_t pkt_len = desc_len;
3711
3712 if (adapter->fmp != NULL)
3713 pkt_len += adapter->fmp->m_pkthdr.len;
3714
3715 last_byte = *(mtod(mp, caddr_t) + desc_len - 1);
3716 if (TBI_ACCEPT(&adapter->hw, status,
3717 current_desc->errors, pkt_len, last_byte,
3718 adapter->min_frame_size,
3719 adapter->hw.mac.max_frame_size)) {
3720 e1000_tbi_adjust_stats_82543(&adapter->hw,
3721 &adapter->stats, pkt_len,
3722 adapter->hw.mac.addr,
3723 adapter->hw.mac.max_frame_size);
3724 if (len > 0)
3725 len--;
3726 } else {
3727 accept_frame = 0;
3728 }
3729 }
3730
3731 if (accept_frame) {
3732 if (em_newbuf(adapter, i, 0) != 0) {
3733 IFNET_STAT_INC(ifp, iqdrops, 1);
3734 goto discard;
3735 }
3736
3737 /* Assign correct length to the current fragment */
3738 mp->m_len = len;
3739
3740 if (adapter->fmp == NULL) {
3741 mp->m_pkthdr.len = len;
3742 adapter->fmp = mp; /* Store the first mbuf */
3743 adapter->lmp = mp;
3744 } else {
3745 /*
3746 * Chain mbuf's together
3747 */
3748
3749 /*
3750 * Adjust length of previous mbuf in chain if
3751 * we received less than 4 bytes in the last
3752 * descriptor.
3753 */
3754 if (prev_len_adj > 0) {
3755 adapter->lmp->m_len -= prev_len_adj;
3756 adapter->fmp->m_pkthdr.len -=
3757 prev_len_adj;
3758 }
3759 adapter->lmp->m_next = mp;
3760 adapter->lmp = adapter->lmp->m_next;
3761 adapter->fmp->m_pkthdr.len += len;
3762 }
3763
3764 if (eop) {
3765 adapter->fmp->m_pkthdr.rcvif = ifp;
3766 IFNET_STAT_INC(ifp, ipackets, 1);
3767
3768 if (ifp->if_capenable & IFCAP_RXCSUM) {
3769 em_rxcsum(adapter, current_desc,
3770 adapter->fmp);
3771 }
3772
3773 if (status & E1000_RXD_STAT_VP) {
3774 adapter->fmp->m_pkthdr.ether_vlantag =
3775 (le16toh(current_desc->special) &
3776 E1000_RXD_SPC_VLAN_MASK);
3777 adapter->fmp->m_flags |= M_VLANTAG;
3778 }
3779 m = adapter->fmp;
3780 adapter->fmp = NULL;
3781 adapter->lmp = NULL;
3782 }
3783 } else {
3784 IFNET_STAT_INC(ifp, ierrors, 1);
3785 discard:
3786 #ifdef foo
3787 /* Reuse loaded DMA map and just update mbuf chain */
3788 mp = adapter->rx_buffer_area[i].m_head;
3789 mp->m_len = mp->m_pkthdr.len = MCLBYTES;
3790 mp->m_data = mp->m_ext.ext_buf;
3791 mp->m_next = NULL;
3792 if (adapter->hw.mac.max_frame_size <=
3793 (MCLBYTES - ETHER_ALIGN))
3794 m_adj(mp, ETHER_ALIGN);
3795 #endif
3796 if (adapter->fmp != NULL) {
3797 m_freem(adapter->fmp);
3798 adapter->fmp = NULL;
3799 adapter->lmp = NULL;
3800 }
3801 m = NULL;
3802 }
3803
3804 /* Zero out the receive descriptors status. */
3805 current_desc->status = 0;
3806
3807 if (m != NULL)
3808 ifp->if_input(ifp, m, NULL, -1);
3809
3810 /* Advance our pointers to the next descriptor. */
3811 if (++i == adapter->num_rx_desc)
3812 i = 0;
3813 current_desc = &adapter->rx_desc_base[i];
3814 }
3815 adapter->next_rx_desc_to_check = i;
3816
3817 /* Advance the E1000's Receive Queue #0 "Tail Pointer". */
3818 if (--i < 0)
3819 i = adapter->num_rx_desc - 1;
3820 E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i);
3821 }
3822
3823 static void
em_rxcsum(struct adapter * adapter,struct e1000_rx_desc * rx_desc,struct mbuf * mp)3824 em_rxcsum(struct adapter *adapter, struct e1000_rx_desc *rx_desc,
3825 struct mbuf *mp)
3826 {
3827 /* 82543 or newer only */
3828 if (adapter->hw.mac.type < e1000_82543 ||
3829 /* Ignore Checksum bit is set */
3830 (rx_desc->status & E1000_RXD_STAT_IXSM))
3831 return;
3832
3833 if ((rx_desc->status & E1000_RXD_STAT_IPCS) &&
3834 !(rx_desc->errors & E1000_RXD_ERR_IPE)) {
3835 /* IP Checksum Good */
3836 mp->m_pkthdr.csum_flags |= CSUM_IP_CHECKED | CSUM_IP_VALID;
3837 }
3838
3839 if ((rx_desc->status & E1000_RXD_STAT_TCPCS) &&
3840 !(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
3841 mp->m_pkthdr.csum_flags |= CSUM_DATA_VALID |
3842 CSUM_PSEUDO_HDR |
3843 CSUM_FRAG_NOT_CHECKED;
3844 mp->m_pkthdr.csum_data = htons(0xffff);
3845 }
3846 }
3847
3848 static void
em_enable_intr(struct adapter * adapter)3849 em_enable_intr(struct adapter *adapter)
3850 {
3851 uint32_t ims_mask = IMS_ENABLE_MASK;
3852
3853 lwkt_serialize_handler_enable(adapter->arpcom.ac_if.if_serializer);
3854
3855 #if 0
3856 /* XXX MSIX */
3857 if (adapter->hw.mac.type == e1000_82574) {
3858 E1000_WRITE_REG(&adapter->hw, EM_EIAC, EM_MSIX_MASK);
3859 ims_mask |= EM_MSIX_MASK;
3860 }
3861 #endif
3862 E1000_WRITE_REG(&adapter->hw, E1000_IMS, ims_mask);
3863 }
3864
3865 static void
em_disable_intr(struct adapter * adapter)3866 em_disable_intr(struct adapter *adapter)
3867 {
3868 uint32_t clear = 0xffffffff;
3869
3870 /*
3871 * The first version of 82542 had an errata where when link was forced
3872 * it would stay up even up even if the cable was disconnected.
3873 * Sequence errors were used to detect the disconnect and then the
3874 * driver would unforce the link. This code in the in the ISR. For
3875 * this to work correctly the Sequence error interrupt had to be
3876 * enabled all the time.
3877 */
3878 if (adapter->hw.mac.type == e1000_82542 &&
3879 adapter->hw.revision_id == E1000_REVISION_2)
3880 clear &= ~E1000_ICR_RXSEQ;
3881 else if (adapter->hw.mac.type == e1000_82574)
3882 E1000_WRITE_REG(&adapter->hw, EM_EIAC, 0);
3883
3884 E1000_WRITE_REG(&adapter->hw, E1000_IMC, clear);
3885
3886 adapter->npoll.ifpc_stcount = 0;
3887
3888 lwkt_serialize_handler_disable(adapter->arpcom.ac_if.if_serializer);
3889 }
3890
3891 /*
3892 * Bit of a misnomer, what this really means is
3893 * to enable OS management of the system... aka
3894 * to disable special hardware management features
3895 */
3896 static void
em_get_mgmt(struct adapter * adapter)3897 em_get_mgmt(struct adapter *adapter)
3898 {
3899 /* A shared code workaround */
3900 #define E1000_82542_MANC2H E1000_MANC2H
3901 if (adapter->flags & EM_FLAG_HAS_MGMT) {
3902 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
3903 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
3904
3905 /* disable hardware interception of ARP */
3906 manc &= ~(E1000_MANC_ARP_EN);
3907
3908 /* enable receiving management packets to the host */
3909 if (adapter->hw.mac.type >= e1000_82571) {
3910 manc |= E1000_MANC_EN_MNG2HOST;
3911 #define E1000_MNG2HOST_PORT_623 (1 << 5)
3912 #define E1000_MNG2HOST_PORT_664 (1 << 6)
3913 manc2h |= E1000_MNG2HOST_PORT_623;
3914 manc2h |= E1000_MNG2HOST_PORT_664;
3915 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
3916 }
3917
3918 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
3919 }
3920 }
3921
3922 /*
3923 * Give control back to hardware management
3924 * controller if there is one.
3925 */
3926 static void
em_rel_mgmt(struct adapter * adapter)3927 em_rel_mgmt(struct adapter *adapter)
3928 {
3929 if (adapter->flags & EM_FLAG_HAS_MGMT) {
3930 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
3931
3932 /* re-enable hardware interception of ARP */
3933 manc |= E1000_MANC_ARP_EN;
3934
3935 if (adapter->hw.mac.type >= e1000_82571)
3936 manc &= ~E1000_MANC_EN_MNG2HOST;
3937
3938 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
3939 }
3940 }
3941
3942 /*
3943 * em_get_hw_control() sets {CTRL_EXT|FWSM}:DRV_LOAD bit.
3944 * For ASF and Pass Through versions of f/w this means that
3945 * the driver is loaded. For AMT version (only with 82573)
3946 * of the f/w this means that the network i/f is open.
3947 */
3948 static void
em_get_hw_control(struct adapter * adapter)3949 em_get_hw_control(struct adapter *adapter)
3950 {
3951 /* Let firmware know the driver has taken over */
3952 if (adapter->hw.mac.type == e1000_82573) {
3953 uint32_t swsm;
3954
3955 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
3956 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
3957 swsm | E1000_SWSM_DRV_LOAD);
3958 } else {
3959 uint32_t ctrl_ext;
3960
3961 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
3962 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
3963 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
3964 }
3965 adapter->flags |= EM_FLAG_HW_CTRL;
3966 }
3967
3968 /*
3969 * em_rel_hw_control() resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
3970 * For ASF and Pass Through versions of f/w this means that the
3971 * driver is no longer loaded. For AMT version (only with 82573)
3972 * of the f/w this means that the network i/f is closed.
3973 */
3974 static void
em_rel_hw_control(struct adapter * adapter)3975 em_rel_hw_control(struct adapter *adapter)
3976 {
3977 if ((adapter->flags & EM_FLAG_HW_CTRL) == 0)
3978 return;
3979 adapter->flags &= ~EM_FLAG_HW_CTRL;
3980
3981 /* Let firmware taken over control of h/w */
3982 if (adapter->hw.mac.type == e1000_82573) {
3983 uint32_t swsm;
3984
3985 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
3986 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
3987 swsm & ~E1000_SWSM_DRV_LOAD);
3988 } else {
3989 uint32_t ctrl_ext;
3990
3991 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
3992 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
3993 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
3994 }
3995 }
3996
3997 static int
em_is_valid_eaddr(const uint8_t * addr)3998 em_is_valid_eaddr(const uint8_t *addr)
3999 {
4000 char zero_addr[ETHER_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 };
4001
4002 if ((addr[0] & 1) || !bcmp(addr, zero_addr, ETHER_ADDR_LEN))
4003 return (FALSE);
4004
4005 return (TRUE);
4006 }
4007
4008 /*
4009 * Enable PCI Wake On Lan capability
4010 */
4011 static void
em_enable_wol(device_t dev)4012 em_enable_wol(device_t dev)
4013 {
4014 uint16_t cap, status;
4015 uint8_t id;
4016
4017 /* First find the capabilities pointer*/
4018 cap = pci_read_config(dev, PCIR_CAP_PTR, 2);
4019
4020 /* Read the PM Capabilities */
4021 id = pci_read_config(dev, cap, 1);
4022 if (id != PCIY_PMG) /* Something wrong */
4023 return;
4024
4025 /*
4026 * OK, we have the power capabilities,
4027 * so now get the status register
4028 */
4029 cap += PCIR_POWER_STATUS;
4030 status = pci_read_config(dev, cap, 2);
4031 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
4032 pci_write_config(dev, cap, status, 2);
4033 }
4034
4035
4036 /*
4037 * 82544 Coexistence issue workaround.
4038 * There are 2 issues.
4039 * 1. Transmit Hang issue.
4040 * To detect this issue, following equation can be used...
4041 * SIZE[3:0] + ADDR[2:0] = SUM[3:0].
4042 * If SUM[3:0] is in between 1 to 4, we will have this issue.
4043 *
4044 * 2. DAC issue.
4045 * To detect this issue, following equation can be used...
4046 * SIZE[3:0] + ADDR[2:0] = SUM[3:0].
4047 * If SUM[3:0] is in between 9 to c, we will have this issue.
4048 *
4049 * WORKAROUND:
4050 * Make sure we do not have ending address
4051 * as 1,2,3,4(Hang) or 9,a,b,c (DAC)
4052 */
4053 static uint32_t
em_82544_fill_desc(bus_addr_t address,uint32_t length,PDESC_ARRAY desc_array)4054 em_82544_fill_desc(bus_addr_t address, uint32_t length, PDESC_ARRAY desc_array)
4055 {
4056 uint32_t safe_terminator;
4057
4058 /*
4059 * Since issue is sensitive to length and address.
4060 * Let us first check the address...
4061 */
4062 if (length <= 4) {
4063 desc_array->descriptor[0].address = address;
4064 desc_array->descriptor[0].length = length;
4065 desc_array->elements = 1;
4066 return (desc_array->elements);
4067 }
4068
4069 safe_terminator =
4070 (uint32_t)((((uint32_t)address & 0x7) + (length & 0xF)) & 0xF);
4071
4072 /* If it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
4073 if (safe_terminator == 0 ||
4074 (safe_terminator > 4 && safe_terminator < 9) ||
4075 (safe_terminator > 0xC && safe_terminator <= 0xF)) {
4076 desc_array->descriptor[0].address = address;
4077 desc_array->descriptor[0].length = length;
4078 desc_array->elements = 1;
4079 return (desc_array->elements);
4080 }
4081
4082 desc_array->descriptor[0].address = address;
4083 desc_array->descriptor[0].length = length - 4;
4084 desc_array->descriptor[1].address = address + (length - 4);
4085 desc_array->descriptor[1].length = 4;
4086 desc_array->elements = 2;
4087 return (desc_array->elements);
4088 }
4089
4090 static void
em_update_stats(struct adapter * adapter)4091 em_update_stats(struct adapter *adapter)
4092 {
4093 struct ifnet *ifp = &adapter->arpcom.ac_if;
4094
4095 if (adapter->hw.phy.media_type == e1000_media_type_copper ||
4096 (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
4097 adapter->stats.symerrs +=
4098 E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
4099 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
4100 }
4101 adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
4102 adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
4103 adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
4104 adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
4105
4106 adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
4107 adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
4108 adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
4109 adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
4110 adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
4111 adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
4112 adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
4113 adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
4114 adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
4115 adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
4116 adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
4117 adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
4118 adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
4119 adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
4120 adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
4121 adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
4122 adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
4123 adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
4124 adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
4125 adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
4126
4127 /* For the 64-bit byte counters the low dword must be read first. */
4128 /* Both registers clear on the read of the high dword */
4129
4130 adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH);
4131 adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH);
4132
4133 adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
4134 adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
4135 adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
4136 adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
4137 adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
4138
4139 adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
4140 adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
4141
4142 adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
4143 adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
4144 adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
4145 adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
4146 adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
4147 adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
4148 adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
4149 adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
4150 adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
4151 adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
4152
4153 if (adapter->hw.mac.type >= e1000_82543) {
4154 adapter->stats.algnerrc +=
4155 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
4156 adapter->stats.rxerrc +=
4157 E1000_READ_REG(&adapter->hw, E1000_RXERRC);
4158 adapter->stats.tncrs +=
4159 E1000_READ_REG(&adapter->hw, E1000_TNCRS);
4160 adapter->stats.cexterr +=
4161 E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
4162 adapter->stats.tsctc +=
4163 E1000_READ_REG(&adapter->hw, E1000_TSCTC);
4164 adapter->stats.tsctfc +=
4165 E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
4166 }
4167
4168 IFNET_STAT_SET(ifp, collisions, adapter->stats.colc);
4169
4170 /* Rx Errors */
4171 IFNET_STAT_SET(ifp, ierrors,
4172 adapter->dropped_pkts + adapter->stats.rxerrc +
4173 adapter->stats.crcerrs + adapter->stats.algnerrc +
4174 adapter->stats.ruc + adapter->stats.roc +
4175 adapter->stats.mpc + adapter->stats.cexterr);
4176
4177 /* Tx Errors */
4178 IFNET_STAT_SET(ifp, oerrors,
4179 adapter->stats.ecol + adapter->stats.latecol +
4180 adapter->watchdog_events);
4181 }
4182
4183 static void
em_print_debug_info(struct adapter * adapter)4184 em_print_debug_info(struct adapter *adapter)
4185 {
4186 device_t dev = adapter->dev;
4187 uint8_t *hw_addr = adapter->hw.hw_addr;
4188
4189 device_printf(dev, "Adapter hardware address = %p \n", hw_addr);
4190 device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n",
4191 E1000_READ_REG(&adapter->hw, E1000_CTRL),
4192 E1000_READ_REG(&adapter->hw, E1000_RCTL));
4193 device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n",
4194 ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\
4195 (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) );
4196 device_printf(dev, "Flow control watermarks high = %d low = %d\n",
4197 adapter->hw.fc.high_water,
4198 adapter->hw.fc.low_water);
4199 device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n",
4200 E1000_READ_REG(&adapter->hw, E1000_TIDV),
4201 E1000_READ_REG(&adapter->hw, E1000_TADV));
4202 device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n",
4203 E1000_READ_REG(&adapter->hw, E1000_RDTR),
4204 E1000_READ_REG(&adapter->hw, E1000_RADV));
4205 device_printf(dev, "fifo workaround = %lld, fifo_reset_count = %lld\n",
4206 (long long)adapter->tx_fifo_wrk_cnt,
4207 (long long)adapter->tx_fifo_reset_cnt);
4208 device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
4209 E1000_READ_REG(&adapter->hw, E1000_TDH(0)),
4210 E1000_READ_REG(&adapter->hw, E1000_TDT(0)));
4211 device_printf(dev, "hw rdh = %d, hw rdt = %d\n",
4212 E1000_READ_REG(&adapter->hw, E1000_RDH(0)),
4213 E1000_READ_REG(&adapter->hw, E1000_RDT(0)));
4214 device_printf(dev, "Num Tx descriptors avail = %d\n",
4215 adapter->num_tx_desc_avail);
4216 device_printf(dev, "Tx Descriptors not avail1 = %ld\n",
4217 adapter->no_tx_desc_avail1);
4218 device_printf(dev, "Tx Descriptors not avail2 = %ld\n",
4219 adapter->no_tx_desc_avail2);
4220 device_printf(dev, "Std mbuf failed = %ld\n",
4221 adapter->mbuf_alloc_failed);
4222 device_printf(dev, "Std mbuf cluster failed = %ld\n",
4223 adapter->mbuf_cluster_failed);
4224 device_printf(dev, "Driver dropped packets = %ld\n",
4225 adapter->dropped_pkts);
4226 device_printf(dev, "Driver tx dma failure in encap = %ld\n",
4227 adapter->no_tx_dma_setup);
4228 }
4229
4230 static void
em_print_hw_stats(struct adapter * adapter)4231 em_print_hw_stats(struct adapter *adapter)
4232 {
4233 device_t dev = adapter->dev;
4234
4235 device_printf(dev, "Excessive collisions = %lld\n",
4236 (long long)adapter->stats.ecol);
4237 #if (DEBUG_HW > 0) /* Dont output these errors normally */
4238 device_printf(dev, "Symbol errors = %lld\n",
4239 (long long)adapter->stats.symerrs);
4240 #endif
4241 device_printf(dev, "Sequence errors = %lld\n",
4242 (long long)adapter->stats.sec);
4243 device_printf(dev, "Defer count = %lld\n",
4244 (long long)adapter->stats.dc);
4245 device_printf(dev, "Missed Packets = %lld\n",
4246 (long long)adapter->stats.mpc);
4247 device_printf(dev, "Receive No Buffers = %lld\n",
4248 (long long)adapter->stats.rnbc);
4249 /* RLEC is inaccurate on some hardware, calculate our own. */
4250 device_printf(dev, "Receive Length Errors = %lld\n",
4251 ((long long)adapter->stats.roc + (long long)adapter->stats.ruc));
4252 device_printf(dev, "Receive errors = %lld\n",
4253 (long long)adapter->stats.rxerrc);
4254 device_printf(dev, "Crc errors = %lld\n",
4255 (long long)adapter->stats.crcerrs);
4256 device_printf(dev, "Alignment errors = %lld\n",
4257 (long long)adapter->stats.algnerrc);
4258 device_printf(dev, "Collision/Carrier extension errors = %lld\n",
4259 (long long)adapter->stats.cexterr);
4260 device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns);
4261 device_printf(dev, "watchdog timeouts = %ld\n",
4262 adapter->watchdog_events);
4263 device_printf(dev, "XON Rcvd = %lld\n",
4264 (long long)adapter->stats.xonrxc);
4265 device_printf(dev, "XON Xmtd = %lld\n",
4266 (long long)adapter->stats.xontxc);
4267 device_printf(dev, "XOFF Rcvd = %lld\n",
4268 (long long)adapter->stats.xoffrxc);
4269 device_printf(dev, "XOFF Xmtd = %lld\n",
4270 (long long)adapter->stats.xofftxc);
4271 device_printf(dev, "Good Packets Rcvd = %lld\n",
4272 (long long)adapter->stats.gprc);
4273 device_printf(dev, "Good Packets Xmtd = %lld\n",
4274 (long long)adapter->stats.gptc);
4275 }
4276
4277 static void
em_print_nvm_info(struct adapter * adapter)4278 em_print_nvm_info(struct adapter *adapter)
4279 {
4280 uint16_t eeprom_data;
4281 int i, j, row = 0;
4282
4283 /* Its a bit crude, but it gets the job done */
4284 kprintf("\nInterface EEPROM Dump:\n");
4285 kprintf("Offset\n0x0000 ");
4286 for (i = 0, j = 0; i < 32; i++, j++) {
4287 if (j == 8) { /* Make the offset block */
4288 j = 0; ++row;
4289 kprintf("\n0x00%x0 ",row);
4290 }
4291 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
4292 kprintf("%04x ", eeprom_data);
4293 }
4294 kprintf("\n");
4295 }
4296
4297 static int
em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)4298 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
4299 {
4300 struct adapter *adapter;
4301 struct ifnet *ifp;
4302 int error, result;
4303
4304 result = -1;
4305 error = sysctl_handle_int(oidp, &result, 0, req);
4306 if (error || !req->newptr)
4307 return (error);
4308
4309 adapter = (struct adapter *)arg1;
4310 ifp = &adapter->arpcom.ac_if;
4311
4312 lwkt_serialize_enter(ifp->if_serializer);
4313
4314 if (result == 1)
4315 em_print_debug_info(adapter);
4316
4317 /*
4318 * This value will cause a hex dump of the
4319 * first 32 16-bit words of the EEPROM to
4320 * the screen.
4321 */
4322 if (result == 2)
4323 em_print_nvm_info(adapter);
4324
4325 lwkt_serialize_exit(ifp->if_serializer);
4326
4327 return (error);
4328 }
4329
4330 static int
em_sysctl_stats(SYSCTL_HANDLER_ARGS)4331 em_sysctl_stats(SYSCTL_HANDLER_ARGS)
4332 {
4333 int error, result;
4334
4335 result = -1;
4336 error = sysctl_handle_int(oidp, &result, 0, req);
4337 if (error || !req->newptr)
4338 return (error);
4339
4340 if (result == 1) {
4341 struct adapter *adapter = (struct adapter *)arg1;
4342 struct ifnet *ifp = &adapter->arpcom.ac_if;
4343
4344 lwkt_serialize_enter(ifp->if_serializer);
4345 em_print_hw_stats(adapter);
4346 lwkt_serialize_exit(ifp->if_serializer);
4347 }
4348 return (error);
4349 }
4350
4351 static void
em_add_sysctl(struct adapter * adapter)4352 em_add_sysctl(struct adapter *adapter)
4353 {
4354 struct sysctl_ctx_list *ctx;
4355 struct sysctl_oid *tree;
4356
4357 ctx = device_get_sysctl_ctx(adapter->dev);
4358 tree = device_get_sysctl_tree(adapter->dev);
4359 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
4360 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
4361 em_sysctl_debug_info, "I", "Debug Information");
4362
4363 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
4364 OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
4365 em_sysctl_stats, "I", "Statistics");
4366
4367 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree),
4368 OID_AUTO, "rxd", CTLFLAG_RD,
4369 &adapter->num_rx_desc, 0, NULL);
4370 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree),
4371 OID_AUTO, "txd", CTLFLAG_RD,
4372 &adapter->num_tx_desc, 0, NULL);
4373
4374 if (adapter->hw.mac.type >= e1000_82540) {
4375 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
4376 OID_AUTO, "int_throttle_ceil",
4377 CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
4378 em_sysctl_int_throttle, "I",
4379 "interrupt throttling rate");
4380 }
4381 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
4382 OID_AUTO, "int_tx_nsegs",
4383 CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
4384 em_sysctl_int_tx_nsegs, "I",
4385 "# segments per TX interrupt");
4386 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree),
4387 OID_AUTO, "wreg_tx_nsegs", CTLFLAG_RW,
4388 &adapter->tx_wreg_nsegs, 0,
4389 "# segments before write to hardware register");
4390 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "tx_nmbuf",
4391 CTLFLAG_RD, &adapter->tx_nmbuf, 0, "# of pending TX mbufs");
4392 SYSCTL_ADD_ULONG(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "tx_gc",
4393 CTLFLAG_RW, &adapter->tx_gc, "# of TX GC");
4394 }
4395
4396 static int
em_sysctl_int_throttle(SYSCTL_HANDLER_ARGS)4397 em_sysctl_int_throttle(SYSCTL_HANDLER_ARGS)
4398 {
4399 struct adapter *adapter = (void *)arg1;
4400 struct ifnet *ifp = &adapter->arpcom.ac_if;
4401 int error, throttle;
4402
4403 throttle = adapter->int_throttle_ceil;
4404 error = sysctl_handle_int(oidp, &throttle, 0, req);
4405 if (error || req->newptr == NULL)
4406 return error;
4407 if (throttle < 0 || throttle > 1000000000 / 256)
4408 return EINVAL;
4409
4410 if (throttle) {
4411 /*
4412 * Set the interrupt throttling rate in 256ns increments,
4413 * recalculate sysctl value assignment to get exact frequency.
4414 */
4415 throttle = 1000000000 / 256 / throttle;
4416
4417 /* Upper 16bits of ITR is reserved and should be zero */
4418 if (throttle & 0xffff0000)
4419 return EINVAL;
4420 }
4421
4422 lwkt_serialize_enter(ifp->if_serializer);
4423
4424 if (throttle)
4425 adapter->int_throttle_ceil = 1000000000 / 256 / throttle;
4426 else
4427 adapter->int_throttle_ceil = 0;
4428
4429 if (ifp->if_flags & IFF_RUNNING)
4430 em_set_itr(adapter, throttle);
4431
4432 lwkt_serialize_exit(ifp->if_serializer);
4433
4434 if (bootverbose) {
4435 if_printf(ifp, "Interrupt moderation set to %d/sec\n",
4436 adapter->int_throttle_ceil);
4437 }
4438 return 0;
4439 }
4440
4441 static int
em_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS)4442 em_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS)
4443 {
4444 struct adapter *adapter = (void *)arg1;
4445 struct ifnet *ifp = &adapter->arpcom.ac_if;
4446 int error, segs;
4447
4448 segs = adapter->tx_int_nsegs;
4449 error = sysctl_handle_int(oidp, &segs, 0, req);
4450 if (error || req->newptr == NULL)
4451 return error;
4452 if (segs <= 0)
4453 return EINVAL;
4454
4455 lwkt_serialize_enter(ifp->if_serializer);
4456
4457 /*
4458 * Don't allow int_tx_nsegs to become:
4459 * o Less the oact_tx_desc
4460 * o Too large that no TX desc will cause TX interrupt to
4461 * be generated (OACTIVE will never recover)
4462 * o Too small that will cause tx_dd[] overflow
4463 */
4464 if (segs < adapter->oact_tx_desc ||
4465 segs >= adapter->num_tx_desc - adapter->oact_tx_desc ||
4466 segs < adapter->num_tx_desc / EM_TXDD_SAFE) {
4467 error = EINVAL;
4468 } else {
4469 error = 0;
4470 adapter->tx_int_nsegs = segs;
4471 }
4472
4473 lwkt_serialize_exit(ifp->if_serializer);
4474
4475 return error;
4476 }
4477
4478 static void
em_set_itr(struct adapter * adapter,uint32_t itr)4479 em_set_itr(struct adapter *adapter, uint32_t itr)
4480 {
4481 E1000_WRITE_REG(&adapter->hw, E1000_ITR, itr);
4482 if (adapter->hw.mac.type == e1000_82574) {
4483 int i;
4484
4485 /*
4486 * When using MSIX interrupts we need to
4487 * throttle using the EITR register
4488 */
4489 for (i = 0; i < 4; ++i) {
4490 E1000_WRITE_REG(&adapter->hw,
4491 E1000_EITR_82574(i), itr);
4492 }
4493 }
4494 }
4495
4496 static void
em_disable_aspm(struct adapter * adapter)4497 em_disable_aspm(struct adapter *adapter)
4498 {
4499 uint16_t link_cap, link_ctrl, disable;
4500 uint8_t pcie_ptr, reg;
4501 device_t dev = adapter->dev;
4502
4503 switch (adapter->hw.mac.type) {
4504 case e1000_82571:
4505 case e1000_82572:
4506 case e1000_82573:
4507 /*
4508 * 82573 specification update
4509 * errata #8 disable L0s
4510 * errata #41 disable L1
4511 *
4512 * 82571/82572 specification update
4513 # errata #13 disable L1
4514 * errata #68 disable L0s
4515 */
4516 disable = PCIEM_LNKCTL_ASPM_L0S | PCIEM_LNKCTL_ASPM_L1;
4517 break;
4518
4519 case e1000_82574:
4520 case e1000_82583:
4521 /*
4522 * 82574 specification update errata #20
4523 * 82583 specification update errata #9
4524 *
4525 * There is no need to disable L1
4526 */
4527 disable = PCIEM_LNKCTL_ASPM_L0S;
4528 break;
4529
4530 default:
4531 return;
4532 }
4533
4534 pcie_ptr = pci_get_pciecap_ptr(dev);
4535 if (pcie_ptr == 0)
4536 return;
4537
4538 link_cap = pci_read_config(dev, pcie_ptr + PCIER_LINKCAP, 2);
4539 if ((link_cap & PCIEM_LNKCAP_ASPM_MASK) == 0)
4540 return;
4541
4542 if (bootverbose) {
4543 if_printf(&adapter->arpcom.ac_if,
4544 "disable ASPM %#02x\n", disable);
4545 }
4546
4547 reg = pcie_ptr + PCIER_LINKCTRL;
4548 link_ctrl = pci_read_config(dev, reg, 2);
4549 link_ctrl &= ~disable;
4550 pci_write_config(dev, reg, link_ctrl, 2);
4551 }
4552
4553 static int
em_tso_pullup(struct adapter * adapter,struct mbuf ** mp)4554 em_tso_pullup(struct adapter *adapter, struct mbuf **mp)
4555 {
4556 int iphlen, hoff, thoff, ex = 0;
4557 struct mbuf *m;
4558 struct ip *ip;
4559
4560 m = *mp;
4561 KASSERT(M_WRITABLE(m), ("TSO mbuf not writable"));
4562
4563 iphlen = m->m_pkthdr.csum_iphlen;
4564 thoff = m->m_pkthdr.csum_thlen;
4565 hoff = m->m_pkthdr.csum_lhlen;
4566
4567 KASSERT(iphlen > 0, ("invalid ip hlen"));
4568 KASSERT(thoff > 0, ("invalid tcp hlen"));
4569 KASSERT(hoff > 0, ("invalid ether hlen"));
4570
4571 if (adapter->flags & EM_FLAG_TSO_PULLEX)
4572 ex = 4;
4573
4574 if (m->m_len < hoff + iphlen + thoff + ex) {
4575 m = m_pullup(m, hoff + iphlen + thoff + ex);
4576 if (m == NULL) {
4577 *mp = NULL;
4578 return ENOBUFS;
4579 }
4580 *mp = m;
4581 }
4582 ip = mtodoff(m, struct ip *, hoff);
4583 ip->ip_len = 0;
4584
4585 return 0;
4586 }
4587
4588 static int
em_tso_setup(struct adapter * adapter,struct mbuf * mp,uint32_t * txd_upper,uint32_t * txd_lower)4589 em_tso_setup(struct adapter *adapter, struct mbuf *mp,
4590 uint32_t *txd_upper, uint32_t *txd_lower)
4591 {
4592 struct e1000_context_desc *TXD;
4593 int hoff, iphlen, thoff, hlen;
4594 int mss, pktlen, curr_txd;
4595
4596 iphlen = mp->m_pkthdr.csum_iphlen;
4597 thoff = mp->m_pkthdr.csum_thlen;
4598 hoff = mp->m_pkthdr.csum_lhlen;
4599 mss = mp->m_pkthdr.tso_segsz;
4600 pktlen = mp->m_pkthdr.len;
4601
4602 if (adapter->csum_flags == CSUM_TSO &&
4603 adapter->csum_iphlen == iphlen &&
4604 adapter->csum_lhlen == hoff &&
4605 adapter->csum_thlen == thoff &&
4606 adapter->csum_mss == mss &&
4607 adapter->csum_pktlen == pktlen) {
4608 *txd_upper = adapter->csum_txd_upper;
4609 *txd_lower = adapter->csum_txd_lower;
4610 return 0;
4611 }
4612 hlen = hoff + iphlen + thoff;
4613
4614 /*
4615 * Setup a new TSO context.
4616 */
4617
4618 curr_txd = adapter->next_avail_tx_desc;
4619 TXD = (struct e1000_context_desc *)&adapter->tx_desc_base[curr_txd];
4620
4621 *txd_lower = E1000_TXD_CMD_DEXT | /* Extended descr type */
4622 E1000_TXD_DTYP_D | /* Data descr type */
4623 E1000_TXD_CMD_TSE; /* Do TSE on this packet */
4624
4625 /* IP and/or TCP header checksum calculation and insertion. */
4626 *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8;
4627
4628 /*
4629 * Start offset for header checksum calculation.
4630 * End offset for header checksum calculation.
4631 * Offset of place put the checksum.
4632 */
4633 TXD->lower_setup.ip_fields.ipcss = hoff;
4634 TXD->lower_setup.ip_fields.ipcse = htole16(hoff + iphlen - 1);
4635 TXD->lower_setup.ip_fields.ipcso = hoff + offsetof(struct ip, ip_sum);
4636
4637 /*
4638 * Start offset for payload checksum calculation.
4639 * End offset for payload checksum calculation.
4640 * Offset of place to put the checksum.
4641 */
4642 TXD->upper_setup.tcp_fields.tucss = hoff + iphlen;
4643 TXD->upper_setup.tcp_fields.tucse = 0;
4644 TXD->upper_setup.tcp_fields.tucso =
4645 hoff + iphlen + offsetof(struct tcphdr, th_sum);
4646
4647 /*
4648 * Payload size per packet w/o any headers.
4649 * Length of all headers up to payload.
4650 */
4651 TXD->tcp_seg_setup.fields.mss = htole16(mss);
4652 TXD->tcp_seg_setup.fields.hdr_len = hlen;
4653 TXD->cmd_and_length = htole32(E1000_TXD_CMD_IFCS |
4654 E1000_TXD_CMD_DEXT | /* Extended descr */
4655 E1000_TXD_CMD_TSE | /* TSE context */
4656 E1000_TXD_CMD_IP | /* Do IP csum */
4657 E1000_TXD_CMD_TCP | /* Do TCP checksum */
4658 (pktlen - hlen)); /* Total len */
4659
4660 /* Save the information for this TSO context */
4661 adapter->csum_flags = CSUM_TSO;
4662 adapter->csum_lhlen = hoff;
4663 adapter->csum_iphlen = iphlen;
4664 adapter->csum_thlen = thoff;
4665 adapter->csum_mss = mss;
4666 adapter->csum_pktlen = pktlen;
4667 adapter->csum_txd_upper = *txd_upper;
4668 adapter->csum_txd_lower = *txd_lower;
4669
4670 if (++curr_txd == adapter->num_tx_desc)
4671 curr_txd = 0;
4672
4673 KKASSERT(adapter->num_tx_desc_avail > 0);
4674 adapter->num_tx_desc_avail--;
4675
4676 adapter->next_avail_tx_desc = curr_txd;
4677 return 1;
4678 }
4679
4680 /*
4681 * Remove all descriptors from the TX ring.
4682 *
4683 * We want to clear all pending descriptors from the TX ring. Zeroing
4684 * happens when the HW reads the regs. We assign the ring itself as
4685 * the data of the next descriptor. We don't care about the data we
4686 * are about to reset the HW.
4687 */
4688 static void
em_flush_tx_ring(struct adapter * adapter)4689 em_flush_tx_ring(struct adapter *adapter)
4690 {
4691 struct e1000_hw *hw = &adapter->hw;
4692 struct e1000_tx_desc *txd;
4693 uint32_t tctl;
4694
4695 tctl = E1000_READ_REG(hw, E1000_TCTL);
4696 E1000_WRITE_REG(hw, E1000_TCTL, tctl | E1000_TCTL_EN);
4697
4698 txd = &adapter->tx_desc_base[adapter->next_avail_tx_desc++];
4699 if (adapter->next_avail_tx_desc == adapter->num_tx_desc)
4700 adapter->next_avail_tx_desc = 0;
4701
4702 /* Just use the ring as a dummy buffer addr */
4703 txd->buffer_addr = adapter->txdma.dma_paddr;
4704 txd->lower.data = htole32(E1000_TXD_CMD_IFCS | 512);
4705 txd->upper.data = 0;
4706
4707 E1000_WRITE_REG(hw, E1000_TDT(0), adapter->next_avail_tx_desc);
4708 usec_delay(250);
4709 }
4710
4711 /*
4712 * Remove all descriptors from the RX ring.
4713 *
4714 * Mark all descriptors in the RX ring as consumed and disable the RX ring.
4715 */
4716 static void
em_flush_rx_ring(struct adapter * adapter)4717 em_flush_rx_ring(struct adapter *adapter)
4718 {
4719 struct e1000_hw *hw = &adapter->hw;
4720 uint32_t rctl, rxdctl;
4721
4722 rctl = E1000_READ_REG(hw, E1000_RCTL);
4723 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
4724 E1000_WRITE_FLUSH(hw);
4725 usec_delay(150);
4726
4727 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0));
4728 /* Zero the lower 14 bits (prefetch and host thresholds) */
4729 rxdctl &= 0xffffc000;
4730 /*
4731 * Update thresholds: prefetch threshold to 31, host threshold to 1
4732 * and make sure the granularity is "descriptors" and not "cache
4733 * lines".
4734 */
4735 rxdctl |= (0x1F | (1 << 8) | E1000_RXDCTL_THRESH_UNIT_DESC);
4736 E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl);
4737
4738 /* Momentarily enable the RX ring for the changes to take effect */
4739 E1000_WRITE_REG(hw, E1000_RCTL, rctl | E1000_RCTL_EN);
4740 E1000_WRITE_FLUSH(hw);
4741 usec_delay(150);
4742 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
4743 }
4744
4745 /*
4746 * Remove all descriptors from the descriptor rings.
4747 *
4748 * In i219, the descriptor rings must be emptied before resetting the HW
4749 * or before changing the device state to D3 during runtime (runtime PM).
4750 *
4751 * Failure to do this will cause the HW to enter a unit hang state which
4752 * can only be released by PCI reset on the device.
4753 */
4754 static void
em_flush_txrx_ring(struct adapter * adapter)4755 em_flush_txrx_ring(struct adapter *adapter)
4756 {
4757 struct e1000_hw *hw = &adapter->hw;
4758 device_t dev = adapter->dev;
4759 uint16_t hang_state;
4760 uint32_t fext_nvm11;
4761
4762 /*
4763 * First, disable MULR fix in FEXTNVM11.
4764 */
4765 fext_nvm11 = E1000_READ_REG(hw, E1000_FEXTNVM11);
4766 fext_nvm11 |= E1000_FEXTNVM11_DISABLE_MULR_FIX;
4767 E1000_WRITE_REG(hw, E1000_FEXTNVM11, fext_nvm11);
4768
4769 /*
4770 * Do nothing if we're not in faulty state, or if the queue is
4771 * empty.
4772 */
4773 hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2);
4774 if ((hang_state & FLUSH_DESC_REQUIRED) &&
4775 E1000_READ_REG(hw, E1000_TDLEN(0)))
4776 em_flush_tx_ring(adapter);
4777
4778 /*
4779 * Recheck, maybe the fault is caused by the RX ring.
4780 */
4781 hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2);
4782 if (hang_state & FLUSH_DESC_REQUIRED)
4783 em_flush_rx_ring(adapter);
4784 }
4785