1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (C) 2013 Emulex
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright notice,
11 * this list of conditions and the following disclaimer.
12 *
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * 3. Neither the name of the Emulex Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived from
19 * this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE.
32 *
33 * Contact Information:
34 * freebsd-drivers@emulex.com
35 *
36 * Emulex
37 * 3333 Susan Street
38 * Costa Mesa, CA 92626
39 */
40
41
42 #include "opt_inet6.h"
43 #include "opt_inet.h"
44
45 #include "oce_if.h"
46 #include "oce_user.h"
47
48 #define is_tso_pkt(m) (m->m_pkthdr.csum_flags & CSUM_TSO)
49
50 /* UE Status Low CSR */
51 static char *ue_status_low_desc[] = {
52 "CEV",
53 "CTX",
54 "DBUF",
55 "ERX",
56 "Host",
57 "MPU",
58 "NDMA",
59 "PTC ",
60 "RDMA ",
61 "RXF ",
62 "RXIPS ",
63 "RXULP0 ",
64 "RXULP1 ",
65 "RXULP2 ",
66 "TIM ",
67 "TPOST ",
68 "TPRE ",
69 "TXIPS ",
70 "TXULP0 ",
71 "TXULP1 ",
72 "UC ",
73 "WDMA ",
74 "TXULP2 ",
75 "HOST1 ",
76 "P0_OB_LINK ",
77 "P1_OB_LINK ",
78 "HOST_GPIO ",
79 "MBOX ",
80 "AXGMAC0",
81 "AXGMAC1",
82 "JTAG",
83 "MPU_INTPEND"
84 };
85
86 /* UE Status High CSR */
87 static char *ue_status_hi_desc[] = {
88 "LPCMEMHOST",
89 "MGMT_MAC",
90 "PCS0ONLINE",
91 "MPU_IRAM",
92 "PCS1ONLINE",
93 "PCTL0",
94 "PCTL1",
95 "PMEM",
96 "RR",
97 "TXPB",
98 "RXPP",
99 "XAUI",
100 "TXP",
101 "ARM",
102 "IPC",
103 "HOST2",
104 "HOST3",
105 "HOST4",
106 "HOST5",
107 "HOST6",
108 "HOST7",
109 "HOST8",
110 "HOST9",
111 "NETC",
112 "Unknown",
113 "Unknown",
114 "Unknown",
115 "Unknown",
116 "Unknown",
117 "Unknown",
118 "Unknown",
119 "Unknown"
120 };
121
122 struct oce_common_cqe_info{
123 uint8_t vtp:1;
124 uint8_t l4_cksum_pass:1;
125 uint8_t ip_cksum_pass:1;
126 uint8_t ipv6_frame:1;
127 uint8_t qnq:1;
128 uint8_t rsvd:3;
129 uint8_t num_frags;
130 uint16_t pkt_size;
131 uint16_t vtag;
132 };
133
134 /* Driver entry points prototypes */
135 static int oce_probe(device_t dev);
136 static int oce_attach(device_t dev);
137 static int oce_detach(device_t dev);
138 static int oce_shutdown(device_t dev);
139 static int oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data);
140 static void oce_init(void *xsc);
141 static int oce_multiq_start(struct ifnet *ifp, struct mbuf *m);
142 static void oce_multiq_flush(struct ifnet *ifp);
143
144 /* Driver interrupt routines protypes */
145 static void oce_intr(void *arg, int pending);
146 static int oce_setup_intr(POCE_SOFTC sc);
147 static int oce_fast_isr(void *arg);
148 static int oce_alloc_intr(POCE_SOFTC sc, int vector,
149 void (*isr) (void *arg, int pending));
150
151 /* Media callbacks prototypes */
152 static void oce_media_status(struct ifnet *ifp, struct ifmediareq *req);
153 static int oce_media_change(struct ifnet *ifp);
154
155 /* Transmit routines prototypes */
156 static int oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index);
157 static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq);
158 static void oce_process_tx_completion(struct oce_wq *wq);
159 static int oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m,
160 struct oce_wq *wq);
161
162 /* Receive routines prototypes */
163 static int oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe);
164 static int oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe);
165 static void oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe);
166 static void oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq);
167 static uint16_t oce_rq_handler_lro(void *arg);
168 static void oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2);
169 static void oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2);
170 static void oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m);
171
172 /* Helper function prototypes in this file */
173 static void oce_attach_ifp(POCE_SOFTC sc);
174 static void oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag);
175 static void oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag);
176 static int oce_vid_config(POCE_SOFTC sc);
177 static void oce_mac_addr_set(POCE_SOFTC sc);
178 static int oce_handle_passthrough(struct ifnet *ifp, caddr_t data);
179 static void oce_local_timer(void *arg);
180 static void oce_if_deactivate(POCE_SOFTC sc);
181 static void oce_if_activate(POCE_SOFTC sc);
182 static void setup_max_queues_want(POCE_SOFTC sc);
183 static void update_queues_got(POCE_SOFTC sc);
184 static void process_link_state(POCE_SOFTC sc,
185 struct oce_async_cqe_link_state *acqe);
186 static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m);
187 static void oce_get_config(POCE_SOFTC sc);
188 static struct mbuf *oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete);
189 static void oce_read_env_variables(POCE_SOFTC sc);
190
191 /* IP specific */
192 #if defined(INET6) || defined(INET)
193 static int oce_init_lro(POCE_SOFTC sc);
194 static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp);
195 #endif
196
197 static device_method_t oce_dispatch[] = {
198 DEVMETHOD(device_probe, oce_probe),
199 DEVMETHOD(device_attach, oce_attach),
200 DEVMETHOD(device_detach, oce_detach),
201 DEVMETHOD(device_shutdown, oce_shutdown),
202
203 DEVMETHOD_END
204 };
205
206 static driver_t oce_driver = {
207 "oce",
208 oce_dispatch,
209 sizeof(OCE_SOFTC)
210 };
211 static devclass_t oce_devclass;
212
213 /* global vars */
214 const char component_revision[32] = {"///" COMPONENT_REVISION "///"};
215
216 /* Module capabilites and parameters */
217 uint32_t oce_max_rsp_handled = OCE_MAX_RSP_HANDLED;
218 uint32_t oce_enable_rss = OCE_MODCAP_RSS;
219 uint32_t oce_rq_buf_size = 2048;
220
221 TUNABLE_INT("hw.oce.max_rsp_handled", &oce_max_rsp_handled);
222 TUNABLE_INT("hw.oce.enable_rss", &oce_enable_rss);
223
224 /* Supported devices table */
225 static uint32_t supportedDevices[] = {
226 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE2,
227 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE3,
228 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_BE3,
229 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201,
230 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201_VF,
231 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_SH
232 };
233
234 DRIVER_MODULE(oce, pci, oce_driver, oce_devclass, 0, 0);
235 MODULE_PNP_INFO("W32:vendor/device", pci, oce, supportedDevices,
236 nitems(supportedDevices));
237 MODULE_DEPEND(oce, pci, 1, 1, 1);
238 MODULE_DEPEND(oce, ether, 1, 1, 1);
239 MODULE_VERSION(oce, 1);
240
241 POCE_SOFTC softc_head = NULL;
242 POCE_SOFTC softc_tail = NULL;
243
244 struct oce_rdma_if *oce_rdma_if = NULL;
245
246 /*****************************************************************************
247 * Driver entry points functions *
248 *****************************************************************************/
249
250 static int
oce_probe(device_t dev)251 oce_probe(device_t dev)
252 {
253 uint16_t vendor = 0;
254 uint16_t device = 0;
255 int i = 0;
256 char str[256] = {0};
257 POCE_SOFTC sc;
258
259 sc = device_get_softc(dev);
260 bzero(sc, sizeof(OCE_SOFTC));
261 sc->dev = dev;
262
263 vendor = pci_get_vendor(dev);
264 device = pci_get_device(dev);
265
266 for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint32_t)); i++) {
267 if (vendor == ((supportedDevices[i] >> 16) & 0xffff)) {
268 if (device == (supportedDevices[i] & 0xffff)) {
269 sprintf(str, "%s:%s", "Emulex CNA NIC function",
270 component_revision);
271 device_set_desc_copy(dev, str);
272
273 switch (device) {
274 case PCI_PRODUCT_BE2:
275 sc->flags |= OCE_FLAGS_BE2;
276 break;
277 case PCI_PRODUCT_BE3:
278 sc->flags |= OCE_FLAGS_BE3;
279 break;
280 case PCI_PRODUCT_XE201:
281 case PCI_PRODUCT_XE201_VF:
282 sc->flags |= OCE_FLAGS_XE201;
283 break;
284 case PCI_PRODUCT_SH:
285 sc->flags |= OCE_FLAGS_SH;
286 break;
287 default:
288 return ENXIO;
289 }
290 return BUS_PROBE_DEFAULT;
291 }
292 }
293 }
294
295 return ENXIO;
296 }
297
298 static int
oce_attach(device_t dev)299 oce_attach(device_t dev)
300 {
301 POCE_SOFTC sc;
302 int rc = 0;
303
304 sc = device_get_softc(dev);
305
306 rc = oce_hw_pci_alloc(sc);
307 if (rc)
308 return rc;
309
310 sc->tx_ring_size = OCE_TX_RING_SIZE;
311 sc->rx_ring_size = OCE_RX_RING_SIZE;
312 /* receive fragment size should be multiple of 2K */
313 sc->rq_frag_size = ((oce_rq_buf_size / 2048) * 2048);
314 sc->flow_control = OCE_DEFAULT_FLOW_CONTROL;
315 sc->promisc = OCE_DEFAULT_PROMISCUOUS;
316
317 LOCK_CREATE(&sc->bmbx_lock, "Mailbox_lock");
318 LOCK_CREATE(&sc->dev_lock, "Device_lock");
319
320 /* initialise the hardware */
321 rc = oce_hw_init(sc);
322 if (rc)
323 goto pci_res_free;
324
325 oce_read_env_variables(sc);
326
327 oce_get_config(sc);
328
329 setup_max_queues_want(sc);
330
331 rc = oce_setup_intr(sc);
332 if (rc)
333 goto mbox_free;
334
335 rc = oce_queue_init_all(sc);
336 if (rc)
337 goto intr_free;
338
339 oce_attach_ifp(sc);
340
341 #if defined(INET6) || defined(INET)
342 rc = oce_init_lro(sc);
343 if (rc)
344 goto ifp_free;
345 #endif
346
347 rc = oce_hw_start(sc);
348 if (rc)
349 goto lro_free;
350
351 sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
352 oce_add_vlan, sc, EVENTHANDLER_PRI_FIRST);
353 sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
354 oce_del_vlan, sc, EVENTHANDLER_PRI_FIRST);
355
356 rc = oce_stats_init(sc);
357 if (rc)
358 goto vlan_free;
359
360 oce_add_sysctls(sc);
361
362 callout_init(&sc->timer, CALLOUT_MPSAFE);
363 rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc);
364 if (rc)
365 goto stats_free;
366
367 sc->next =NULL;
368 if (softc_tail != NULL) {
369 softc_tail->next = sc;
370 } else {
371 softc_head = sc;
372 }
373 softc_tail = sc;
374
375 gone_in_dev(dev, 15, "relatively uncommon 10GbE NIC");
376
377 return 0;
378
379 stats_free:
380 callout_drain(&sc->timer);
381 oce_stats_free(sc);
382 vlan_free:
383 if (sc->vlan_attach)
384 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach);
385 if (sc->vlan_detach)
386 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach);
387 oce_hw_intr_disable(sc);
388 lro_free:
389 #if defined(INET6) || defined(INET)
390 oce_free_lro(sc);
391 ifp_free:
392 #endif
393 ether_ifdetach(sc->ifp);
394 if_free(sc->ifp);
395 oce_queue_release_all(sc);
396 intr_free:
397 oce_intr_free(sc);
398 mbox_free:
399 oce_dma_free(sc, &sc->bsmbx);
400 pci_res_free:
401 oce_hw_pci_free(sc);
402 LOCK_DESTROY(&sc->dev_lock);
403 LOCK_DESTROY(&sc->bmbx_lock);
404 return rc;
405
406 }
407
408 static int
oce_detach(device_t dev)409 oce_detach(device_t dev)
410 {
411 POCE_SOFTC sc = device_get_softc(dev);
412 POCE_SOFTC poce_sc_tmp, *ppoce_sc_tmp1, poce_sc_tmp2 = NULL;
413
414 poce_sc_tmp = softc_head;
415 ppoce_sc_tmp1 = &softc_head;
416 while (poce_sc_tmp != NULL) {
417 if (poce_sc_tmp == sc) {
418 *ppoce_sc_tmp1 = sc->next;
419 if (sc->next == NULL) {
420 softc_tail = poce_sc_tmp2;
421 }
422 break;
423 }
424 poce_sc_tmp2 = poce_sc_tmp;
425 ppoce_sc_tmp1 = &poce_sc_tmp->next;
426 poce_sc_tmp = poce_sc_tmp->next;
427 }
428
429 LOCK(&sc->dev_lock);
430 oce_if_deactivate(sc);
431 UNLOCK(&sc->dev_lock);
432
433 callout_drain(&sc->timer);
434
435 if (sc->vlan_attach != NULL)
436 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach);
437 if (sc->vlan_detach != NULL)
438 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach);
439
440 ether_ifdetach(sc->ifp);
441
442 if_free(sc->ifp);
443
444 oce_hw_shutdown(sc);
445
446 bus_generic_detach(dev);
447
448 return 0;
449 }
450
451 static int
oce_shutdown(device_t dev)452 oce_shutdown(device_t dev)
453 {
454 int rc;
455
456 rc = oce_detach(dev);
457
458 return rc;
459 }
460
461 static int
oce_ioctl(struct ifnet * ifp,u_long command,caddr_t data)462 oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
463 {
464 struct ifreq *ifr = (struct ifreq *)data;
465 POCE_SOFTC sc = ifp->if_softc;
466 struct ifi2creq i2c;
467 uint8_t offset = 0;
468 int rc = 0;
469 uint32_t u;
470
471 switch (command) {
472 case SIOCGIFMEDIA:
473 rc = ifmedia_ioctl(ifp, ifr, &sc->media, command);
474 break;
475
476 case SIOCSIFMTU:
477 if (ifr->ifr_mtu > OCE_MAX_MTU)
478 rc = EINVAL;
479 else
480 ifp->if_mtu = ifr->ifr_mtu;
481 break;
482
483 case SIOCSIFFLAGS:
484 if (ifp->if_flags & IFF_UP) {
485 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
486 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING;
487 oce_init(sc);
488 }
489 device_printf(sc->dev, "Interface Up\n");
490 } else {
491 LOCK(&sc->dev_lock);
492
493 sc->ifp->if_drv_flags &=
494 ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
495 oce_if_deactivate(sc);
496
497 UNLOCK(&sc->dev_lock);
498
499 device_printf(sc->dev, "Interface Down\n");
500 }
501
502 if ((ifp->if_flags & IFF_PROMISC) && !sc->promisc) {
503 if (!oce_rxf_set_promiscuous(sc, (1 | (1 << 1))))
504 sc->promisc = TRUE;
505 } else if (!(ifp->if_flags & IFF_PROMISC) && sc->promisc) {
506 if (!oce_rxf_set_promiscuous(sc, 0))
507 sc->promisc = FALSE;
508 }
509
510 break;
511
512 case SIOCADDMULTI:
513 case SIOCDELMULTI:
514 rc = oce_hw_update_multicast(sc);
515 if (rc)
516 device_printf(sc->dev,
517 "Update multicast address failed\n");
518 break;
519
520 case SIOCSIFCAP:
521 u = ifr->ifr_reqcap ^ ifp->if_capenable;
522
523 if (u & IFCAP_TXCSUM) {
524 ifp->if_capenable ^= IFCAP_TXCSUM;
525 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
526
527 if (IFCAP_TSO & ifp->if_capenable &&
528 !(IFCAP_TXCSUM & ifp->if_capenable)) {
529 u &= ~IFCAP_TSO;
530 ifp->if_capenable &= ~IFCAP_TSO;
531 ifp->if_hwassist &= ~CSUM_TSO;
532 if_printf(ifp,
533 "TSO disabled due to -txcsum.\n");
534 }
535 }
536
537 if (u & IFCAP_RXCSUM)
538 ifp->if_capenable ^= IFCAP_RXCSUM;
539
540 if (u & IFCAP_TSO4) {
541 ifp->if_capenable ^= IFCAP_TSO4;
542
543 if (IFCAP_TSO & ifp->if_capenable) {
544 if (IFCAP_TXCSUM & ifp->if_capenable)
545 ifp->if_hwassist |= CSUM_TSO;
546 else {
547 ifp->if_capenable &= ~IFCAP_TSO;
548 ifp->if_hwassist &= ~CSUM_TSO;
549 if_printf(ifp,
550 "Enable txcsum first.\n");
551 rc = EAGAIN;
552 }
553 } else
554 ifp->if_hwassist &= ~CSUM_TSO;
555 }
556
557 if (u & IFCAP_VLAN_HWTAGGING)
558 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
559
560 if (u & IFCAP_VLAN_HWFILTER) {
561 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
562 oce_vid_config(sc);
563 }
564 #if defined(INET6) || defined(INET)
565 if (u & IFCAP_LRO) {
566 ifp->if_capenable ^= IFCAP_LRO;
567 if(sc->enable_hwlro) {
568 if(ifp->if_capenable & IFCAP_LRO) {
569 rc = oce_mbox_nic_set_iface_lro_config(sc, 1);
570 }else {
571 rc = oce_mbox_nic_set_iface_lro_config(sc, 0);
572 }
573 }
574 }
575 #endif
576
577 break;
578
579 case SIOCGI2C:
580 rc = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
581 if (rc)
582 break;
583
584 if (i2c.dev_addr == PAGE_NUM_A0) {
585 offset = i2c.offset;
586 } else if (i2c.dev_addr == PAGE_NUM_A2) {
587 offset = TRANSCEIVER_A0_SIZE + i2c.offset;
588 } else {
589 rc = EINVAL;
590 break;
591 }
592
593 if (i2c.len > sizeof(i2c.data) ||
594 i2c.len + offset > sizeof(sfp_vpd_dump_buffer)) {
595 rc = EINVAL;
596 break;
597 }
598
599 rc = oce_mbox_read_transrecv_data(sc, i2c.dev_addr);
600 if (rc) {
601 rc = -rc;
602 break;
603 }
604
605 memcpy(&i2c.data[0], &sfp_vpd_dump_buffer[offset], i2c.len);
606
607 rc = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c));
608 break;
609
610 case SIOCGPRIVATE_0:
611 rc = priv_check(curthread, PRIV_DRIVER);
612 if (rc != 0)
613 break;
614 rc = oce_handle_passthrough(ifp, data);
615 break;
616 default:
617 rc = ether_ioctl(ifp, command, data);
618 break;
619 }
620
621 return rc;
622 }
623
624 static void
oce_init(void * arg)625 oce_init(void *arg)
626 {
627 POCE_SOFTC sc = arg;
628
629 LOCK(&sc->dev_lock);
630
631 if (sc->ifp->if_flags & IFF_UP) {
632 oce_if_deactivate(sc);
633 oce_if_activate(sc);
634 }
635
636 UNLOCK(&sc->dev_lock);
637
638 }
639
640 static int
oce_multiq_start(struct ifnet * ifp,struct mbuf * m)641 oce_multiq_start(struct ifnet *ifp, struct mbuf *m)
642 {
643 POCE_SOFTC sc = ifp->if_softc;
644 struct oce_wq *wq = NULL;
645 int queue_index = 0;
646 int status = 0;
647
648 if (!sc->link_status)
649 return ENXIO;
650
651 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
652 queue_index = m->m_pkthdr.flowid % sc->nwqs;
653
654 wq = sc->wq[queue_index];
655
656 LOCK(&wq->tx_lock);
657 status = oce_multiq_transmit(ifp, m, wq);
658 UNLOCK(&wq->tx_lock);
659
660 return status;
661
662 }
663
664 static void
oce_multiq_flush(struct ifnet * ifp)665 oce_multiq_flush(struct ifnet *ifp)
666 {
667 POCE_SOFTC sc = ifp->if_softc;
668 struct mbuf *m;
669 int i = 0;
670
671 for (i = 0; i < sc->nwqs; i++) {
672 while ((m = buf_ring_dequeue_sc(sc->wq[i]->br)) != NULL)
673 m_freem(m);
674 }
675 if_qflush(ifp);
676 }
677
678 /*****************************************************************************
679 * Driver interrupt routines functions *
680 *****************************************************************************/
681
682 static void
oce_intr(void * arg,int pending)683 oce_intr(void *arg, int pending)
684 {
685
686 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg;
687 POCE_SOFTC sc = ii->sc;
688 struct oce_eq *eq = ii->eq;
689 struct oce_eqe *eqe;
690 struct oce_cq *cq = NULL;
691 int i, num_eqes = 0;
692
693 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map,
694 BUS_DMASYNC_POSTWRITE);
695 do {
696 eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe);
697 if (eqe->evnt == 0)
698 break;
699 eqe->evnt = 0;
700 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map,
701 BUS_DMASYNC_POSTWRITE);
702 RING_GET(eq->ring, 1);
703 num_eqes++;
704
705 } while (TRUE);
706
707 if (!num_eqes)
708 goto eq_arm; /* Spurious */
709
710 /* Clear EQ entries, but dont arm */
711 oce_arm_eq(sc, eq->eq_id, num_eqes, FALSE, FALSE);
712
713 /* Process TX, RX and MCC. But dont arm CQ*/
714 for (i = 0; i < eq->cq_valid; i++) {
715 cq = eq->cq[i];
716 (*cq->cq_handler)(cq->cb_arg);
717 }
718
719 /* Arm all cqs connected to this EQ */
720 for (i = 0; i < eq->cq_valid; i++) {
721 cq = eq->cq[i];
722 oce_arm_cq(sc, cq->cq_id, 0, TRUE);
723 }
724
725 eq_arm:
726 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE);
727
728 return;
729 }
730
731 static int
oce_setup_intr(POCE_SOFTC sc)732 oce_setup_intr(POCE_SOFTC sc)
733 {
734 int rc = 0, use_intx = 0;
735 int vector = 0, req_vectors = 0;
736 int tot_req_vectors, tot_vectors;
737
738 if (is_rss_enabled(sc))
739 req_vectors = MAX((sc->nrqs - 1), sc->nwqs);
740 else
741 req_vectors = 1;
742
743 tot_req_vectors = req_vectors;
744 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
745 if (req_vectors > 1) {
746 tot_req_vectors += OCE_RDMA_VECTORS;
747 sc->roce_intr_count = OCE_RDMA_VECTORS;
748 }
749 }
750
751 if (sc->flags & OCE_FLAGS_MSIX_CAPABLE) {
752 sc->intr_count = req_vectors;
753 tot_vectors = tot_req_vectors;
754 rc = pci_alloc_msix(sc->dev, &tot_vectors);
755 if (rc != 0) {
756 use_intx = 1;
757 pci_release_msi(sc->dev);
758 } else {
759 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
760 if (tot_vectors < tot_req_vectors) {
761 if (sc->intr_count < (2 * OCE_RDMA_VECTORS)) {
762 sc->roce_intr_count = (tot_vectors / 2);
763 }
764 sc->intr_count = tot_vectors - sc->roce_intr_count;
765 }
766 } else {
767 sc->intr_count = tot_vectors;
768 }
769 sc->flags |= OCE_FLAGS_USING_MSIX;
770 }
771 } else
772 use_intx = 1;
773
774 if (use_intx)
775 sc->intr_count = 1;
776
777 /* Scale number of queues based on intr we got */
778 update_queues_got(sc);
779
780 if (use_intx) {
781 device_printf(sc->dev, "Using legacy interrupt\n");
782 rc = oce_alloc_intr(sc, vector, oce_intr);
783 if (rc)
784 goto error;
785 } else {
786 for (; vector < sc->intr_count; vector++) {
787 rc = oce_alloc_intr(sc, vector, oce_intr);
788 if (rc)
789 goto error;
790 }
791 }
792
793 return 0;
794 error:
795 oce_intr_free(sc);
796 return rc;
797 }
798
799 static int
oce_fast_isr(void * arg)800 oce_fast_isr(void *arg)
801 {
802 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg;
803 POCE_SOFTC sc = ii->sc;
804
805 if (ii->eq == NULL)
806 return FILTER_STRAY;
807
808 oce_arm_eq(sc, ii->eq->eq_id, 0, FALSE, TRUE);
809
810 taskqueue_enqueue(ii->tq, &ii->task);
811
812 ii->eq->intr++;
813
814 return FILTER_HANDLED;
815 }
816
817 static int
oce_alloc_intr(POCE_SOFTC sc,int vector,void (* isr)(void * arg,int pending))818 oce_alloc_intr(POCE_SOFTC sc, int vector, void (*isr) (void *arg, int pending))
819 {
820 POCE_INTR_INFO ii;
821 int rc = 0, rr;
822
823 if (vector >= OCE_MAX_EQ)
824 return (EINVAL);
825
826 ii = &sc->intrs[vector];
827
828 /* Set the resource id for the interrupt.
829 * MSIx is vector + 1 for the resource id,
830 * INTx is 0 for the resource id.
831 */
832 if (sc->flags & OCE_FLAGS_USING_MSIX)
833 rr = vector + 1;
834 else
835 rr = 0;
836 ii->intr_res = bus_alloc_resource_any(sc->dev,
837 SYS_RES_IRQ,
838 &rr, RF_ACTIVE|RF_SHAREABLE);
839 ii->irq_rr = rr;
840 if (ii->intr_res == NULL) {
841 device_printf(sc->dev,
842 "Could not allocate interrupt\n");
843 rc = ENXIO;
844 return rc;
845 }
846
847 TASK_INIT(&ii->task, 0, isr, ii);
848 ii->vector = vector;
849 sprintf(ii->task_name, "oce_task[%d]", ii->vector);
850 ii->tq = taskqueue_create_fast(ii->task_name,
851 M_NOWAIT,
852 taskqueue_thread_enqueue,
853 &ii->tq);
854 taskqueue_start_threads(&ii->tq, 1, PI_NET, "%s taskq",
855 device_get_nameunit(sc->dev));
856
857 ii->sc = sc;
858 rc = bus_setup_intr(sc->dev,
859 ii->intr_res,
860 INTR_TYPE_NET,
861 oce_fast_isr, NULL, ii, &ii->tag);
862 return rc;
863
864 }
865
866 void
oce_intr_free(POCE_SOFTC sc)867 oce_intr_free(POCE_SOFTC sc)
868 {
869 int i = 0;
870
871 for (i = 0; i < sc->intr_count; i++) {
872
873 if (sc->intrs[i].tag != NULL)
874 bus_teardown_intr(sc->dev, sc->intrs[i].intr_res,
875 sc->intrs[i].tag);
876 if (sc->intrs[i].tq != NULL)
877 taskqueue_free(sc->intrs[i].tq);
878
879 if (sc->intrs[i].intr_res != NULL)
880 bus_release_resource(sc->dev, SYS_RES_IRQ,
881 sc->intrs[i].irq_rr,
882 sc->intrs[i].intr_res);
883 sc->intrs[i].tag = NULL;
884 sc->intrs[i].intr_res = NULL;
885 }
886
887 if (sc->flags & OCE_FLAGS_USING_MSIX)
888 pci_release_msi(sc->dev);
889
890 }
891
892 /******************************************************************************
893 * Media callbacks functions *
894 ******************************************************************************/
895
896 static void
oce_media_status(struct ifnet * ifp,struct ifmediareq * req)897 oce_media_status(struct ifnet *ifp, struct ifmediareq *req)
898 {
899 POCE_SOFTC sc = (POCE_SOFTC) ifp->if_softc;
900
901 req->ifm_status = IFM_AVALID;
902 req->ifm_active = IFM_ETHER;
903
904 if (sc->link_status == 1)
905 req->ifm_status |= IFM_ACTIVE;
906 else
907 return;
908
909 switch (sc->link_speed) {
910 case 1: /* 10 Mbps */
911 req->ifm_active |= IFM_10_T | IFM_FDX;
912 sc->speed = 10;
913 break;
914 case 2: /* 100 Mbps */
915 req->ifm_active |= IFM_100_TX | IFM_FDX;
916 sc->speed = 100;
917 break;
918 case 3: /* 1 Gbps */
919 req->ifm_active |= IFM_1000_T | IFM_FDX;
920 sc->speed = 1000;
921 break;
922 case 4: /* 10 Gbps */
923 req->ifm_active |= IFM_10G_SR | IFM_FDX;
924 sc->speed = 10000;
925 break;
926 case 5: /* 20 Gbps */
927 req->ifm_active |= IFM_10G_SR | IFM_FDX;
928 sc->speed = 20000;
929 break;
930 case 6: /* 25 Gbps */
931 req->ifm_active |= IFM_10G_SR | IFM_FDX;
932 sc->speed = 25000;
933 break;
934 case 7: /* 40 Gbps */
935 req->ifm_active |= IFM_40G_SR4 | IFM_FDX;
936 sc->speed = 40000;
937 break;
938 default:
939 sc->speed = 0;
940 break;
941 }
942
943 return;
944 }
945
946 int
oce_media_change(struct ifnet * ifp)947 oce_media_change(struct ifnet *ifp)
948 {
949 return 0;
950 }
951
oce_is_pkt_dest_bmc(POCE_SOFTC sc,struct mbuf * m,boolean_t * os2bmc,struct mbuf ** m_new)952 static void oce_is_pkt_dest_bmc(POCE_SOFTC sc,
953 struct mbuf *m, boolean_t *os2bmc,
954 struct mbuf **m_new)
955 {
956 struct ether_header *eh = NULL;
957
958 eh = mtod(m, struct ether_header *);
959
960 if (!is_os2bmc_enabled(sc) || *os2bmc) {
961 *os2bmc = FALSE;
962 goto done;
963 }
964 if (!ETHER_IS_MULTICAST(eh->ether_dhost))
965 goto done;
966
967 if (is_mc_allowed_on_bmc(sc, eh) ||
968 is_bc_allowed_on_bmc(sc, eh) ||
969 is_arp_allowed_on_bmc(sc, ntohs(eh->ether_type))) {
970 *os2bmc = TRUE;
971 goto done;
972 }
973
974 if (mtod(m, struct ip *)->ip_p == IPPROTO_IPV6) {
975 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
976 uint8_t nexthdr = ip6->ip6_nxt;
977 if (nexthdr == IPPROTO_ICMPV6) {
978 struct icmp6_hdr *icmp6 = (struct icmp6_hdr *)(ip6 + 1);
979 switch (icmp6->icmp6_type) {
980 case ND_ROUTER_ADVERT:
981 *os2bmc = is_ipv6_ra_filt_enabled(sc);
982 goto done;
983 case ND_NEIGHBOR_ADVERT:
984 *os2bmc = is_ipv6_na_filt_enabled(sc);
985 goto done;
986 default:
987 break;
988 }
989 }
990 }
991
992 if (mtod(m, struct ip *)->ip_p == IPPROTO_UDP) {
993 struct ip *ip = mtod(m, struct ip *);
994 int iphlen = ip->ip_hl << 2;
995 struct udphdr *uh = (struct udphdr *)((caddr_t)ip + iphlen);
996 switch (uh->uh_dport) {
997 case DHCP_CLIENT_PORT:
998 *os2bmc = is_dhcp_client_filt_enabled(sc);
999 goto done;
1000 case DHCP_SERVER_PORT:
1001 *os2bmc = is_dhcp_srvr_filt_enabled(sc);
1002 goto done;
1003 case NET_BIOS_PORT1:
1004 case NET_BIOS_PORT2:
1005 *os2bmc = is_nbios_filt_enabled(sc);
1006 goto done;
1007 case DHCPV6_RAS_PORT:
1008 *os2bmc = is_ipv6_ras_filt_enabled(sc);
1009 goto done;
1010 default:
1011 break;
1012 }
1013 }
1014 done:
1015 if (*os2bmc) {
1016 *m_new = m_dup(m, M_NOWAIT);
1017 if (!*m_new) {
1018 *os2bmc = FALSE;
1019 return;
1020 }
1021 *m_new = oce_insert_vlan_tag(sc, *m_new, NULL);
1022 }
1023 }
1024
1025 /*****************************************************************************
1026 * Transmit routines functions *
1027 *****************************************************************************/
1028
1029 static int
oce_tx(POCE_SOFTC sc,struct mbuf ** mpp,int wq_index)1030 oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index)
1031 {
1032 int rc = 0, i, retry_cnt = 0;
1033 bus_dma_segment_t segs[OCE_MAX_TX_ELEMENTS];
1034 struct mbuf *m, *m_temp, *m_new = NULL;
1035 struct oce_wq *wq = sc->wq[wq_index];
1036 struct oce_packet_desc *pd;
1037 struct oce_nic_hdr_wqe *nichdr;
1038 struct oce_nic_frag_wqe *nicfrag;
1039 struct ether_header *eh = NULL;
1040 int num_wqes;
1041 uint32_t reg_value;
1042 boolean_t complete = TRUE;
1043 boolean_t os2bmc = FALSE;
1044
1045 m = *mpp;
1046 if (!m)
1047 return EINVAL;
1048
1049 if (!(m->m_flags & M_PKTHDR)) {
1050 rc = ENXIO;
1051 goto free_ret;
1052 }
1053
1054 /* Don't allow non-TSO packets longer than MTU */
1055 if (!is_tso_pkt(m)) {
1056 eh = mtod(m, struct ether_header *);
1057 if(m->m_pkthdr.len > ETHER_MAX_FRAME(sc->ifp, eh->ether_type, FALSE))
1058 goto free_ret;
1059 }
1060
1061 if(oce_tx_asic_stall_verify(sc, m)) {
1062 m = oce_insert_vlan_tag(sc, m, &complete);
1063 if(!m) {
1064 device_printf(sc->dev, "Insertion unsuccessful\n");
1065 return 0;
1066 }
1067 }
1068
1069 /* Lancer, SH ASIC has a bug wherein Packets that are 32 bytes or less
1070 * may cause a transmit stall on that port. So the work-around is to
1071 * pad short packets (<= 32 bytes) to a 36-byte length.
1072 */
1073 if(IS_SH(sc) || IS_XE201(sc) ) {
1074 if(m->m_pkthdr.len <= 32) {
1075 char buf[36];
1076 bzero((void *)buf, 36);
1077 m_append(m, (36 - m->m_pkthdr.len), buf);
1078 }
1079 }
1080
1081 tx_start:
1082 if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1083 /* consolidate packet buffers for TSO/LSO segment offload */
1084 #if defined(INET6) || defined(INET)
1085 m = oce_tso_setup(sc, mpp);
1086 #else
1087 m = NULL;
1088 #endif
1089 if (m == NULL) {
1090 rc = ENXIO;
1091 goto free_ret;
1092 }
1093 }
1094
1095 pd = &wq->pckts[wq->pkt_desc_head];
1096
1097 retry:
1098 rc = bus_dmamap_load_mbuf_sg(wq->tag,
1099 pd->map,
1100 m, segs, &pd->nsegs, BUS_DMA_NOWAIT);
1101 if (rc == 0) {
1102 num_wqes = pd->nsegs + 1;
1103 if (IS_BE(sc) || IS_SH(sc)) {
1104 /*Dummy required only for BE3.*/
1105 if (num_wqes & 1)
1106 num_wqes++;
1107 }
1108 if (num_wqes >= RING_NUM_FREE(wq->ring)) {
1109 bus_dmamap_unload(wq->tag, pd->map);
1110 return EBUSY;
1111 }
1112 atomic_store_rel_int(&wq->pkt_desc_head,
1113 (wq->pkt_desc_head + 1) % \
1114 OCE_WQ_PACKET_ARRAY_SIZE);
1115 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_PREWRITE);
1116 pd->mbuf = m;
1117
1118 nichdr =
1119 RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_hdr_wqe);
1120 nichdr->u0.dw[0] = 0;
1121 nichdr->u0.dw[1] = 0;
1122 nichdr->u0.dw[2] = 0;
1123 nichdr->u0.dw[3] = 0;
1124
1125 nichdr->u0.s.complete = complete;
1126 nichdr->u0.s.mgmt = os2bmc;
1127 nichdr->u0.s.event = 1;
1128 nichdr->u0.s.crc = 1;
1129 nichdr->u0.s.forward = 0;
1130 nichdr->u0.s.ipcs = (m->m_pkthdr.csum_flags & CSUM_IP) ? 1 : 0;
1131 nichdr->u0.s.udpcs =
1132 (m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0;
1133 nichdr->u0.s.tcpcs =
1134 (m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0;
1135 nichdr->u0.s.num_wqe = num_wqes;
1136 nichdr->u0.s.total_length = m->m_pkthdr.len;
1137
1138 if (m->m_flags & M_VLANTAG) {
1139 nichdr->u0.s.vlan = 1; /*Vlan present*/
1140 nichdr->u0.s.vlan_tag = m->m_pkthdr.ether_vtag;
1141 }
1142
1143 if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1144 if (m->m_pkthdr.tso_segsz) {
1145 nichdr->u0.s.lso = 1;
1146 nichdr->u0.s.lso_mss = m->m_pkthdr.tso_segsz;
1147 }
1148 if (!IS_BE(sc) || !IS_SH(sc))
1149 nichdr->u0.s.ipcs = 1;
1150 }
1151
1152 RING_PUT(wq->ring, 1);
1153 atomic_add_int(&wq->ring->num_used, 1);
1154
1155 for (i = 0; i < pd->nsegs; i++) {
1156 nicfrag =
1157 RING_GET_PRODUCER_ITEM_VA(wq->ring,
1158 struct oce_nic_frag_wqe);
1159 nicfrag->u0.s.rsvd0 = 0;
1160 nicfrag->u0.s.frag_pa_hi = ADDR_HI(segs[i].ds_addr);
1161 nicfrag->u0.s.frag_pa_lo = ADDR_LO(segs[i].ds_addr);
1162 nicfrag->u0.s.frag_len = segs[i].ds_len;
1163 pd->wqe_idx = wq->ring->pidx;
1164 RING_PUT(wq->ring, 1);
1165 atomic_add_int(&wq->ring->num_used, 1);
1166 }
1167 if (num_wqes > (pd->nsegs + 1)) {
1168 nicfrag =
1169 RING_GET_PRODUCER_ITEM_VA(wq->ring,
1170 struct oce_nic_frag_wqe);
1171 nicfrag->u0.dw[0] = 0;
1172 nicfrag->u0.dw[1] = 0;
1173 nicfrag->u0.dw[2] = 0;
1174 nicfrag->u0.dw[3] = 0;
1175 pd->wqe_idx = wq->ring->pidx;
1176 RING_PUT(wq->ring, 1);
1177 atomic_add_int(&wq->ring->num_used, 1);
1178 pd->nsegs++;
1179 }
1180
1181 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1);
1182 wq->tx_stats.tx_reqs++;
1183 wq->tx_stats.tx_wrbs += num_wqes;
1184 wq->tx_stats.tx_bytes += m->m_pkthdr.len;
1185 wq->tx_stats.tx_pkts++;
1186
1187 bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map,
1188 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1189 reg_value = (num_wqes << 16) | wq->wq_id;
1190
1191 /* if os2bmc is not enabled or if the pkt is already tagged as
1192 bmc, do nothing
1193 */
1194 oce_is_pkt_dest_bmc(sc, m, &os2bmc, &m_new);
1195
1196 if_inc_counter(sc->ifp, IFCOUNTER_OBYTES, m->m_pkthdr.len);
1197 if (m->m_flags & M_MCAST)
1198 if_inc_counter(sc->ifp, IFCOUNTER_OMCASTS, 1);
1199 ETHER_BPF_MTAP(sc->ifp, m);
1200
1201 OCE_WRITE_REG32(sc, db, wq->db_offset, reg_value);
1202
1203 } else if (rc == EFBIG) {
1204 if (retry_cnt == 0) {
1205 m_temp = m_defrag(m, M_NOWAIT);
1206 if (m_temp == NULL)
1207 goto free_ret;
1208 m = m_temp;
1209 *mpp = m_temp;
1210 retry_cnt = retry_cnt + 1;
1211 goto retry;
1212 } else
1213 goto free_ret;
1214 } else if (rc == ENOMEM)
1215 return rc;
1216 else
1217 goto free_ret;
1218
1219 if (os2bmc) {
1220 m = m_new;
1221 goto tx_start;
1222 }
1223
1224 return 0;
1225
1226 free_ret:
1227 m_freem(*mpp);
1228 *mpp = NULL;
1229 return rc;
1230 }
1231
1232 static void
oce_process_tx_completion(struct oce_wq * wq)1233 oce_process_tx_completion(struct oce_wq *wq)
1234 {
1235 struct oce_packet_desc *pd;
1236 POCE_SOFTC sc = (POCE_SOFTC) wq->parent;
1237 struct mbuf *m;
1238
1239 pd = &wq->pckts[wq->pkt_desc_tail];
1240 atomic_store_rel_int(&wq->pkt_desc_tail,
1241 (wq->pkt_desc_tail + 1) % OCE_WQ_PACKET_ARRAY_SIZE);
1242 atomic_subtract_int(&wq->ring->num_used, pd->nsegs + 1);
1243 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1244 bus_dmamap_unload(wq->tag, pd->map);
1245
1246 m = pd->mbuf;
1247 m_freem(m);
1248 pd->mbuf = NULL;
1249
1250 if (sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) {
1251 if (wq->ring->num_used < (wq->ring->num_items / 2)) {
1252 sc->ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE);
1253 oce_tx_restart(sc, wq);
1254 }
1255 }
1256 }
1257
1258 static void
oce_tx_restart(POCE_SOFTC sc,struct oce_wq * wq)1259 oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq)
1260 {
1261
1262 if ((sc->ifp->if_drv_flags & IFF_DRV_RUNNING) != IFF_DRV_RUNNING)
1263 return;
1264
1265 if (!drbr_empty(sc->ifp, wq->br))
1266 taskqueue_enqueue(taskqueue_swi, &wq->txtask);
1267
1268 }
1269
1270 #if defined(INET6) || defined(INET)
1271 static struct mbuf *
oce_tso_setup(POCE_SOFTC sc,struct mbuf ** mpp)1272 oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp)
1273 {
1274 struct mbuf *m;
1275 #ifdef INET
1276 struct ip *ip;
1277 #endif
1278 #ifdef INET6
1279 struct ip6_hdr *ip6;
1280 #endif
1281 struct ether_vlan_header *eh;
1282 struct tcphdr *th;
1283 uint16_t etype;
1284 int total_len = 0, ehdrlen = 0;
1285
1286 m = *mpp;
1287
1288 if (M_WRITABLE(m) == 0) {
1289 m = m_dup(*mpp, M_NOWAIT);
1290 if (!m)
1291 return NULL;
1292 m_freem(*mpp);
1293 *mpp = m;
1294 }
1295
1296 eh = mtod(m, struct ether_vlan_header *);
1297 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
1298 etype = ntohs(eh->evl_proto);
1299 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
1300 } else {
1301 etype = ntohs(eh->evl_encap_proto);
1302 ehdrlen = ETHER_HDR_LEN;
1303 }
1304
1305 switch (etype) {
1306 #ifdef INET
1307 case ETHERTYPE_IP:
1308 ip = (struct ip *)(m->m_data + ehdrlen);
1309 if (ip->ip_p != IPPROTO_TCP)
1310 return NULL;
1311 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
1312
1313 total_len = ehdrlen + (ip->ip_hl << 2) + (th->th_off << 2);
1314 break;
1315 #endif
1316 #ifdef INET6
1317 case ETHERTYPE_IPV6:
1318 ip6 = (struct ip6_hdr *)(m->m_data + ehdrlen);
1319 if (ip6->ip6_nxt != IPPROTO_TCP)
1320 return NULL;
1321 th = (struct tcphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr));
1322
1323 total_len = ehdrlen + sizeof(struct ip6_hdr) + (th->th_off << 2);
1324 break;
1325 #endif
1326 default:
1327 return NULL;
1328 }
1329
1330 m = m_pullup(m, total_len);
1331 *mpp = m;
1332 return m;
1333 }
1334 #endif /* INET6 || INET */
1335
1336 void
oce_tx_task(void * arg,int npending)1337 oce_tx_task(void *arg, int npending)
1338 {
1339 struct oce_wq *wq = arg;
1340 POCE_SOFTC sc = wq->parent;
1341 struct ifnet *ifp = sc->ifp;
1342 int rc = 0;
1343
1344 LOCK(&wq->tx_lock);
1345 rc = oce_multiq_transmit(ifp, NULL, wq);
1346 if (rc) {
1347 device_printf(sc->dev,
1348 "TX[%d] restart failed\n", wq->queue_index);
1349 }
1350 UNLOCK(&wq->tx_lock);
1351 }
1352
1353 void
oce_start(struct ifnet * ifp)1354 oce_start(struct ifnet *ifp)
1355 {
1356 POCE_SOFTC sc = ifp->if_softc;
1357 struct mbuf *m;
1358 int rc = 0;
1359 int def_q = 0; /* Defualt tx queue is 0*/
1360
1361 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
1362 IFF_DRV_RUNNING)
1363 return;
1364
1365 if (!sc->link_status)
1366 return;
1367
1368 while (true) {
1369 IF_DEQUEUE(&sc->ifp->if_snd, m);
1370 if (m == NULL)
1371 break;
1372
1373 LOCK(&sc->wq[def_q]->tx_lock);
1374 rc = oce_tx(sc, &m, def_q);
1375 UNLOCK(&sc->wq[def_q]->tx_lock);
1376 if (rc) {
1377 if (m != NULL) {
1378 sc->wq[def_q]->tx_stats.tx_stops ++;
1379 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1380 IFQ_DRV_PREPEND(&ifp->if_snd, m);
1381 m = NULL;
1382 }
1383 break;
1384 }
1385 }
1386 }
1387
1388 /* Handle the Completion Queue for transmit */
1389 uint16_t
oce_wq_handler(void * arg)1390 oce_wq_handler(void *arg)
1391 {
1392 struct oce_wq *wq = (struct oce_wq *)arg;
1393 POCE_SOFTC sc = wq->parent;
1394 struct oce_cq *cq = wq->cq;
1395 struct oce_nic_tx_cqe *cqe;
1396 int num_cqes = 0;
1397
1398 LOCK(&wq->tx_compl_lock);
1399 bus_dmamap_sync(cq->ring->dma.tag,
1400 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
1401 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
1402 while (cqe->u0.dw[3]) {
1403 DW_SWAP((uint32_t *) cqe, sizeof(oce_wq_cqe));
1404
1405 wq->ring->cidx = cqe->u0.s.wqe_index + 1;
1406 if (wq->ring->cidx >= wq->ring->num_items)
1407 wq->ring->cidx -= wq->ring->num_items;
1408
1409 oce_process_tx_completion(wq);
1410 wq->tx_stats.tx_compl++;
1411 cqe->u0.dw[3] = 0;
1412 RING_GET(cq->ring, 1);
1413 bus_dmamap_sync(cq->ring->dma.tag,
1414 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
1415 cqe =
1416 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
1417 num_cqes++;
1418 }
1419
1420 if (num_cqes)
1421 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE);
1422
1423 UNLOCK(&wq->tx_compl_lock);
1424 return num_cqes;
1425 }
1426
1427 static int
oce_multiq_transmit(struct ifnet * ifp,struct mbuf * m,struct oce_wq * wq)1428 oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, struct oce_wq *wq)
1429 {
1430 POCE_SOFTC sc = ifp->if_softc;
1431 int status = 0, queue_index = 0;
1432 struct mbuf *next = NULL;
1433 struct buf_ring *br = NULL;
1434
1435 br = wq->br;
1436 queue_index = wq->queue_index;
1437
1438 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
1439 IFF_DRV_RUNNING) {
1440 if (m != NULL)
1441 status = drbr_enqueue(ifp, br, m);
1442 return status;
1443 }
1444
1445 if (m != NULL) {
1446 if ((status = drbr_enqueue(ifp, br, m)) != 0)
1447 return status;
1448 }
1449 while ((next = drbr_peek(ifp, br)) != NULL) {
1450 if (oce_tx(sc, &next, queue_index)) {
1451 if (next == NULL) {
1452 drbr_advance(ifp, br);
1453 } else {
1454 drbr_putback(ifp, br, next);
1455 wq->tx_stats.tx_stops ++;
1456 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1457 }
1458 break;
1459 }
1460 drbr_advance(ifp, br);
1461 }
1462
1463 return 0;
1464 }
1465
1466 /*****************************************************************************
1467 * Receive routines functions *
1468 *****************************************************************************/
1469
1470 static void
oce_correct_header(struct mbuf * m,struct nic_hwlro_cqe_part1 * cqe1,struct nic_hwlro_cqe_part2 * cqe2)1471 oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2)
1472 {
1473 uint32_t *p;
1474 struct ether_header *eh = NULL;
1475 struct tcphdr *tcp_hdr = NULL;
1476 struct ip *ip4_hdr = NULL;
1477 struct ip6_hdr *ip6 = NULL;
1478 uint32_t payload_len = 0;
1479
1480 eh = mtod(m, struct ether_header *);
1481 /* correct IP header */
1482 if(!cqe2->ipv6_frame) {
1483 ip4_hdr = (struct ip *)((char*)eh + sizeof(struct ether_header));
1484 ip4_hdr->ip_ttl = cqe2->frame_lifespan;
1485 ip4_hdr->ip_len = htons(cqe2->coalesced_size - sizeof(struct ether_header));
1486 tcp_hdr = (struct tcphdr *)((char*)ip4_hdr + sizeof(struct ip));
1487 }else {
1488 ip6 = (struct ip6_hdr *)((char*)eh + sizeof(struct ether_header));
1489 ip6->ip6_ctlun.ip6_un1.ip6_un1_hlim = cqe2->frame_lifespan;
1490 payload_len = cqe2->coalesced_size - sizeof(struct ether_header)
1491 - sizeof(struct ip6_hdr);
1492 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = htons(payload_len);
1493 tcp_hdr = (struct tcphdr *)((char*)ip6 + sizeof(struct ip6_hdr));
1494 }
1495
1496 /* correct tcp header */
1497 tcp_hdr->th_ack = htonl(cqe2->tcp_ack_num);
1498 if(cqe2->push) {
1499 tcp_hdr->th_flags |= TH_PUSH;
1500 }
1501 tcp_hdr->th_win = htons(cqe2->tcp_window);
1502 tcp_hdr->th_sum = 0xffff;
1503 if(cqe2->ts_opt) {
1504 p = (uint32_t *)((char*)tcp_hdr + sizeof(struct tcphdr) + 2);
1505 *p = cqe1->tcp_timestamp_val;
1506 *(p+1) = cqe1->tcp_timestamp_ecr;
1507 }
1508
1509 return;
1510 }
1511
1512 static void
oce_rx_mbuf_chain(struct oce_rq * rq,struct oce_common_cqe_info * cqe_info,struct mbuf ** m)1513 oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m)
1514 {
1515 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1516 uint32_t i = 0, frag_len = 0;
1517 uint32_t len = cqe_info->pkt_size;
1518 struct oce_packet_desc *pd;
1519 struct mbuf *tail = NULL;
1520
1521 for (i = 0; i < cqe_info->num_frags; i++) {
1522 if (rq->ring->cidx == rq->ring->pidx) {
1523 device_printf(sc->dev,
1524 "oce_rx_mbuf_chain: Invalid RX completion - Queue is empty\n");
1525 return;
1526 }
1527 pd = &rq->pckts[rq->ring->cidx];
1528
1529 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1530 bus_dmamap_unload(rq->tag, pd->map);
1531 RING_GET(rq->ring, 1);
1532 rq->pending--;
1533
1534 frag_len = (len > rq->cfg.frag_size) ? rq->cfg.frag_size : len;
1535 pd->mbuf->m_len = frag_len;
1536
1537 if (tail != NULL) {
1538 /* additional fragments */
1539 pd->mbuf->m_flags &= ~M_PKTHDR;
1540 tail->m_next = pd->mbuf;
1541 if(rq->islro)
1542 tail->m_nextpkt = NULL;
1543 tail = pd->mbuf;
1544 } else {
1545 /* first fragment, fill out much of the packet header */
1546 pd->mbuf->m_pkthdr.len = len;
1547 if(rq->islro)
1548 pd->mbuf->m_nextpkt = NULL;
1549 pd->mbuf->m_pkthdr.csum_flags = 0;
1550 if (IF_CSUM_ENABLED(sc)) {
1551 if (cqe_info->l4_cksum_pass) {
1552 if(!cqe_info->ipv6_frame) { /* IPV4 */
1553 pd->mbuf->m_pkthdr.csum_flags |=
1554 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1555 }else { /* IPV6 frame */
1556 if(rq->islro) {
1557 pd->mbuf->m_pkthdr.csum_flags |=
1558 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1559 }
1560 }
1561 pd->mbuf->m_pkthdr.csum_data = 0xffff;
1562 }
1563 if (cqe_info->ip_cksum_pass) {
1564 pd->mbuf->m_pkthdr.csum_flags |=
1565 (CSUM_IP_CHECKED|CSUM_IP_VALID);
1566 }
1567 }
1568 *m = tail = pd->mbuf;
1569 }
1570 pd->mbuf = NULL;
1571 len -= frag_len;
1572 }
1573
1574 return;
1575 }
1576
1577 static void
oce_rx_lro(struct oce_rq * rq,struct nic_hwlro_singleton_cqe * cqe,struct nic_hwlro_cqe_part2 * cqe2)1578 oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2)
1579 {
1580 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1581 struct nic_hwlro_cqe_part1 *cqe1 = NULL;
1582 struct mbuf *m = NULL;
1583 struct oce_common_cqe_info cq_info;
1584
1585 /* parse cqe */
1586 if(cqe2 == NULL) {
1587 cq_info.pkt_size = cqe->pkt_size;
1588 cq_info.vtag = cqe->vlan_tag;
1589 cq_info.l4_cksum_pass = cqe->l4_cksum_pass;
1590 cq_info.ip_cksum_pass = cqe->ip_cksum_pass;
1591 cq_info.ipv6_frame = cqe->ipv6_frame;
1592 cq_info.vtp = cqe->vtp;
1593 cq_info.qnq = cqe->qnq;
1594 }else {
1595 cqe1 = (struct nic_hwlro_cqe_part1 *)cqe;
1596 cq_info.pkt_size = cqe2->coalesced_size;
1597 cq_info.vtag = cqe2->vlan_tag;
1598 cq_info.l4_cksum_pass = cqe2->l4_cksum_pass;
1599 cq_info.ip_cksum_pass = cqe2->ip_cksum_pass;
1600 cq_info.ipv6_frame = cqe2->ipv6_frame;
1601 cq_info.vtp = cqe2->vtp;
1602 cq_info.qnq = cqe1->qnq;
1603 }
1604
1605 cq_info.vtag = BSWAP_16(cq_info.vtag);
1606
1607 cq_info.num_frags = cq_info.pkt_size / rq->cfg.frag_size;
1608 if(cq_info.pkt_size % rq->cfg.frag_size)
1609 cq_info.num_frags++;
1610
1611 oce_rx_mbuf_chain(rq, &cq_info, &m);
1612
1613 if (m) {
1614 if(cqe2) {
1615 //assert(cqe2->valid != 0);
1616
1617 //assert(cqe2->cqe_type != 2);
1618 oce_correct_header(m, cqe1, cqe2);
1619 }
1620
1621 m->m_pkthdr.rcvif = sc->ifp;
1622 if (rq->queue_index)
1623 m->m_pkthdr.flowid = (rq->queue_index - 1);
1624 else
1625 m->m_pkthdr.flowid = rq->queue_index;
1626 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
1627
1628 /* This deternies if vlan tag is Valid */
1629 if (cq_info.vtp) {
1630 if (sc->function_mode & FNM_FLEX10_MODE) {
1631 /* FLEX10. If QnQ is not set, neglect VLAN */
1632 if (cq_info.qnq) {
1633 m->m_pkthdr.ether_vtag = cq_info.vtag;
1634 m->m_flags |= M_VLANTAG;
1635 }
1636 } else if (sc->pvid != (cq_info.vtag & VLAN_VID_MASK)) {
1637 /* In UMC mode generally pvid will be striped by
1638 hw. But in some cases we have seen it comes
1639 with pvid. So if pvid == vlan, neglect vlan.
1640 */
1641 m->m_pkthdr.ether_vtag = cq_info.vtag;
1642 m->m_flags |= M_VLANTAG;
1643 }
1644 }
1645 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1);
1646
1647 (*sc->ifp->if_input) (sc->ifp, m);
1648
1649 /* Update rx stats per queue */
1650 rq->rx_stats.rx_pkts++;
1651 rq->rx_stats.rx_bytes += cq_info.pkt_size;
1652 rq->rx_stats.rx_frags += cq_info.num_frags;
1653 rq->rx_stats.rx_ucast_pkts++;
1654 }
1655 return;
1656 }
1657
1658 static void
oce_rx(struct oce_rq * rq,struct oce_nic_rx_cqe * cqe)1659 oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe)
1660 {
1661 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1662 int len;
1663 struct mbuf *m = NULL;
1664 struct oce_common_cqe_info cq_info;
1665 uint16_t vtag = 0;
1666
1667 /* Is it a flush compl that has no data */
1668 if(!cqe->u0.s.num_fragments)
1669 goto exit;
1670
1671 len = cqe->u0.s.pkt_size;
1672 if (!len) {
1673 /*partial DMA workaround for Lancer*/
1674 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments);
1675 goto exit;
1676 }
1677
1678 if (!oce_cqe_portid_valid(sc, cqe)) {
1679 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments);
1680 goto exit;
1681 }
1682
1683 /* Get vlan_tag value */
1684 if(IS_BE(sc) || IS_SH(sc))
1685 vtag = BSWAP_16(cqe->u0.s.vlan_tag);
1686 else
1687 vtag = cqe->u0.s.vlan_tag;
1688
1689 cq_info.l4_cksum_pass = cqe->u0.s.l4_cksum_pass;
1690 cq_info.ip_cksum_pass = cqe->u0.s.ip_cksum_pass;
1691 cq_info.ipv6_frame = cqe->u0.s.ip_ver;
1692 cq_info.num_frags = cqe->u0.s.num_fragments;
1693 cq_info.pkt_size = cqe->u0.s.pkt_size;
1694
1695 oce_rx_mbuf_chain(rq, &cq_info, &m);
1696
1697 if (m) {
1698 m->m_pkthdr.rcvif = sc->ifp;
1699 if (rq->queue_index)
1700 m->m_pkthdr.flowid = (rq->queue_index - 1);
1701 else
1702 m->m_pkthdr.flowid = rq->queue_index;
1703 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
1704
1705 /* This deternies if vlan tag is Valid */
1706 if (oce_cqe_vtp_valid(sc, cqe)) {
1707 if (sc->function_mode & FNM_FLEX10_MODE) {
1708 /* FLEX10. If QnQ is not set, neglect VLAN */
1709 if (cqe->u0.s.qnq) {
1710 m->m_pkthdr.ether_vtag = vtag;
1711 m->m_flags |= M_VLANTAG;
1712 }
1713 } else if (sc->pvid != (vtag & VLAN_VID_MASK)) {
1714 /* In UMC mode generally pvid will be striped by
1715 hw. But in some cases we have seen it comes
1716 with pvid. So if pvid == vlan, neglect vlan.
1717 */
1718 m->m_pkthdr.ether_vtag = vtag;
1719 m->m_flags |= M_VLANTAG;
1720 }
1721 }
1722
1723 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1);
1724 #if defined(INET6) || defined(INET)
1725 /* Try to queue to LRO */
1726 if (IF_LRO_ENABLED(sc) &&
1727 (cqe->u0.s.ip_cksum_pass) &&
1728 (cqe->u0.s.l4_cksum_pass) &&
1729 (!cqe->u0.s.ip_ver) &&
1730 (rq->lro.lro_cnt != 0)) {
1731 if (tcp_lro_rx(&rq->lro, m, 0) == 0) {
1732 rq->lro_pkts_queued ++;
1733 goto post_done;
1734 }
1735 /* If LRO posting fails then try to post to STACK */
1736 }
1737 #endif
1738
1739 (*sc->ifp->if_input) (sc->ifp, m);
1740 #if defined(INET6) || defined(INET)
1741 post_done:
1742 #endif
1743 /* Update rx stats per queue */
1744 rq->rx_stats.rx_pkts++;
1745 rq->rx_stats.rx_bytes += cqe->u0.s.pkt_size;
1746 rq->rx_stats.rx_frags += cqe->u0.s.num_fragments;
1747 if (cqe->u0.s.pkt_type == OCE_MULTICAST_PACKET)
1748 rq->rx_stats.rx_mcast_pkts++;
1749 if (cqe->u0.s.pkt_type == OCE_UNICAST_PACKET)
1750 rq->rx_stats.rx_ucast_pkts++;
1751 }
1752 exit:
1753 return;
1754 }
1755
1756 void
oce_discard_rx_comp(struct oce_rq * rq,int num_frags)1757 oce_discard_rx_comp(struct oce_rq *rq, int num_frags)
1758 {
1759 uint32_t i = 0;
1760 struct oce_packet_desc *pd;
1761 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1762
1763 for (i = 0; i < num_frags; i++) {
1764 if (rq->ring->cidx == rq->ring->pidx) {
1765 device_printf(sc->dev,
1766 "oce_discard_rx_comp: Invalid RX completion - Queue is empty\n");
1767 return;
1768 }
1769 pd = &rq->pckts[rq->ring->cidx];
1770 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1771 bus_dmamap_unload(rq->tag, pd->map);
1772 if (pd->mbuf != NULL) {
1773 m_freem(pd->mbuf);
1774 pd->mbuf = NULL;
1775 }
1776
1777 RING_GET(rq->ring, 1);
1778 rq->pending--;
1779 }
1780 }
1781
1782 static int
oce_cqe_vtp_valid(POCE_SOFTC sc,struct oce_nic_rx_cqe * cqe)1783 oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe)
1784 {
1785 struct oce_nic_rx_cqe_v1 *cqe_v1;
1786 int vtp = 0;
1787
1788 if (sc->be3_native) {
1789 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe;
1790 vtp = cqe_v1->u0.s.vlan_tag_present;
1791 } else
1792 vtp = cqe->u0.s.vlan_tag_present;
1793
1794 return vtp;
1795
1796 }
1797
1798 static int
oce_cqe_portid_valid(POCE_SOFTC sc,struct oce_nic_rx_cqe * cqe)1799 oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe)
1800 {
1801 struct oce_nic_rx_cqe_v1 *cqe_v1;
1802 int port_id = 0;
1803
1804 if (sc->be3_native && (IS_BE(sc) || IS_SH(sc))) {
1805 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe;
1806 port_id = cqe_v1->u0.s.port;
1807 if (sc->port_id != port_id)
1808 return 0;
1809 } else
1810 ;/* For BE3 legacy and Lancer this is dummy */
1811
1812 return 1;
1813
1814 }
1815
1816 #if defined(INET6) || defined(INET)
1817 void
oce_rx_flush_lro(struct oce_rq * rq)1818 oce_rx_flush_lro(struct oce_rq *rq)
1819 {
1820 struct lro_ctrl *lro = &rq->lro;
1821 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1822
1823 if (!IF_LRO_ENABLED(sc))
1824 return;
1825
1826 tcp_lro_flush_all(lro);
1827 rq->lro_pkts_queued = 0;
1828
1829 return;
1830 }
1831
1832 static int
oce_init_lro(POCE_SOFTC sc)1833 oce_init_lro(POCE_SOFTC sc)
1834 {
1835 struct lro_ctrl *lro = NULL;
1836 int i = 0, rc = 0;
1837
1838 for (i = 0; i < sc->nrqs; i++) {
1839 lro = &sc->rq[i]->lro;
1840 rc = tcp_lro_init(lro);
1841 if (rc != 0) {
1842 device_printf(sc->dev, "LRO init failed\n");
1843 return rc;
1844 }
1845 lro->ifp = sc->ifp;
1846 }
1847
1848 return rc;
1849 }
1850
1851 void
oce_free_lro(POCE_SOFTC sc)1852 oce_free_lro(POCE_SOFTC sc)
1853 {
1854 struct lro_ctrl *lro = NULL;
1855 int i = 0;
1856
1857 for (i = 0; i < sc->nrqs; i++) {
1858 lro = &sc->rq[i]->lro;
1859 if (lro)
1860 tcp_lro_free(lro);
1861 }
1862 }
1863 #endif
1864
1865 int
oce_alloc_rx_bufs(struct oce_rq * rq,int count)1866 oce_alloc_rx_bufs(struct oce_rq *rq, int count)
1867 {
1868 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1869 int i, in, rc;
1870 struct oce_packet_desc *pd;
1871 bus_dma_segment_t segs[6];
1872 int nsegs, added = 0;
1873 struct oce_nic_rqe *rqe;
1874 pd_rxulp_db_t rxdb_reg;
1875 uint32_t val = 0;
1876 uint32_t oce_max_rq_posts = 64;
1877
1878 bzero(&rxdb_reg, sizeof(pd_rxulp_db_t));
1879 for (i = 0; i < count; i++) {
1880 in = (rq->ring->pidx + 1) % OCE_RQ_PACKET_ARRAY_SIZE;
1881
1882 pd = &rq->pckts[rq->ring->pidx];
1883 pd->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, oce_rq_buf_size);
1884 if (pd->mbuf == NULL) {
1885 device_printf(sc->dev, "mbuf allocation failed, size = %d\n",oce_rq_buf_size);
1886 break;
1887 }
1888 pd->mbuf->m_nextpkt = NULL;
1889
1890 pd->mbuf->m_len = pd->mbuf->m_pkthdr.len = rq->cfg.frag_size;
1891
1892 rc = bus_dmamap_load_mbuf_sg(rq->tag,
1893 pd->map,
1894 pd->mbuf,
1895 segs, &nsegs, BUS_DMA_NOWAIT);
1896 if (rc) {
1897 m_free(pd->mbuf);
1898 device_printf(sc->dev, "bus_dmamap_load_mbuf_sg failed rc = %d\n", rc);
1899 break;
1900 }
1901
1902 if (nsegs != 1) {
1903 i--;
1904 continue;
1905 }
1906
1907 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_PREREAD);
1908
1909 rqe = RING_GET_PRODUCER_ITEM_VA(rq->ring, struct oce_nic_rqe);
1910 rqe->u0.s.frag_pa_hi = ADDR_HI(segs[0].ds_addr);
1911 rqe->u0.s.frag_pa_lo = ADDR_LO(segs[0].ds_addr);
1912 DW_SWAP(u32ptr(rqe), sizeof(struct oce_nic_rqe));
1913 RING_PUT(rq->ring, 1);
1914 added++;
1915 rq->pending++;
1916 }
1917 oce_max_rq_posts = sc->enable_hwlro ? OCE_HWLRO_MAX_RQ_POSTS : OCE_MAX_RQ_POSTS;
1918 if (added != 0) {
1919 for (i = added / oce_max_rq_posts; i > 0; i--) {
1920 rxdb_reg.bits.num_posted = oce_max_rq_posts;
1921 rxdb_reg.bits.qid = rq->rq_id;
1922 if(rq->islro) {
1923 val |= rq->rq_id & DB_LRO_RQ_ID_MASK;
1924 val |= oce_max_rq_posts << 16;
1925 OCE_WRITE_REG32(sc, db, DB_OFFSET, val);
1926 }else {
1927 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0);
1928 }
1929 added -= oce_max_rq_posts;
1930 }
1931 if (added > 0) {
1932 rxdb_reg.bits.qid = rq->rq_id;
1933 rxdb_reg.bits.num_posted = added;
1934 if(rq->islro) {
1935 val |= rq->rq_id & DB_LRO_RQ_ID_MASK;
1936 val |= added << 16;
1937 OCE_WRITE_REG32(sc, db, DB_OFFSET, val);
1938 }else {
1939 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0);
1940 }
1941 }
1942 }
1943
1944 return 0;
1945 }
1946
1947 static void
oce_check_rx_bufs(POCE_SOFTC sc,uint32_t num_cqes,struct oce_rq * rq)1948 oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq)
1949 {
1950 if (num_cqes) {
1951 oce_arm_cq(sc, rq->cq->cq_id, num_cqes, FALSE);
1952 if(!sc->enable_hwlro) {
1953 if((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) > 1)
1954 oce_alloc_rx_bufs(rq, ((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) - 1));
1955 }else {
1956 if ((OCE_RQ_PACKET_ARRAY_SIZE -1 - rq->pending) > 64)
1957 oce_alloc_rx_bufs(rq, 64);
1958 }
1959 }
1960
1961 return;
1962 }
1963
1964 uint16_t
oce_rq_handler_lro(void * arg)1965 oce_rq_handler_lro(void *arg)
1966 {
1967 struct oce_rq *rq = (struct oce_rq *)arg;
1968 struct oce_cq *cq = rq->cq;
1969 POCE_SOFTC sc = rq->parent;
1970 struct nic_hwlro_singleton_cqe *cqe;
1971 struct nic_hwlro_cqe_part2 *cqe2;
1972 int num_cqes = 0;
1973
1974 LOCK(&rq->rx_lock);
1975 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
1976 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe);
1977 while (cqe->valid) {
1978 if(cqe->cqe_type == 0) { /* singleton cqe */
1979 /* we should not get singleton cqe after cqe1 on same rq */
1980 if(rq->cqe_firstpart != NULL) {
1981 device_printf(sc->dev, "Got singleton cqe after cqe1 \n");
1982 goto exit_rq_handler_lro;
1983 }
1984 if(cqe->error != 0) {
1985 rq->rx_stats.rxcp_err++;
1986 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
1987 }
1988 oce_rx_lro(rq, cqe, NULL);
1989 rq->rx_stats.rx_compl++;
1990 cqe->valid = 0;
1991 RING_GET(cq->ring, 1);
1992 num_cqes++;
1993 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
1994 break;
1995 }else if(cqe->cqe_type == 0x1) { /* first part */
1996 /* we should not get cqe1 after cqe1 on same rq */
1997 if(rq->cqe_firstpart != NULL) {
1998 device_printf(sc->dev, "Got cqe1 after cqe1 \n");
1999 goto exit_rq_handler_lro;
2000 }
2001 rq->cqe_firstpart = (struct nic_hwlro_cqe_part1 *)cqe;
2002 RING_GET(cq->ring, 1);
2003 }else if(cqe->cqe_type == 0x2) { /* second part */
2004 cqe2 = (struct nic_hwlro_cqe_part2 *)cqe;
2005 if(cqe2->error != 0) {
2006 rq->rx_stats.rxcp_err++;
2007 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
2008 }
2009 /* We should not get cqe2 without cqe1 */
2010 if(rq->cqe_firstpart == NULL) {
2011 device_printf(sc->dev, "Got cqe2 without cqe1 \n");
2012 goto exit_rq_handler_lro;
2013 }
2014 oce_rx_lro(rq, (struct nic_hwlro_singleton_cqe *)rq->cqe_firstpart, cqe2);
2015
2016 rq->rx_stats.rx_compl++;
2017 rq->cqe_firstpart->valid = 0;
2018 cqe2->valid = 0;
2019 rq->cqe_firstpart = NULL;
2020
2021 RING_GET(cq->ring, 1);
2022 num_cqes += 2;
2023 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
2024 break;
2025 }
2026
2027 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2028 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe);
2029 }
2030 oce_check_rx_bufs(sc, num_cqes, rq);
2031 exit_rq_handler_lro:
2032 UNLOCK(&rq->rx_lock);
2033 return 0;
2034 }
2035
2036 /* Handle the Completion Queue for receive */
2037 uint16_t
oce_rq_handler(void * arg)2038 oce_rq_handler(void *arg)
2039 {
2040 struct epoch_tracker et;
2041 struct oce_rq *rq = (struct oce_rq *)arg;
2042 struct oce_cq *cq = rq->cq;
2043 POCE_SOFTC sc = rq->parent;
2044 struct oce_nic_rx_cqe *cqe;
2045 int num_cqes = 0;
2046
2047 NET_EPOCH_ENTER(et);
2048 if(rq->islro) {
2049 oce_rq_handler_lro(arg);
2050 NET_EPOCH_EXIT(et);
2051 return 0;
2052 }
2053 LOCK(&rq->rx_lock);
2054 bus_dmamap_sync(cq->ring->dma.tag,
2055 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2056 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe);
2057 while (cqe->u0.dw[2]) {
2058 DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe));
2059
2060 if (cqe->u0.s.error == 0) {
2061 oce_rx(rq, cqe);
2062 } else {
2063 rq->rx_stats.rxcp_err++;
2064 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
2065 /* Post L3/L4 errors to stack.*/
2066 oce_rx(rq, cqe);
2067 }
2068 rq->rx_stats.rx_compl++;
2069 cqe->u0.dw[2] = 0;
2070
2071 #if defined(INET6) || defined(INET)
2072 if (IF_LRO_ENABLED(sc) && rq->lro_pkts_queued >= 16) {
2073 oce_rx_flush_lro(rq);
2074 }
2075 #endif
2076
2077 RING_GET(cq->ring, 1);
2078 bus_dmamap_sync(cq->ring->dma.tag,
2079 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2080 cqe =
2081 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe);
2082 num_cqes++;
2083 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
2084 break;
2085 }
2086
2087 #if defined(INET6) || defined(INET)
2088 if (IF_LRO_ENABLED(sc))
2089 oce_rx_flush_lro(rq);
2090 #endif
2091
2092 oce_check_rx_bufs(sc, num_cqes, rq);
2093 UNLOCK(&rq->rx_lock);
2094 NET_EPOCH_EXIT(et);
2095 return 0;
2096
2097 }
2098
2099 /*****************************************************************************
2100 * Helper function prototypes in this file *
2101 *****************************************************************************/
2102
2103 static void
oce_attach_ifp(POCE_SOFTC sc)2104 oce_attach_ifp(POCE_SOFTC sc)
2105 {
2106
2107 sc->ifp = if_alloc(IFT_ETHER);
2108
2109 ifmedia_init(&sc->media, IFM_IMASK, oce_media_change, oce_media_status);
2110 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2111 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO);
2112
2113 sc->ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_KNOWSEPOCH;
2114 sc->ifp->if_ioctl = oce_ioctl;
2115 sc->ifp->if_start = oce_start;
2116 sc->ifp->if_init = oce_init;
2117 sc->ifp->if_mtu = ETHERMTU;
2118 sc->ifp->if_softc = sc;
2119 sc->ifp->if_transmit = oce_multiq_start;
2120 sc->ifp->if_qflush = oce_multiq_flush;
2121
2122 if_initname(sc->ifp,
2123 device_get_name(sc->dev), device_get_unit(sc->dev));
2124
2125 sc->ifp->if_snd.ifq_drv_maxlen = OCE_MAX_TX_DESC - 1;
2126 IFQ_SET_MAXLEN(&sc->ifp->if_snd, sc->ifp->if_snd.ifq_drv_maxlen);
2127 IFQ_SET_READY(&sc->ifp->if_snd);
2128
2129 sc->ifp->if_hwassist = OCE_IF_HWASSIST;
2130 sc->ifp->if_hwassist |= CSUM_TSO;
2131 sc->ifp->if_hwassist |= (CSUM_IP | CSUM_TCP | CSUM_UDP);
2132
2133 sc->ifp->if_capabilities = OCE_IF_CAPABILITIES;
2134 sc->ifp->if_capabilities |= IFCAP_HWCSUM;
2135 sc->ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2136
2137 #if defined(INET6) || defined(INET)
2138 sc->ifp->if_capabilities |= IFCAP_TSO;
2139 sc->ifp->if_capabilities |= IFCAP_LRO;
2140 sc->ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
2141 #endif
2142
2143 sc->ifp->if_capenable = sc->ifp->if_capabilities;
2144 sc->ifp->if_baudrate = IF_Gbps(10);
2145
2146 sc->ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
2147 sc->ifp->if_hw_tsomaxsegcount = OCE_MAX_TX_ELEMENTS;
2148 sc->ifp->if_hw_tsomaxsegsize = 4096;
2149
2150 ether_ifattach(sc->ifp, sc->macaddr.mac_addr);
2151 }
2152
2153 static void
oce_add_vlan(void * arg,struct ifnet * ifp,uint16_t vtag)2154 oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag)
2155 {
2156 POCE_SOFTC sc = ifp->if_softc;
2157
2158 if (ifp->if_softc != arg)
2159 return;
2160 if ((vtag == 0) || (vtag > 4095))
2161 return;
2162
2163 sc->vlan_tag[vtag] = 1;
2164 sc->vlans_added++;
2165 if (sc->vlans_added <= (sc->max_vlans + 1))
2166 oce_vid_config(sc);
2167 }
2168
2169 static void
oce_del_vlan(void * arg,struct ifnet * ifp,uint16_t vtag)2170 oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag)
2171 {
2172 POCE_SOFTC sc = ifp->if_softc;
2173
2174 if (ifp->if_softc != arg)
2175 return;
2176 if ((vtag == 0) || (vtag > 4095))
2177 return;
2178
2179 sc->vlan_tag[vtag] = 0;
2180 sc->vlans_added--;
2181 oce_vid_config(sc);
2182 }
2183
2184 /*
2185 * A max of 64 vlans can be configured in BE. If the user configures
2186 * more, place the card in vlan promiscuous mode.
2187 */
2188 static int
oce_vid_config(POCE_SOFTC sc)2189 oce_vid_config(POCE_SOFTC sc)
2190 {
2191 struct normal_vlan vtags[MAX_VLANFILTER_SIZE];
2192 uint16_t ntags = 0, i;
2193 int status = 0;
2194
2195 if ((sc->vlans_added <= MAX_VLANFILTER_SIZE) &&
2196 (sc->ifp->if_capenable & IFCAP_VLAN_HWFILTER)) {
2197 for (i = 0; i < MAX_VLANS; i++) {
2198 if (sc->vlan_tag[i]) {
2199 vtags[ntags].vtag = i;
2200 ntags++;
2201 }
2202 }
2203 if (ntags)
2204 status = oce_config_vlan(sc, (uint8_t) sc->if_id,
2205 vtags, ntags, 1, 0);
2206 } else
2207 status = oce_config_vlan(sc, (uint8_t) sc->if_id,
2208 NULL, 0, 1, 1);
2209 return status;
2210 }
2211
2212 static void
oce_mac_addr_set(POCE_SOFTC sc)2213 oce_mac_addr_set(POCE_SOFTC sc)
2214 {
2215 uint32_t old_pmac_id = sc->pmac_id;
2216 int status = 0;
2217
2218 status = bcmp((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr,
2219 sc->macaddr.size_of_struct);
2220 if (!status)
2221 return;
2222
2223 status = oce_mbox_macaddr_add(sc, (uint8_t *)(IF_LLADDR(sc->ifp)),
2224 sc->if_id, &sc->pmac_id);
2225 if (!status) {
2226 status = oce_mbox_macaddr_del(sc, sc->if_id, old_pmac_id);
2227 bcopy((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr,
2228 sc->macaddr.size_of_struct);
2229 }
2230 if (status)
2231 device_printf(sc->dev, "Failed update macaddress\n");
2232
2233 }
2234
2235 static int
oce_handle_passthrough(struct ifnet * ifp,caddr_t data)2236 oce_handle_passthrough(struct ifnet *ifp, caddr_t data)
2237 {
2238 POCE_SOFTC sc = ifp->if_softc;
2239 struct ifreq *ifr = (struct ifreq *)data;
2240 int rc = ENXIO;
2241 char cookie[32] = {0};
2242 void *priv_data = ifr_data_get_ptr(ifr);
2243 void *ioctl_ptr;
2244 uint32_t req_size;
2245 struct mbx_hdr req;
2246 OCE_DMA_MEM dma_mem;
2247
2248 if (copyin(priv_data, cookie, strlen(IOCTL_COOKIE)))
2249 return EFAULT;
2250
2251 if (memcmp(cookie, IOCTL_COOKIE, strlen(IOCTL_COOKIE)))
2252 return EINVAL;
2253
2254 ioctl_ptr = (char *)priv_data + strlen(IOCTL_COOKIE);
2255 if (copyin(ioctl_ptr, &req, sizeof(struct mbx_hdr)))
2256 return EFAULT;
2257
2258 req_size = le32toh(req.u0.req.request_length);
2259 if (req_size > 65536)
2260 return EINVAL;
2261
2262 req_size += sizeof(struct mbx_hdr);
2263 rc = oce_dma_alloc(sc, req_size, &dma_mem, 0);
2264 if (rc)
2265 return ENOMEM;
2266
2267 if (copyin(ioctl_ptr, OCE_DMAPTR(&dma_mem,char), req_size)) {
2268 rc = EFAULT;
2269 goto dma_free;
2270 }
2271
2272 rc = oce_pass_through_mbox(sc, &dma_mem, req_size);
2273 if (rc) {
2274 rc = EIO;
2275 goto dma_free;
2276 }
2277
2278 if (copyout(OCE_DMAPTR(&dma_mem,char), ioctl_ptr, req_size)) {
2279 rc = EFAULT;
2280 goto dma_free;
2281 }
2282
2283 /*
2284 firmware is filling all the attributes for this ioctl except
2285 the driver version..so fill it
2286 */
2287 if(req.u0.rsp.opcode == OPCODE_COMMON_GET_CNTL_ATTRIBUTES) {
2288 struct mbx_common_get_cntl_attr *fw_cmd =
2289 (struct mbx_common_get_cntl_attr *)ioctl_ptr;
2290 _Static_assert(sizeof(COMPONENT_REVISION) <=
2291 sizeof(fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str),
2292 "driver version string too long");
2293
2294 rc = copyout(COMPONENT_REVISION,
2295 fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str,
2296 sizeof(COMPONENT_REVISION));
2297 }
2298
2299 dma_free:
2300 oce_dma_free(sc, &dma_mem);
2301 return rc;
2302
2303 }
2304
2305 static void
oce_eqd_set_periodic(POCE_SOFTC sc)2306 oce_eqd_set_periodic(POCE_SOFTC sc)
2307 {
2308 struct oce_set_eqd set_eqd[OCE_MAX_EQ];
2309 struct oce_aic_obj *aic;
2310 struct oce_eq *eqo;
2311 uint64_t now = 0, delta;
2312 int eqd, i, num = 0;
2313 uint32_t tx_reqs = 0, rxpkts = 0, pps;
2314 struct oce_wq *wq;
2315 struct oce_rq *rq;
2316
2317 #define ticks_to_msecs(t) (1000 * (t) / hz)
2318
2319 for (i = 0 ; i < sc->neqs; i++) {
2320 eqo = sc->eq[i];
2321 aic = &sc->aic_obj[i];
2322 /* When setting the static eq delay from the user space */
2323 if (!aic->enable) {
2324 if (aic->ticks)
2325 aic->ticks = 0;
2326 eqd = aic->et_eqd;
2327 goto modify_eqd;
2328 }
2329
2330 if (i == 0) {
2331 rq = sc->rq[0];
2332 rxpkts = rq->rx_stats.rx_pkts;
2333 } else
2334 rxpkts = 0;
2335 if (i + 1 < sc->nrqs) {
2336 rq = sc->rq[i + 1];
2337 rxpkts += rq->rx_stats.rx_pkts;
2338 }
2339 if (i < sc->nwqs) {
2340 wq = sc->wq[i];
2341 tx_reqs = wq->tx_stats.tx_reqs;
2342 } else
2343 tx_reqs = 0;
2344 now = ticks;
2345
2346 if (!aic->ticks || now < aic->ticks ||
2347 rxpkts < aic->prev_rxpkts || tx_reqs < aic->prev_txreqs) {
2348 aic->prev_rxpkts = rxpkts;
2349 aic->prev_txreqs = tx_reqs;
2350 aic->ticks = now;
2351 continue;
2352 }
2353
2354 delta = ticks_to_msecs(now - aic->ticks);
2355
2356 pps = (((uint32_t)(rxpkts - aic->prev_rxpkts) * 1000) / delta) +
2357 (((uint32_t)(tx_reqs - aic->prev_txreqs) * 1000) / delta);
2358 eqd = (pps / 15000) << 2;
2359 if (eqd < 8)
2360 eqd = 0;
2361
2362 /* Make sure that the eq delay is in the known range */
2363 eqd = min(eqd, aic->max_eqd);
2364 eqd = max(eqd, aic->min_eqd);
2365
2366 aic->prev_rxpkts = rxpkts;
2367 aic->prev_txreqs = tx_reqs;
2368 aic->ticks = now;
2369
2370 modify_eqd:
2371 if (eqd != aic->cur_eqd) {
2372 set_eqd[num].delay_multiplier = (eqd * 65)/100;
2373 set_eqd[num].eq_id = eqo->eq_id;
2374 aic->cur_eqd = eqd;
2375 num++;
2376 }
2377 }
2378
2379 /* Is there atleast one eq that needs to be modified? */
2380 for(i = 0; i < num; i += 8) {
2381 if((num - i) >=8 )
2382 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], 8);
2383 else
2384 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], (num - i));
2385 }
2386
2387 }
2388
oce_detect_hw_error(POCE_SOFTC sc)2389 static void oce_detect_hw_error(POCE_SOFTC sc)
2390 {
2391
2392 uint32_t ue_low = 0, ue_high = 0, ue_low_mask = 0, ue_high_mask = 0;
2393 uint32_t sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2394 uint32_t i;
2395
2396 if (sc->hw_error)
2397 return;
2398
2399 if (IS_XE201(sc)) {
2400 sliport_status = OCE_READ_REG32(sc, db, SLIPORT_STATUS_OFFSET);
2401 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2402 sliport_err1 = OCE_READ_REG32(sc, db, SLIPORT_ERROR1_OFFSET);
2403 sliport_err2 = OCE_READ_REG32(sc, db, SLIPORT_ERROR2_OFFSET);
2404 }
2405 } else {
2406 ue_low = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW);
2407 ue_high = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HIGH);
2408 ue_low_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW_MASK);
2409 ue_high_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HI_MASK);
2410
2411 ue_low = (ue_low & ~ue_low_mask);
2412 ue_high = (ue_high & ~ue_high_mask);
2413 }
2414
2415 /* On certain platforms BE hardware can indicate spurious UEs.
2416 * Allow the h/w to stop working completely in case of a real UE.
2417 * Hence not setting the hw_error for UE detection.
2418 */
2419 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2420 sc->hw_error = TRUE;
2421 device_printf(sc->dev, "Error detected in the card\n");
2422 }
2423
2424 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2425 device_printf(sc->dev,
2426 "ERR: sliport status 0x%x\n", sliport_status);
2427 device_printf(sc->dev,
2428 "ERR: sliport error1 0x%x\n", sliport_err1);
2429 device_printf(sc->dev,
2430 "ERR: sliport error2 0x%x\n", sliport_err2);
2431 }
2432
2433 if (ue_low) {
2434 for (i = 0; ue_low; ue_low >>= 1, i++) {
2435 if (ue_low & 1)
2436 device_printf(sc->dev, "UE: %s bit set\n",
2437 ue_status_low_desc[i]);
2438 }
2439 }
2440
2441 if (ue_high) {
2442 for (i = 0; ue_high; ue_high >>= 1, i++) {
2443 if (ue_high & 1)
2444 device_printf(sc->dev, "UE: %s bit set\n",
2445 ue_status_hi_desc[i]);
2446 }
2447 }
2448
2449 }
2450
2451 static void
oce_local_timer(void * arg)2452 oce_local_timer(void *arg)
2453 {
2454 POCE_SOFTC sc = arg;
2455 int i = 0;
2456
2457 oce_detect_hw_error(sc);
2458 oce_refresh_nic_stats(sc);
2459 oce_refresh_queue_stats(sc);
2460 oce_mac_addr_set(sc);
2461
2462 /* TX Watch Dog*/
2463 for (i = 0; i < sc->nwqs; i++)
2464 oce_tx_restart(sc, sc->wq[i]);
2465
2466 /* calculate and set the eq delay for optimal interrupt rate */
2467 if (IS_BE(sc) || IS_SH(sc))
2468 oce_eqd_set_periodic(sc);
2469
2470 callout_reset(&sc->timer, hz, oce_local_timer, sc);
2471 }
2472
2473 static void
oce_tx_compl_clean(POCE_SOFTC sc)2474 oce_tx_compl_clean(POCE_SOFTC sc)
2475 {
2476 struct oce_wq *wq;
2477 int i = 0, timeo = 0, num_wqes = 0;
2478 int pending_txqs = sc->nwqs;
2479
2480 /* Stop polling for compls when HW has been silent for 10ms or
2481 * hw_error or no outstanding completions expected
2482 */
2483 do {
2484 pending_txqs = sc->nwqs;
2485
2486 for_all_wq_queues(sc, wq, i) {
2487 num_wqes = oce_wq_handler(wq);
2488
2489 if(num_wqes)
2490 timeo = 0;
2491
2492 if(!wq->ring->num_used)
2493 pending_txqs--;
2494 }
2495
2496 if (pending_txqs == 0 || ++timeo > 10 || sc->hw_error)
2497 break;
2498
2499 DELAY(1000);
2500 } while (TRUE);
2501
2502 for_all_wq_queues(sc, wq, i) {
2503 while(wq->ring->num_used) {
2504 LOCK(&wq->tx_compl_lock);
2505 oce_process_tx_completion(wq);
2506 UNLOCK(&wq->tx_compl_lock);
2507 }
2508 }
2509
2510 }
2511
2512 /* NOTE : This should only be called holding
2513 * DEVICE_LOCK.
2514 */
2515 static void
oce_if_deactivate(POCE_SOFTC sc)2516 oce_if_deactivate(POCE_SOFTC sc)
2517 {
2518 int i;
2519 struct oce_rq *rq;
2520 struct oce_wq *wq;
2521 struct oce_eq *eq;
2522
2523 sc->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
2524
2525 oce_tx_compl_clean(sc);
2526
2527 /* Stop intrs and finish any bottom halves pending */
2528 oce_hw_intr_disable(sc);
2529
2530 /* Since taskqueue_drain takes a Gaint Lock, We should not acquire
2531 any other lock. So unlock device lock and require after
2532 completing taskqueue_drain.
2533 */
2534 UNLOCK(&sc->dev_lock);
2535 for (i = 0; i < sc->intr_count; i++) {
2536 if (sc->intrs[i].tq != NULL) {
2537 taskqueue_drain(sc->intrs[i].tq, &sc->intrs[i].task);
2538 }
2539 }
2540 LOCK(&sc->dev_lock);
2541
2542 /* Delete RX queue in card with flush param */
2543 oce_stop_rx(sc);
2544
2545 /* Invalidate any pending cq and eq entries*/
2546 for_all_evnt_queues(sc, eq, i)
2547 oce_drain_eq(eq);
2548 for_all_rq_queues(sc, rq, i)
2549 oce_drain_rq_cq(rq);
2550 for_all_wq_queues(sc, wq, i)
2551 oce_drain_wq_cq(wq);
2552
2553 /* But still we need to get MCC aync events.
2554 So enable intrs and also arm first EQ
2555 */
2556 oce_hw_intr_enable(sc);
2557 oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE);
2558
2559 DELAY(10);
2560 }
2561
2562 static void
oce_if_activate(POCE_SOFTC sc)2563 oce_if_activate(POCE_SOFTC sc)
2564 {
2565 struct oce_eq *eq;
2566 struct oce_rq *rq;
2567 struct oce_wq *wq;
2568 int i, rc = 0;
2569
2570 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING;
2571
2572 oce_hw_intr_disable(sc);
2573
2574 oce_start_rx(sc);
2575
2576 for_all_rq_queues(sc, rq, i) {
2577 rc = oce_start_rq(rq);
2578 if (rc)
2579 device_printf(sc->dev, "Unable to start RX\n");
2580 }
2581
2582 for_all_wq_queues(sc, wq, i) {
2583 rc = oce_start_wq(wq);
2584 if (rc)
2585 device_printf(sc->dev, "Unable to start TX\n");
2586 }
2587
2588 for_all_evnt_queues(sc, eq, i)
2589 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE);
2590
2591 oce_hw_intr_enable(sc);
2592
2593 }
2594
2595 static void
process_link_state(POCE_SOFTC sc,struct oce_async_cqe_link_state * acqe)2596 process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe)
2597 {
2598 /* Update Link status */
2599 if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) ==
2600 ASYNC_EVENT_LINK_UP) {
2601 sc->link_status = ASYNC_EVENT_LINK_UP;
2602 if_link_state_change(sc->ifp, LINK_STATE_UP);
2603 } else {
2604 sc->link_status = ASYNC_EVENT_LINK_DOWN;
2605 if_link_state_change(sc->ifp, LINK_STATE_DOWN);
2606 }
2607 }
2608
oce_async_grp5_osbmc_process(POCE_SOFTC sc,struct oce_async_evt_grp5_os2bmc * evt)2609 static void oce_async_grp5_osbmc_process(POCE_SOFTC sc,
2610 struct oce_async_evt_grp5_os2bmc *evt)
2611 {
2612 DW_SWAP(evt, sizeof(struct oce_async_evt_grp5_os2bmc));
2613 if (evt->u.s.mgmt_enable)
2614 sc->flags |= OCE_FLAGS_OS2BMC;
2615 else
2616 return;
2617
2618 sc->bmc_filt_mask = evt->u.s.arp_filter;
2619 sc->bmc_filt_mask |= (evt->u.s.dhcp_client_filt << 1);
2620 sc->bmc_filt_mask |= (evt->u.s.dhcp_server_filt << 2);
2621 sc->bmc_filt_mask |= (evt->u.s.net_bios_filt << 3);
2622 sc->bmc_filt_mask |= (evt->u.s.bcast_filt << 4);
2623 sc->bmc_filt_mask |= (evt->u.s.ipv6_nbr_filt << 5);
2624 sc->bmc_filt_mask |= (evt->u.s.ipv6_ra_filt << 6);
2625 sc->bmc_filt_mask |= (evt->u.s.ipv6_ras_filt << 7);
2626 sc->bmc_filt_mask |= (evt->u.s.mcast_filt << 8);
2627 }
2628
oce_process_grp5_events(POCE_SOFTC sc,struct oce_mq_cqe * cqe)2629 static void oce_process_grp5_events(POCE_SOFTC sc, struct oce_mq_cqe *cqe)
2630 {
2631 struct oce_async_event_grp5_pvid_state *gcqe;
2632 struct oce_async_evt_grp5_os2bmc *bmccqe;
2633
2634 switch (cqe->u0.s.async_type) {
2635 case ASYNC_EVENT_PVID_STATE:
2636 /* GRP5 PVID */
2637 gcqe = (struct oce_async_event_grp5_pvid_state *)cqe;
2638 if (gcqe->enabled)
2639 sc->pvid = gcqe->tag & VLAN_VID_MASK;
2640 else
2641 sc->pvid = 0;
2642 break;
2643 case ASYNC_EVENT_OS2BMC:
2644 bmccqe = (struct oce_async_evt_grp5_os2bmc *)cqe;
2645 oce_async_grp5_osbmc_process(sc, bmccqe);
2646 break;
2647 default:
2648 break;
2649 }
2650 }
2651
2652 /* Handle the Completion Queue for the Mailbox/Async notifications */
2653 uint16_t
oce_mq_handler(void * arg)2654 oce_mq_handler(void *arg)
2655 {
2656 struct oce_mq *mq = (struct oce_mq *)arg;
2657 POCE_SOFTC sc = mq->parent;
2658 struct oce_cq *cq = mq->cq;
2659 int num_cqes = 0, evt_type = 0, optype = 0;
2660 struct oce_mq_cqe *cqe;
2661 struct oce_async_cqe_link_state *acqe;
2662 struct oce_async_event_qnq *dbgcqe;
2663
2664 bus_dmamap_sync(cq->ring->dma.tag,
2665 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2666 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe);
2667
2668 while (cqe->u0.dw[3]) {
2669 DW_SWAP((uint32_t *) cqe, sizeof(oce_mq_cqe));
2670 if (cqe->u0.s.async_event) {
2671 evt_type = cqe->u0.s.event_type;
2672 optype = cqe->u0.s.async_type;
2673 if (evt_type == ASYNC_EVENT_CODE_LINK_STATE) {
2674 /* Link status evt */
2675 acqe = (struct oce_async_cqe_link_state *)cqe;
2676 process_link_state(sc, acqe);
2677 } else if (evt_type == ASYNC_EVENT_GRP5) {
2678 oce_process_grp5_events(sc, cqe);
2679 } else if (evt_type == ASYNC_EVENT_CODE_DEBUG &&
2680 optype == ASYNC_EVENT_DEBUG_QNQ) {
2681 dbgcqe = (struct oce_async_event_qnq *)cqe;
2682 if(dbgcqe->valid)
2683 sc->qnqid = dbgcqe->vlan_tag;
2684 sc->qnq_debug_event = TRUE;
2685 }
2686 }
2687 cqe->u0.dw[3] = 0;
2688 RING_GET(cq->ring, 1);
2689 bus_dmamap_sync(cq->ring->dma.tag,
2690 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2691 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe);
2692 num_cqes++;
2693 }
2694
2695 if (num_cqes)
2696 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE);
2697
2698 return 0;
2699 }
2700
2701 static void
setup_max_queues_want(POCE_SOFTC sc)2702 setup_max_queues_want(POCE_SOFTC sc)
2703 {
2704 /* Check if it is FLEX machine. Is so dont use RSS */
2705 if ((sc->function_mode & FNM_FLEX10_MODE) ||
2706 (sc->function_mode & FNM_UMC_MODE) ||
2707 (sc->function_mode & FNM_VNIC_MODE) ||
2708 (!is_rss_enabled(sc)) ||
2709 IS_BE2(sc)) {
2710 sc->nrqs = 1;
2711 sc->nwqs = 1;
2712 } else {
2713 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1;
2714 sc->nwqs = MIN(OCE_NCPUS, sc->nrssqs);
2715 }
2716
2717 if (IS_BE2(sc) && is_rss_enabled(sc))
2718 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1;
2719 }
2720
2721 static void
update_queues_got(POCE_SOFTC sc)2722 update_queues_got(POCE_SOFTC sc)
2723 {
2724 if (is_rss_enabled(sc)) {
2725 sc->nrqs = sc->intr_count + 1;
2726 sc->nwqs = sc->intr_count;
2727 } else {
2728 sc->nrqs = 1;
2729 sc->nwqs = 1;
2730 }
2731
2732 if (IS_BE2(sc))
2733 sc->nwqs = 1;
2734 }
2735
2736 static int
oce_check_ipv6_ext_hdr(struct mbuf * m)2737 oce_check_ipv6_ext_hdr(struct mbuf *m)
2738 {
2739 struct ether_header *eh = mtod(m, struct ether_header *);
2740 caddr_t m_datatemp = m->m_data;
2741
2742 if (eh->ether_type == htons(ETHERTYPE_IPV6)) {
2743 m->m_data += sizeof(struct ether_header);
2744 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
2745
2746 if((ip6->ip6_nxt != IPPROTO_TCP) && \
2747 (ip6->ip6_nxt != IPPROTO_UDP)){
2748 struct ip6_ext *ip6e = NULL;
2749 m->m_data += sizeof(struct ip6_hdr);
2750
2751 ip6e = (struct ip6_ext *) mtod(m, struct ip6_ext *);
2752 if(ip6e->ip6e_len == 0xff) {
2753 m->m_data = m_datatemp;
2754 return TRUE;
2755 }
2756 }
2757 m->m_data = m_datatemp;
2758 }
2759 return FALSE;
2760 }
2761
2762 static int
is_be3_a1(POCE_SOFTC sc)2763 is_be3_a1(POCE_SOFTC sc)
2764 {
2765 if((sc->flags & OCE_FLAGS_BE3) && ((sc->asic_revision & 0xFF) < 2)) {
2766 return TRUE;
2767 }
2768 return FALSE;
2769 }
2770
2771 static struct mbuf *
oce_insert_vlan_tag(POCE_SOFTC sc,struct mbuf * m,boolean_t * complete)2772 oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete)
2773 {
2774 uint16_t vlan_tag = 0;
2775
2776 if(!M_WRITABLE(m))
2777 return NULL;
2778
2779 /* Embed vlan tag in the packet if it is not part of it */
2780 if(m->m_flags & M_VLANTAG) {
2781 vlan_tag = EVL_VLANOFTAG(m->m_pkthdr.ether_vtag);
2782 m->m_flags &= ~M_VLANTAG;
2783 }
2784
2785 /* if UMC, ignore vlan tag insertion and instead insert pvid */
2786 if(sc->pvid) {
2787 if(!vlan_tag)
2788 vlan_tag = sc->pvid;
2789 if (complete)
2790 *complete = FALSE;
2791 }
2792
2793 if(vlan_tag) {
2794 m = ether_vlanencap(m, vlan_tag);
2795 }
2796
2797 if(sc->qnqid) {
2798 m = ether_vlanencap(m, sc->qnqid);
2799
2800 if (complete)
2801 *complete = FALSE;
2802 }
2803 return m;
2804 }
2805
2806 static int
oce_tx_asic_stall_verify(POCE_SOFTC sc,struct mbuf * m)2807 oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m)
2808 {
2809 if(is_be3_a1(sc) && IS_QNQ_OR_UMC(sc) && \
2810 oce_check_ipv6_ext_hdr(m)) {
2811 return TRUE;
2812 }
2813 return FALSE;
2814 }
2815
2816 static void
oce_get_config(POCE_SOFTC sc)2817 oce_get_config(POCE_SOFTC sc)
2818 {
2819 int rc = 0;
2820 uint32_t max_rss = 0;
2821
2822 if ((IS_BE(sc) || IS_SH(sc)) && (!sc->be3_native))
2823 max_rss = OCE_LEGACY_MODE_RSS;
2824 else
2825 max_rss = OCE_MAX_RSS;
2826
2827 if (!IS_BE(sc)) {
2828 rc = oce_get_profile_config(sc, max_rss);
2829 if (rc) {
2830 sc->nwqs = OCE_MAX_WQ;
2831 sc->nrssqs = max_rss;
2832 sc->nrqs = sc->nrssqs + 1;
2833 }
2834 }
2835 else { /* For BE3 don't rely on fw for determining the resources */
2836 sc->nrssqs = max_rss;
2837 sc->nrqs = sc->nrssqs + 1;
2838 sc->nwqs = OCE_MAX_WQ;
2839 sc->max_vlans = MAX_VLANFILTER_SIZE;
2840 }
2841 }
2842
2843 static void
oce_rdma_close(void)2844 oce_rdma_close(void)
2845 {
2846 if (oce_rdma_if != NULL) {
2847 oce_rdma_if = NULL;
2848 }
2849 }
2850
2851 static void
oce_get_mac_addr(POCE_SOFTC sc,uint8_t * macaddr)2852 oce_get_mac_addr(POCE_SOFTC sc, uint8_t *macaddr)
2853 {
2854 memcpy(macaddr, sc->macaddr.mac_addr, 6);
2855 }
2856
2857 int
oce_register_rdma(POCE_RDMA_INFO rdma_info,POCE_RDMA_IF rdma_if)2858 oce_register_rdma(POCE_RDMA_INFO rdma_info, POCE_RDMA_IF rdma_if)
2859 {
2860 POCE_SOFTC sc;
2861 struct oce_dev_info di;
2862 int i;
2863
2864 if ((rdma_info == NULL) || (rdma_if == NULL)) {
2865 return -EINVAL;
2866 }
2867
2868 if ((rdma_info->size != OCE_RDMA_INFO_SIZE) ||
2869 (rdma_if->size != OCE_RDMA_IF_SIZE)) {
2870 return -ENXIO;
2871 }
2872
2873 rdma_info->close = oce_rdma_close;
2874 rdma_info->mbox_post = oce_mbox_post;
2875 rdma_info->common_req_hdr_init = mbx_common_req_hdr_init;
2876 rdma_info->get_mac_addr = oce_get_mac_addr;
2877
2878 oce_rdma_if = rdma_if;
2879
2880 sc = softc_head;
2881 while (sc != NULL) {
2882 if (oce_rdma_if->announce != NULL) {
2883 memset(&di, 0, sizeof(di));
2884 di.dev = sc->dev;
2885 di.softc = sc;
2886 di.ifp = sc->ifp;
2887 di.db_bhandle = sc->db_bhandle;
2888 di.db_btag = sc->db_btag;
2889 di.db_page_size = 4096;
2890 if (sc->flags & OCE_FLAGS_USING_MSIX) {
2891 di.intr_mode = OCE_INTERRUPT_MODE_MSIX;
2892 } else if (sc->flags & OCE_FLAGS_USING_MSI) {
2893 di.intr_mode = OCE_INTERRUPT_MODE_MSI;
2894 } else {
2895 di.intr_mode = OCE_INTERRUPT_MODE_INTX;
2896 }
2897 di.dev_family = OCE_GEN2_FAMILY; // fixme: must detect skyhawk
2898 if (di.intr_mode != OCE_INTERRUPT_MODE_INTX) {
2899 di.msix.num_vectors = sc->intr_count + sc->roce_intr_count;
2900 di.msix.start_vector = sc->intr_count;
2901 for (i=0; i<di.msix.num_vectors; i++) {
2902 di.msix.vector_list[i] = sc->intrs[i].vector;
2903 }
2904 } else {
2905 }
2906 memcpy(di.mac_addr, sc->macaddr.mac_addr, 6);
2907 di.vendor_id = pci_get_vendor(sc->dev);
2908 di.dev_id = pci_get_device(sc->dev);
2909
2910 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
2911 di.flags |= OCE_RDMA_INFO_RDMA_SUPPORTED;
2912 }
2913
2914 rdma_if->announce(&di);
2915 sc = sc->next;
2916 }
2917 }
2918
2919 return 0;
2920 }
2921
2922 static void
oce_read_env_variables(POCE_SOFTC sc)2923 oce_read_env_variables( POCE_SOFTC sc )
2924 {
2925 char *value = NULL;
2926 int rc = 0;
2927
2928 /* read if user wants to enable hwlro or swlro */
2929 //value = getenv("oce_enable_hwlro");
2930 if(value && IS_SH(sc)) {
2931 sc->enable_hwlro = strtol(value, NULL, 10);
2932 if(sc->enable_hwlro) {
2933 rc = oce_mbox_nic_query_lro_capabilities(sc, NULL, NULL);
2934 if(rc) {
2935 device_printf(sc->dev, "no hardware lro support\n");
2936 device_printf(sc->dev, "software lro enabled\n");
2937 sc->enable_hwlro = 0;
2938 }else {
2939 device_printf(sc->dev, "hardware lro enabled\n");
2940 oce_max_rsp_handled = 32;
2941 }
2942 }else {
2943 device_printf(sc->dev, "software lro enabled\n");
2944 }
2945 }else {
2946 sc->enable_hwlro = 0;
2947 }
2948
2949 /* read mbuf size */
2950 //value = getenv("oce_rq_buf_size");
2951 if(value && IS_SH(sc)) {
2952 oce_rq_buf_size = strtol(value, NULL, 10);
2953 switch(oce_rq_buf_size) {
2954 case 2048:
2955 case 4096:
2956 case 9216:
2957 case 16384:
2958 break;
2959
2960 default:
2961 device_printf(sc->dev, " Supported oce_rq_buf_size values are 2K, 4K, 9K, 16K \n");
2962 oce_rq_buf_size = 2048;
2963 }
2964 }
2965
2966 return;
2967 }
2968