1 /*-
2 * Copyright (C) 2013 Emulex
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 *
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * 3. Neither the name of the Emulex Corporation nor the names of its
16 * contributors may be used to endorse or promote products derived from
17 * this software without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 *
31 * Contact Information:
32 * freebsd-drivers@emulex.com
33 *
34 * Emulex
35 * 3333 Susan Street
36 * Costa Mesa, CA 92626
37 */
38
39 /* $FreeBSD$ */
40
41 #include "opt_inet6.h"
42 #include "opt_inet.h"
43
44 #include "oce_if.h"
45 #include "oce_user.h"
46
47 #define is_tso_pkt(m) (m->m_pkthdr.csum_flags & CSUM_TSO)
48
49 /* UE Status Low CSR */
50 static char *ue_status_low_desc[] = {
51 "CEV",
52 "CTX",
53 "DBUF",
54 "ERX",
55 "Host",
56 "MPU",
57 "NDMA",
58 "PTC ",
59 "RDMA ",
60 "RXF ",
61 "RXIPS ",
62 "RXULP0 ",
63 "RXULP1 ",
64 "RXULP2 ",
65 "TIM ",
66 "TPOST ",
67 "TPRE ",
68 "TXIPS ",
69 "TXULP0 ",
70 "TXULP1 ",
71 "UC ",
72 "WDMA ",
73 "TXULP2 ",
74 "HOST1 ",
75 "P0_OB_LINK ",
76 "P1_OB_LINK ",
77 "HOST_GPIO ",
78 "MBOX ",
79 "AXGMAC0",
80 "AXGMAC1",
81 "JTAG",
82 "MPU_INTPEND"
83 };
84
85 /* UE Status High CSR */
86 static char *ue_status_hi_desc[] = {
87 "LPCMEMHOST",
88 "MGMT_MAC",
89 "PCS0ONLINE",
90 "MPU_IRAM",
91 "PCS1ONLINE",
92 "PCTL0",
93 "PCTL1",
94 "PMEM",
95 "RR",
96 "TXPB",
97 "RXPP",
98 "XAUI",
99 "TXP",
100 "ARM",
101 "IPC",
102 "HOST2",
103 "HOST3",
104 "HOST4",
105 "HOST5",
106 "HOST6",
107 "HOST7",
108 "HOST8",
109 "HOST9",
110 "NETC",
111 "Unknown",
112 "Unknown",
113 "Unknown",
114 "Unknown",
115 "Unknown",
116 "Unknown",
117 "Unknown",
118 "Unknown"
119 };
120
121 struct oce_common_cqe_info{
122 uint8_t vtp:1;
123 uint8_t l4_cksum_pass:1;
124 uint8_t ip_cksum_pass:1;
125 uint8_t ipv6_frame:1;
126 uint8_t qnq:1;
127 uint8_t rsvd:3;
128 uint8_t num_frags;
129 uint16_t pkt_size;
130 uint16_t vtag;
131 };
132
133
134 /* Driver entry points prototypes */
135 static int oce_probe(device_t dev);
136 static int oce_attach(device_t dev);
137 static int oce_detach(device_t dev);
138 static int oce_shutdown(device_t dev);
139 static int oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data);
140 static void oce_init(void *xsc);
141 static int oce_multiq_start(struct ifnet *ifp, struct mbuf *m);
142 static void oce_multiq_flush(struct ifnet *ifp);
143
144 /* Driver interrupt routines protypes */
145 static void oce_intr(void *arg, int pending);
146 static int oce_setup_intr(POCE_SOFTC sc);
147 static int oce_fast_isr(void *arg);
148 static int oce_alloc_intr(POCE_SOFTC sc, int vector,
149 void (*isr) (void *arg, int pending));
150
151 /* Media callbacks prototypes */
152 static void oce_media_status(struct ifnet *ifp, struct ifmediareq *req);
153 static int oce_media_change(struct ifnet *ifp);
154
155 /* Transmit routines prototypes */
156 static int oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index);
157 static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq);
158 static void oce_process_tx_completion(struct oce_wq *wq);
159 static int oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m,
160 struct oce_wq *wq);
161
162 /* Receive routines prototypes */
163 static int oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe);
164 static int oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe);
165 static void oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe);
166 static void oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq);
167 static uint16_t oce_rq_handler_lro(void *arg);
168 static void oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2);
169 static void oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2);
170 static void oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m);
171
172 /* Helper function prototypes in this file */
173 static int oce_attach_ifp(POCE_SOFTC sc);
174 static void oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag);
175 static void oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag);
176 static int oce_vid_config(POCE_SOFTC sc);
177 static void oce_mac_addr_set(POCE_SOFTC sc);
178 static int oce_handle_passthrough(struct ifnet *ifp, caddr_t data);
179 static void oce_local_timer(void *arg);
180 static void oce_if_deactivate(POCE_SOFTC sc);
181 static void oce_if_activate(POCE_SOFTC sc);
182 static void setup_max_queues_want(POCE_SOFTC sc);
183 static void update_queues_got(POCE_SOFTC sc);
184 static void process_link_state(POCE_SOFTC sc,
185 struct oce_async_cqe_link_state *acqe);
186 static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m);
187 static void oce_get_config(POCE_SOFTC sc);
188 static struct mbuf *oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete);
189 static void oce_read_env_variables(POCE_SOFTC sc);
190
191
192 /* IP specific */
193 #if defined(INET6) || defined(INET)
194 static int oce_init_lro(POCE_SOFTC sc);
195 static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp);
196 #endif
197
198 static device_method_t oce_dispatch[] = {
199 DEVMETHOD(device_probe, oce_probe),
200 DEVMETHOD(device_attach, oce_attach),
201 DEVMETHOD(device_detach, oce_detach),
202 DEVMETHOD(device_shutdown, oce_shutdown),
203
204 DEVMETHOD_END
205 };
206
207 static driver_t oce_driver = {
208 "oce",
209 oce_dispatch,
210 sizeof(OCE_SOFTC)
211 };
212 static devclass_t oce_devclass;
213
214
215 DRIVER_MODULE(oce, pci, oce_driver, oce_devclass, 0, 0);
216 MODULE_DEPEND(oce, pci, 1, 1, 1);
217 MODULE_DEPEND(oce, ether, 1, 1, 1);
218 MODULE_VERSION(oce, 1);
219
220
221 /* global vars */
222 const char component_revision[32] = {"///" COMPONENT_REVISION "///"};
223
224 /* Module capabilites and parameters */
225 uint32_t oce_max_rsp_handled = OCE_MAX_RSP_HANDLED;
226 uint32_t oce_enable_rss = OCE_MODCAP_RSS;
227 uint32_t oce_rq_buf_size = 2048;
228
229 TUNABLE_INT("hw.oce.max_rsp_handled", &oce_max_rsp_handled);
230 TUNABLE_INT("hw.oce.enable_rss", &oce_enable_rss);
231
232
233 /* Supported devices table */
234 static uint32_t supportedDevices[] = {
235 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE2,
236 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE3,
237 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_BE3,
238 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201,
239 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201_VF,
240 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_SH
241 };
242
243 POCE_SOFTC softc_head = NULL;
244 POCE_SOFTC softc_tail = NULL;
245
246 struct oce_rdma_if *oce_rdma_if = NULL;
247
248 /*****************************************************************************
249 * Driver entry points functions *
250 *****************************************************************************/
251
252 static int
oce_probe(device_t dev)253 oce_probe(device_t dev)
254 {
255 uint16_t vendor = 0;
256 uint16_t device = 0;
257 int i = 0;
258 char str[256] = {0};
259 POCE_SOFTC sc;
260
261 sc = device_get_softc(dev);
262 bzero(sc, sizeof(OCE_SOFTC));
263 sc->dev = dev;
264
265 vendor = pci_get_vendor(dev);
266 device = pci_get_device(dev);
267
268 for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint32_t)); i++) {
269 if (vendor == ((supportedDevices[i] >> 16) & 0xffff)) {
270 if (device == (supportedDevices[i] & 0xffff)) {
271 sprintf(str, "%s:%s", "Emulex CNA NIC function",
272 component_revision);
273 device_set_desc_copy(dev, str);
274
275 switch (device) {
276 case PCI_PRODUCT_BE2:
277 sc->flags |= OCE_FLAGS_BE2;
278 break;
279 case PCI_PRODUCT_BE3:
280 sc->flags |= OCE_FLAGS_BE3;
281 break;
282 case PCI_PRODUCT_XE201:
283 case PCI_PRODUCT_XE201_VF:
284 sc->flags |= OCE_FLAGS_XE201;
285 break;
286 case PCI_PRODUCT_SH:
287 sc->flags |= OCE_FLAGS_SH;
288 break;
289 default:
290 return ENXIO;
291 }
292 return BUS_PROBE_DEFAULT;
293 }
294 }
295 }
296
297 return ENXIO;
298 }
299
300
301 static int
oce_attach(device_t dev)302 oce_attach(device_t dev)
303 {
304 POCE_SOFTC sc;
305 int rc = 0;
306
307 sc = device_get_softc(dev);
308
309 rc = oce_hw_pci_alloc(sc);
310 if (rc)
311 return rc;
312
313 sc->tx_ring_size = OCE_TX_RING_SIZE;
314 sc->rx_ring_size = OCE_RX_RING_SIZE;
315 /* receive fragment size should be multiple of 2K */
316 sc->rq_frag_size = ((oce_rq_buf_size / 2048) * 2048);
317 sc->flow_control = OCE_DEFAULT_FLOW_CONTROL;
318 sc->promisc = OCE_DEFAULT_PROMISCUOUS;
319
320 LOCK_CREATE(&sc->bmbx_lock, "Mailbox_lock");
321 LOCK_CREATE(&sc->dev_lock, "Device_lock");
322
323 /* initialise the hardware */
324 rc = oce_hw_init(sc);
325 if (rc)
326 goto pci_res_free;
327
328 oce_read_env_variables(sc);
329
330 oce_get_config(sc);
331
332 setup_max_queues_want(sc);
333
334 rc = oce_setup_intr(sc);
335 if (rc)
336 goto mbox_free;
337
338 rc = oce_queue_init_all(sc);
339 if (rc)
340 goto intr_free;
341
342 rc = oce_attach_ifp(sc);
343 if (rc)
344 goto queues_free;
345
346 #if defined(INET6) || defined(INET)
347 rc = oce_init_lro(sc);
348 if (rc)
349 goto ifp_free;
350 #endif
351
352 rc = oce_hw_start(sc);
353 if (rc)
354 goto lro_free;
355
356 sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
357 oce_add_vlan, sc, EVENTHANDLER_PRI_FIRST);
358 sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
359 oce_del_vlan, sc, EVENTHANDLER_PRI_FIRST);
360
361 rc = oce_stats_init(sc);
362 if (rc)
363 goto vlan_free;
364
365 oce_add_sysctls(sc);
366
367 callout_init(&sc->timer, CALLOUT_MPSAFE);
368 rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc);
369 if (rc)
370 goto stats_free;
371
372 sc->next =NULL;
373 if (softc_tail != NULL) {
374 softc_tail->next = sc;
375 } else {
376 softc_head = sc;
377 }
378 softc_tail = sc;
379
380 return 0;
381
382 stats_free:
383 callout_drain(&sc->timer);
384 oce_stats_free(sc);
385 vlan_free:
386 if (sc->vlan_attach)
387 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach);
388 if (sc->vlan_detach)
389 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach);
390 oce_hw_intr_disable(sc);
391 lro_free:
392 #if defined(INET6) || defined(INET)
393 oce_free_lro(sc);
394 ifp_free:
395 #endif
396 ether_ifdetach(sc->ifp);
397 if_free(sc->ifp);
398 queues_free:
399 oce_queue_release_all(sc);
400 intr_free:
401 oce_intr_free(sc);
402 mbox_free:
403 oce_dma_free(sc, &sc->bsmbx);
404 pci_res_free:
405 oce_hw_pci_free(sc);
406 LOCK_DESTROY(&sc->dev_lock);
407 LOCK_DESTROY(&sc->bmbx_lock);
408 return rc;
409
410 }
411
412
413 static int
oce_detach(device_t dev)414 oce_detach(device_t dev)
415 {
416 POCE_SOFTC sc = device_get_softc(dev);
417 POCE_SOFTC poce_sc_tmp, *ppoce_sc_tmp1, poce_sc_tmp2 = NULL;
418
419 poce_sc_tmp = softc_head;
420 ppoce_sc_tmp1 = &softc_head;
421 while (poce_sc_tmp != NULL) {
422 if (poce_sc_tmp == sc) {
423 *ppoce_sc_tmp1 = sc->next;
424 if (sc->next == NULL) {
425 softc_tail = poce_sc_tmp2;
426 }
427 break;
428 }
429 poce_sc_tmp2 = poce_sc_tmp;
430 ppoce_sc_tmp1 = &poce_sc_tmp->next;
431 poce_sc_tmp = poce_sc_tmp->next;
432 }
433
434 LOCK(&sc->dev_lock);
435 oce_if_deactivate(sc);
436 UNLOCK(&sc->dev_lock);
437
438 callout_drain(&sc->timer);
439
440 if (sc->vlan_attach != NULL)
441 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach);
442 if (sc->vlan_detach != NULL)
443 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach);
444
445 ether_ifdetach(sc->ifp);
446
447 if_free(sc->ifp);
448
449 oce_hw_shutdown(sc);
450
451 bus_generic_detach(dev);
452
453 return 0;
454 }
455
456
457 static int
oce_shutdown(device_t dev)458 oce_shutdown(device_t dev)
459 {
460 int rc;
461
462 rc = oce_detach(dev);
463
464 return rc;
465 }
466
467
468 static int
oce_ioctl(struct ifnet * ifp,u_long command,caddr_t data)469 oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
470 {
471 struct ifreq *ifr = (struct ifreq *)data;
472 POCE_SOFTC sc = ifp->if_softc;
473 struct ifi2creq i2c;
474 uint8_t offset = 0;
475 int rc = 0;
476 uint32_t u;
477
478 switch (command) {
479
480 case SIOCGIFMEDIA:
481 rc = ifmedia_ioctl(ifp, ifr, &sc->media, command);
482 break;
483
484 case SIOCSIFMTU:
485 if (ifr->ifr_mtu > OCE_MAX_MTU)
486 rc = EINVAL;
487 else
488 ifp->if_mtu = ifr->ifr_mtu;
489 break;
490
491 case SIOCSIFFLAGS:
492 if (ifp->if_flags & IFF_UP) {
493 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
494 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING;
495 oce_init(sc);
496 }
497 device_printf(sc->dev, "Interface Up\n");
498 } else {
499 LOCK(&sc->dev_lock);
500
501 sc->ifp->if_drv_flags &=
502 ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
503 oce_if_deactivate(sc);
504
505 UNLOCK(&sc->dev_lock);
506
507 device_printf(sc->dev, "Interface Down\n");
508 }
509
510 if ((ifp->if_flags & IFF_PROMISC) && !sc->promisc) {
511 if (!oce_rxf_set_promiscuous(sc, (1 | (1 << 1))))
512 sc->promisc = TRUE;
513 } else if (!(ifp->if_flags & IFF_PROMISC) && sc->promisc) {
514 if (!oce_rxf_set_promiscuous(sc, 0))
515 sc->promisc = FALSE;
516 }
517
518 break;
519
520 case SIOCADDMULTI:
521 case SIOCDELMULTI:
522 rc = oce_hw_update_multicast(sc);
523 if (rc)
524 device_printf(sc->dev,
525 "Update multicast address failed\n");
526 break;
527
528 case SIOCSIFCAP:
529 u = ifr->ifr_reqcap ^ ifp->if_capenable;
530
531 if (u & IFCAP_TXCSUM) {
532 ifp->if_capenable ^= IFCAP_TXCSUM;
533 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
534
535 if (IFCAP_TSO & ifp->if_capenable &&
536 !(IFCAP_TXCSUM & ifp->if_capenable)) {
537 u &= ~IFCAP_TSO;
538 ifp->if_capenable &= ~IFCAP_TSO;
539 ifp->if_hwassist &= ~CSUM_TSO;
540 if_printf(ifp,
541 "TSO disabled due to -txcsum.\n");
542 }
543 }
544
545 if (u & IFCAP_RXCSUM)
546 ifp->if_capenable ^= IFCAP_RXCSUM;
547
548 if (u & IFCAP_TSO4) {
549 ifp->if_capenable ^= IFCAP_TSO4;
550
551 if (IFCAP_TSO & ifp->if_capenable) {
552 if (IFCAP_TXCSUM & ifp->if_capenable)
553 ifp->if_hwassist |= CSUM_TSO;
554 else {
555 ifp->if_capenable &= ~IFCAP_TSO;
556 ifp->if_hwassist &= ~CSUM_TSO;
557 if_printf(ifp,
558 "Enable txcsum first.\n");
559 rc = EAGAIN;
560 }
561 } else
562 ifp->if_hwassist &= ~CSUM_TSO;
563 }
564
565 if (u & IFCAP_VLAN_HWTAGGING)
566 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
567
568 if (u & IFCAP_VLAN_HWFILTER) {
569 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
570 oce_vid_config(sc);
571 }
572 #if defined(INET6) || defined(INET)
573 if (u & IFCAP_LRO) {
574 ifp->if_capenable ^= IFCAP_LRO;
575 if(sc->enable_hwlro) {
576 if(ifp->if_capenable & IFCAP_LRO) {
577 rc = oce_mbox_nic_set_iface_lro_config(sc, 1);
578 }else {
579 rc = oce_mbox_nic_set_iface_lro_config(sc, 0);
580 }
581 }
582 }
583 #endif
584
585 break;
586
587 case SIOCGI2C:
588 rc = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
589 if (rc)
590 break;
591
592 if (i2c.dev_addr != PAGE_NUM_A0 &&
593 i2c.dev_addr != PAGE_NUM_A2) {
594 rc = EINVAL;
595 break;
596 }
597
598 if (i2c.len > sizeof(i2c.data)) {
599 rc = EINVAL;
600 break;
601 }
602
603 rc = oce_mbox_read_transrecv_data(sc, i2c.dev_addr);
604 if(rc) {
605 rc = -rc;
606 break;
607 }
608
609 if (i2c.dev_addr == PAGE_NUM_A0)
610 offset = i2c.offset;
611 else
612 offset = TRANSCEIVER_A0_SIZE + i2c.offset;
613
614 memcpy(&i2c.data[0], &sfp_vpd_dump_buffer[offset], i2c.len);
615
616 rc = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c));
617 break;
618
619 case SIOCGPRIVATE_0:
620 rc = priv_check(curthread, PRIV_DRIVER);
621 if (rc != 0)
622 break;
623 rc = oce_handle_passthrough(ifp, data);
624 break;
625 default:
626 rc = ether_ioctl(ifp, command, data);
627 break;
628 }
629
630 return rc;
631 }
632
633
634 static void
oce_init(void * arg)635 oce_init(void *arg)
636 {
637 POCE_SOFTC sc = arg;
638
639 LOCK(&sc->dev_lock);
640
641 if (sc->ifp->if_flags & IFF_UP) {
642 oce_if_deactivate(sc);
643 oce_if_activate(sc);
644 }
645
646 UNLOCK(&sc->dev_lock);
647
648 }
649
650
651 static int
oce_multiq_start(struct ifnet * ifp,struct mbuf * m)652 oce_multiq_start(struct ifnet *ifp, struct mbuf *m)
653 {
654 POCE_SOFTC sc = ifp->if_softc;
655 struct oce_wq *wq = NULL;
656 int queue_index = 0;
657 int status = 0;
658
659 if (!sc->link_status)
660 return ENXIO;
661
662 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
663 queue_index = m->m_pkthdr.flowid % sc->nwqs;
664
665 wq = sc->wq[queue_index];
666
667 LOCK(&wq->tx_lock);
668 status = oce_multiq_transmit(ifp, m, wq);
669 UNLOCK(&wq->tx_lock);
670
671 return status;
672
673 }
674
675
676 static void
oce_multiq_flush(struct ifnet * ifp)677 oce_multiq_flush(struct ifnet *ifp)
678 {
679 POCE_SOFTC sc = ifp->if_softc;
680 struct mbuf *m;
681 int i = 0;
682
683 for (i = 0; i < sc->nwqs; i++) {
684 while ((m = buf_ring_dequeue_sc(sc->wq[i]->br)) != NULL)
685 m_freem(m);
686 }
687 if_qflush(ifp);
688 }
689
690
691
692 /*****************************************************************************
693 * Driver interrupt routines functions *
694 *****************************************************************************/
695
696 static void
oce_intr(void * arg,int pending)697 oce_intr(void *arg, int pending)
698 {
699
700 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg;
701 POCE_SOFTC sc = ii->sc;
702 struct oce_eq *eq = ii->eq;
703 struct oce_eqe *eqe;
704 struct oce_cq *cq = NULL;
705 int i, num_eqes = 0;
706
707
708 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map,
709 BUS_DMASYNC_POSTWRITE);
710 do {
711 eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe);
712 if (eqe->evnt == 0)
713 break;
714 eqe->evnt = 0;
715 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map,
716 BUS_DMASYNC_POSTWRITE);
717 RING_GET(eq->ring, 1);
718 num_eqes++;
719
720 } while (TRUE);
721
722 if (!num_eqes)
723 goto eq_arm; /* Spurious */
724
725 /* Clear EQ entries, but dont arm */
726 oce_arm_eq(sc, eq->eq_id, num_eqes, FALSE, FALSE);
727
728 /* Process TX, RX and MCC. But dont arm CQ*/
729 for (i = 0; i < eq->cq_valid; i++) {
730 cq = eq->cq[i];
731 (*cq->cq_handler)(cq->cb_arg);
732 }
733
734 /* Arm all cqs connected to this EQ */
735 for (i = 0; i < eq->cq_valid; i++) {
736 cq = eq->cq[i];
737 oce_arm_cq(sc, cq->cq_id, 0, TRUE);
738 }
739
740 eq_arm:
741 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE);
742
743 return;
744 }
745
746
747 static int
oce_setup_intr(POCE_SOFTC sc)748 oce_setup_intr(POCE_SOFTC sc)
749 {
750 int rc = 0, use_intx = 0;
751 int vector = 0, req_vectors = 0;
752 int tot_req_vectors, tot_vectors;
753
754 if (is_rss_enabled(sc))
755 req_vectors = MAX((sc->nrqs - 1), sc->nwqs);
756 else
757 req_vectors = 1;
758
759 tot_req_vectors = req_vectors;
760 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
761 if (req_vectors > 1) {
762 tot_req_vectors += OCE_RDMA_VECTORS;
763 sc->roce_intr_count = OCE_RDMA_VECTORS;
764 }
765 }
766
767 if (sc->flags & OCE_FLAGS_MSIX_CAPABLE) {
768 sc->intr_count = req_vectors;
769 tot_vectors = tot_req_vectors;
770 rc = pci_alloc_msix(sc->dev, &tot_vectors);
771 if (rc != 0) {
772 use_intx = 1;
773 pci_release_msi(sc->dev);
774 } else {
775 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
776 if (tot_vectors < tot_req_vectors) {
777 if (sc->intr_count < (2 * OCE_RDMA_VECTORS)) {
778 sc->roce_intr_count = (tot_vectors / 2);
779 }
780 sc->intr_count = tot_vectors - sc->roce_intr_count;
781 }
782 } else {
783 sc->intr_count = tot_vectors;
784 }
785 sc->flags |= OCE_FLAGS_USING_MSIX;
786 }
787 } else
788 use_intx = 1;
789
790 if (use_intx)
791 sc->intr_count = 1;
792
793 /* Scale number of queues based on intr we got */
794 update_queues_got(sc);
795
796 if (use_intx) {
797 device_printf(sc->dev, "Using legacy interrupt\n");
798 rc = oce_alloc_intr(sc, vector, oce_intr);
799 if (rc)
800 goto error;
801 } else {
802 for (; vector < sc->intr_count; vector++) {
803 rc = oce_alloc_intr(sc, vector, oce_intr);
804 if (rc)
805 goto error;
806 }
807 }
808
809 return 0;
810 error:
811 oce_intr_free(sc);
812 return rc;
813 }
814
815
816 static int
oce_fast_isr(void * arg)817 oce_fast_isr(void *arg)
818 {
819 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg;
820 POCE_SOFTC sc = ii->sc;
821
822 if (ii->eq == NULL)
823 return FILTER_STRAY;
824
825 oce_arm_eq(sc, ii->eq->eq_id, 0, FALSE, TRUE);
826
827 taskqueue_enqueue(ii->tq, &ii->task);
828
829 ii->eq->intr++;
830
831 return FILTER_HANDLED;
832 }
833
834
835 static int
oce_alloc_intr(POCE_SOFTC sc,int vector,void (* isr)(void * arg,int pending))836 oce_alloc_intr(POCE_SOFTC sc, int vector, void (*isr) (void *arg, int pending))
837 {
838 POCE_INTR_INFO ii;
839 int rc = 0, rr;
840
841 if (vector >= OCE_MAX_EQ)
842 return (EINVAL);
843
844 ii = &sc->intrs[vector];
845
846 /* Set the resource id for the interrupt.
847 * MSIx is vector + 1 for the resource id,
848 * INTx is 0 for the resource id.
849 */
850 if (sc->flags & OCE_FLAGS_USING_MSIX)
851 rr = vector + 1;
852 else
853 rr = 0;
854 ii->intr_res = bus_alloc_resource_any(sc->dev,
855 SYS_RES_IRQ,
856 &rr, RF_ACTIVE|RF_SHAREABLE);
857 ii->irq_rr = rr;
858 if (ii->intr_res == NULL) {
859 device_printf(sc->dev,
860 "Could not allocate interrupt\n");
861 rc = ENXIO;
862 return rc;
863 }
864
865 TASK_INIT(&ii->task, 0, isr, ii);
866 ii->vector = vector;
867 sprintf(ii->task_name, "oce_task[%d]", ii->vector);
868 ii->tq = taskqueue_create_fast(ii->task_name,
869 M_NOWAIT,
870 taskqueue_thread_enqueue,
871 &ii->tq);
872 taskqueue_start_threads(&ii->tq, 1, PI_NET, "%s taskq",
873 device_get_nameunit(sc->dev));
874
875 ii->sc = sc;
876 rc = bus_setup_intr(sc->dev,
877 ii->intr_res,
878 INTR_TYPE_NET,
879 oce_fast_isr, NULL, ii, &ii->tag);
880 return rc;
881
882 }
883
884
885 void
oce_intr_free(POCE_SOFTC sc)886 oce_intr_free(POCE_SOFTC sc)
887 {
888 int i = 0;
889
890 for (i = 0; i < sc->intr_count; i++) {
891
892 if (sc->intrs[i].tag != NULL)
893 bus_teardown_intr(sc->dev, sc->intrs[i].intr_res,
894 sc->intrs[i].tag);
895 if (sc->intrs[i].tq != NULL)
896 taskqueue_free(sc->intrs[i].tq);
897
898 if (sc->intrs[i].intr_res != NULL)
899 bus_release_resource(sc->dev, SYS_RES_IRQ,
900 sc->intrs[i].irq_rr,
901 sc->intrs[i].intr_res);
902 sc->intrs[i].tag = NULL;
903 sc->intrs[i].intr_res = NULL;
904 }
905
906 if (sc->flags & OCE_FLAGS_USING_MSIX)
907 pci_release_msi(sc->dev);
908
909 }
910
911
912
913 /******************************************************************************
914 * Media callbacks functions *
915 ******************************************************************************/
916
917 static void
oce_media_status(struct ifnet * ifp,struct ifmediareq * req)918 oce_media_status(struct ifnet *ifp, struct ifmediareq *req)
919 {
920 POCE_SOFTC sc = (POCE_SOFTC) ifp->if_softc;
921
922
923 req->ifm_status = IFM_AVALID;
924 req->ifm_active = IFM_ETHER;
925
926 if (sc->link_status == 1)
927 req->ifm_status |= IFM_ACTIVE;
928 else
929 return;
930
931 switch (sc->link_speed) {
932 case 1: /* 10 Mbps */
933 req->ifm_active |= IFM_10_T | IFM_FDX;
934 sc->speed = 10;
935 break;
936 case 2: /* 100 Mbps */
937 req->ifm_active |= IFM_100_TX | IFM_FDX;
938 sc->speed = 100;
939 break;
940 case 3: /* 1 Gbps */
941 req->ifm_active |= IFM_1000_T | IFM_FDX;
942 sc->speed = 1000;
943 break;
944 case 4: /* 10 Gbps */
945 req->ifm_active |= IFM_10G_SR | IFM_FDX;
946 sc->speed = 10000;
947 break;
948 case 5: /* 20 Gbps */
949 req->ifm_active |= IFM_10G_SR | IFM_FDX;
950 sc->speed = 20000;
951 break;
952 case 6: /* 25 Gbps */
953 req->ifm_active |= IFM_10G_SR | IFM_FDX;
954 sc->speed = 25000;
955 break;
956 case 7: /* 40 Gbps */
957 req->ifm_active |= IFM_40G_SR4 | IFM_FDX;
958 sc->speed = 40000;
959 break;
960 default:
961 sc->speed = 0;
962 break;
963 }
964
965 return;
966 }
967
968
969 int
oce_media_change(struct ifnet * ifp)970 oce_media_change(struct ifnet *ifp)
971 {
972 return 0;
973 }
974
975
oce_is_pkt_dest_bmc(POCE_SOFTC sc,struct mbuf * m,boolean_t * os2bmc,struct mbuf ** m_new)976 static void oce_is_pkt_dest_bmc(POCE_SOFTC sc,
977 struct mbuf *m, boolean_t *os2bmc,
978 struct mbuf **m_new)
979 {
980 struct ether_header *eh = NULL;
981
982 eh = mtod(m, struct ether_header *);
983
984 if (!is_os2bmc_enabled(sc) || *os2bmc) {
985 *os2bmc = FALSE;
986 goto done;
987 }
988 if (!ETHER_IS_MULTICAST(eh->ether_dhost))
989 goto done;
990
991 if (is_mc_allowed_on_bmc(sc, eh) ||
992 is_bc_allowed_on_bmc(sc, eh) ||
993 is_arp_allowed_on_bmc(sc, ntohs(eh->ether_type))) {
994 *os2bmc = TRUE;
995 goto done;
996 }
997
998 if (mtod(m, struct ip *)->ip_p == IPPROTO_IPV6) {
999 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
1000 uint8_t nexthdr = ip6->ip6_nxt;
1001 if (nexthdr == IPPROTO_ICMPV6) {
1002 struct icmp6_hdr *icmp6 = (struct icmp6_hdr *)(ip6 + 1);
1003 switch (icmp6->icmp6_type) {
1004 case ND_ROUTER_ADVERT:
1005 *os2bmc = is_ipv6_ra_filt_enabled(sc);
1006 goto done;
1007 case ND_NEIGHBOR_ADVERT:
1008 *os2bmc = is_ipv6_na_filt_enabled(sc);
1009 goto done;
1010 default:
1011 break;
1012 }
1013 }
1014 }
1015
1016 if (mtod(m, struct ip *)->ip_p == IPPROTO_UDP) {
1017 struct ip *ip = mtod(m, struct ip *);
1018 int iphlen = ip->ip_hl << 2;
1019 struct udphdr *uh = (struct udphdr *)((caddr_t)ip + iphlen);
1020 switch (uh->uh_dport) {
1021 case DHCP_CLIENT_PORT:
1022 *os2bmc = is_dhcp_client_filt_enabled(sc);
1023 goto done;
1024 case DHCP_SERVER_PORT:
1025 *os2bmc = is_dhcp_srvr_filt_enabled(sc);
1026 goto done;
1027 case NET_BIOS_PORT1:
1028 case NET_BIOS_PORT2:
1029 *os2bmc = is_nbios_filt_enabled(sc);
1030 goto done;
1031 case DHCPV6_RAS_PORT:
1032 *os2bmc = is_ipv6_ras_filt_enabled(sc);
1033 goto done;
1034 default:
1035 break;
1036 }
1037 }
1038 done:
1039 if (*os2bmc) {
1040 *m_new = m_dup(m, M_NOWAIT);
1041 if (!*m_new) {
1042 *os2bmc = FALSE;
1043 return;
1044 }
1045 *m_new = oce_insert_vlan_tag(sc, *m_new, NULL);
1046 }
1047 }
1048
1049
1050
1051 /*****************************************************************************
1052 * Transmit routines functions *
1053 *****************************************************************************/
1054
1055 static int
oce_tx(POCE_SOFTC sc,struct mbuf ** mpp,int wq_index)1056 oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index)
1057 {
1058 int rc = 0, i, retry_cnt = 0;
1059 bus_dma_segment_t segs[OCE_MAX_TX_ELEMENTS];
1060 struct mbuf *m, *m_temp, *m_new = NULL;
1061 struct oce_wq *wq = sc->wq[wq_index];
1062 struct oce_packet_desc *pd;
1063 struct oce_nic_hdr_wqe *nichdr;
1064 struct oce_nic_frag_wqe *nicfrag;
1065 struct ether_header *eh = NULL;
1066 int num_wqes;
1067 uint32_t reg_value;
1068 boolean_t complete = TRUE;
1069 boolean_t os2bmc = FALSE;
1070
1071 m = *mpp;
1072 if (!m)
1073 return EINVAL;
1074
1075 if (!(m->m_flags & M_PKTHDR)) {
1076 rc = ENXIO;
1077 goto free_ret;
1078 }
1079
1080 /* Don't allow non-TSO packets longer than MTU */
1081 if (!is_tso_pkt(m)) {
1082 eh = mtod(m, struct ether_header *);
1083 if(m->m_pkthdr.len > ETHER_MAX_FRAME(sc->ifp, eh->ether_type, FALSE))
1084 goto free_ret;
1085 }
1086
1087 if(oce_tx_asic_stall_verify(sc, m)) {
1088 m = oce_insert_vlan_tag(sc, m, &complete);
1089 if(!m) {
1090 device_printf(sc->dev, "Insertion unsuccessful\n");
1091 return 0;
1092 }
1093
1094 }
1095
1096 /* Lancer, SH ASIC has a bug wherein Packets that are 32 bytes or less
1097 * may cause a transmit stall on that port. So the work-around is to
1098 * pad short packets (<= 32 bytes) to a 36-byte length.
1099 */
1100 if(IS_SH(sc) || IS_XE201(sc) ) {
1101 if(m->m_pkthdr.len <= 32) {
1102 char buf[36];
1103 bzero((void *)buf, 36);
1104 m_append(m, (36 - m->m_pkthdr.len), buf);
1105 }
1106 }
1107
1108 tx_start:
1109 if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1110 /* consolidate packet buffers for TSO/LSO segment offload */
1111 #if defined(INET6) || defined(INET)
1112 m = oce_tso_setup(sc, mpp);
1113 #else
1114 m = NULL;
1115 #endif
1116 if (m == NULL) {
1117 rc = ENXIO;
1118 goto free_ret;
1119 }
1120 }
1121
1122
1123 pd = &wq->pckts[wq->pkt_desc_head];
1124
1125 retry:
1126 rc = bus_dmamap_load_mbuf_sg(wq->tag,
1127 pd->map,
1128 m, segs, &pd->nsegs, BUS_DMA_NOWAIT);
1129 if (rc == 0) {
1130 num_wqes = pd->nsegs + 1;
1131 if (IS_BE(sc) || IS_SH(sc)) {
1132 /*Dummy required only for BE3.*/
1133 if (num_wqes & 1)
1134 num_wqes++;
1135 }
1136 if (num_wqes >= RING_NUM_FREE(wq->ring)) {
1137 bus_dmamap_unload(wq->tag, pd->map);
1138 return EBUSY;
1139 }
1140 atomic_store_rel_int(&wq->pkt_desc_head,
1141 (wq->pkt_desc_head + 1) % \
1142 OCE_WQ_PACKET_ARRAY_SIZE);
1143 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_PREWRITE);
1144 pd->mbuf = m;
1145
1146 nichdr =
1147 RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_hdr_wqe);
1148 nichdr->u0.dw[0] = 0;
1149 nichdr->u0.dw[1] = 0;
1150 nichdr->u0.dw[2] = 0;
1151 nichdr->u0.dw[3] = 0;
1152
1153 nichdr->u0.s.complete = complete;
1154 nichdr->u0.s.mgmt = os2bmc;
1155 nichdr->u0.s.event = 1;
1156 nichdr->u0.s.crc = 1;
1157 nichdr->u0.s.forward = 0;
1158 nichdr->u0.s.ipcs = (m->m_pkthdr.csum_flags & CSUM_IP) ? 1 : 0;
1159 nichdr->u0.s.udpcs =
1160 (m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0;
1161 nichdr->u0.s.tcpcs =
1162 (m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0;
1163 nichdr->u0.s.num_wqe = num_wqes;
1164 nichdr->u0.s.total_length = m->m_pkthdr.len;
1165
1166 if (m->m_flags & M_VLANTAG) {
1167 nichdr->u0.s.vlan = 1; /*Vlan present*/
1168 nichdr->u0.s.vlan_tag = m->m_pkthdr.ether_vtag;
1169 }
1170
1171 if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1172 if (m->m_pkthdr.tso_segsz) {
1173 nichdr->u0.s.lso = 1;
1174 nichdr->u0.s.lso_mss = m->m_pkthdr.tso_segsz;
1175 }
1176 if (!IS_BE(sc) || !IS_SH(sc))
1177 nichdr->u0.s.ipcs = 1;
1178 }
1179
1180 RING_PUT(wq->ring, 1);
1181 atomic_add_int(&wq->ring->num_used, 1);
1182
1183 for (i = 0; i < pd->nsegs; i++) {
1184 nicfrag =
1185 RING_GET_PRODUCER_ITEM_VA(wq->ring,
1186 struct oce_nic_frag_wqe);
1187 nicfrag->u0.s.rsvd0 = 0;
1188 nicfrag->u0.s.frag_pa_hi = ADDR_HI(segs[i].ds_addr);
1189 nicfrag->u0.s.frag_pa_lo = ADDR_LO(segs[i].ds_addr);
1190 nicfrag->u0.s.frag_len = segs[i].ds_len;
1191 pd->wqe_idx = wq->ring->pidx;
1192 RING_PUT(wq->ring, 1);
1193 atomic_add_int(&wq->ring->num_used, 1);
1194 }
1195 if (num_wqes > (pd->nsegs + 1)) {
1196 nicfrag =
1197 RING_GET_PRODUCER_ITEM_VA(wq->ring,
1198 struct oce_nic_frag_wqe);
1199 nicfrag->u0.dw[0] = 0;
1200 nicfrag->u0.dw[1] = 0;
1201 nicfrag->u0.dw[2] = 0;
1202 nicfrag->u0.dw[3] = 0;
1203 pd->wqe_idx = wq->ring->pidx;
1204 RING_PUT(wq->ring, 1);
1205 atomic_add_int(&wq->ring->num_used, 1);
1206 pd->nsegs++;
1207 }
1208
1209 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1);
1210 wq->tx_stats.tx_reqs++;
1211 wq->tx_stats.tx_wrbs += num_wqes;
1212 wq->tx_stats.tx_bytes += m->m_pkthdr.len;
1213 wq->tx_stats.tx_pkts++;
1214
1215 bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map,
1216 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1217 reg_value = (num_wqes << 16) | wq->wq_id;
1218
1219 /* if os2bmc is not enabled or if the pkt is already tagged as
1220 bmc, do nothing
1221 */
1222 oce_is_pkt_dest_bmc(sc, m, &os2bmc, &m_new);
1223
1224 OCE_WRITE_REG32(sc, db, wq->db_offset, reg_value);
1225
1226 } else if (rc == EFBIG) {
1227 if (retry_cnt == 0) {
1228 m_temp = m_defrag(m, M_NOWAIT);
1229 if (m_temp == NULL)
1230 goto free_ret;
1231 m = m_temp;
1232 *mpp = m_temp;
1233 retry_cnt = retry_cnt + 1;
1234 goto retry;
1235 } else
1236 goto free_ret;
1237 } else if (rc == ENOMEM)
1238 return rc;
1239 else
1240 goto free_ret;
1241
1242 if (os2bmc) {
1243 m = m_new;
1244 goto tx_start;
1245 }
1246
1247 return 0;
1248
1249 free_ret:
1250 m_freem(*mpp);
1251 *mpp = NULL;
1252 return rc;
1253 }
1254
1255
1256 static void
oce_process_tx_completion(struct oce_wq * wq)1257 oce_process_tx_completion(struct oce_wq *wq)
1258 {
1259 struct oce_packet_desc *pd;
1260 POCE_SOFTC sc = (POCE_SOFTC) wq->parent;
1261 struct mbuf *m;
1262
1263 pd = &wq->pckts[wq->pkt_desc_tail];
1264 atomic_store_rel_int(&wq->pkt_desc_tail,
1265 (wq->pkt_desc_tail + 1) % OCE_WQ_PACKET_ARRAY_SIZE);
1266 atomic_subtract_int(&wq->ring->num_used, pd->nsegs + 1);
1267 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1268 bus_dmamap_unload(wq->tag, pd->map);
1269
1270 m = pd->mbuf;
1271 m_freem(m);
1272 pd->mbuf = NULL;
1273
1274
1275 if (sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) {
1276 if (wq->ring->num_used < (wq->ring->num_items / 2)) {
1277 sc->ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE);
1278 oce_tx_restart(sc, wq);
1279 }
1280 }
1281 }
1282
1283
1284 static void
oce_tx_restart(POCE_SOFTC sc,struct oce_wq * wq)1285 oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq)
1286 {
1287
1288 if ((sc->ifp->if_drv_flags & IFF_DRV_RUNNING) != IFF_DRV_RUNNING)
1289 return;
1290
1291 #if __FreeBSD_version >= 800000
1292 if (!drbr_empty(sc->ifp, wq->br))
1293 #else
1294 if (!IFQ_DRV_IS_EMPTY(&sc->ifp->if_snd))
1295 #endif
1296 taskqueue_enqueue(taskqueue_swi, &wq->txtask);
1297
1298 }
1299
1300
1301 #if defined(INET6) || defined(INET)
1302 static struct mbuf *
oce_tso_setup(POCE_SOFTC sc,struct mbuf ** mpp)1303 oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp)
1304 {
1305 struct mbuf *m;
1306 #ifdef INET
1307 struct ip *ip;
1308 #endif
1309 #ifdef INET6
1310 struct ip6_hdr *ip6;
1311 #endif
1312 struct ether_vlan_header *eh;
1313 struct tcphdr *th;
1314 uint16_t etype;
1315 int total_len = 0, ehdrlen = 0;
1316
1317 m = *mpp;
1318
1319 if (M_WRITABLE(m) == 0) {
1320 m = m_dup(*mpp, M_NOWAIT);
1321 if (!m)
1322 return NULL;
1323 m_freem(*mpp);
1324 *mpp = m;
1325 }
1326
1327 eh = mtod(m, struct ether_vlan_header *);
1328 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
1329 etype = ntohs(eh->evl_proto);
1330 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
1331 } else {
1332 etype = ntohs(eh->evl_encap_proto);
1333 ehdrlen = ETHER_HDR_LEN;
1334 }
1335
1336 switch (etype) {
1337 #ifdef INET
1338 case ETHERTYPE_IP:
1339 ip = (struct ip *)(m->m_data + ehdrlen);
1340 if (ip->ip_p != IPPROTO_TCP)
1341 return NULL;
1342 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
1343
1344 total_len = ehdrlen + (ip->ip_hl << 2) + (th->th_off << 2);
1345 break;
1346 #endif
1347 #ifdef INET6
1348 case ETHERTYPE_IPV6:
1349 ip6 = (struct ip6_hdr *)(m->m_data + ehdrlen);
1350 if (ip6->ip6_nxt != IPPROTO_TCP)
1351 return NULL;
1352 th = (struct tcphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr));
1353
1354 total_len = ehdrlen + sizeof(struct ip6_hdr) + (th->th_off << 2);
1355 break;
1356 #endif
1357 default:
1358 return NULL;
1359 }
1360
1361 m = m_pullup(m, total_len);
1362 if (!m)
1363 return NULL;
1364 *mpp = m;
1365 return m;
1366
1367 }
1368 #endif /* INET6 || INET */
1369
1370 void
oce_tx_task(void * arg,int npending)1371 oce_tx_task(void *arg, int npending)
1372 {
1373 struct oce_wq *wq = arg;
1374 POCE_SOFTC sc = wq->parent;
1375 struct ifnet *ifp = sc->ifp;
1376 int rc = 0;
1377
1378 #if __FreeBSD_version >= 800000
1379 LOCK(&wq->tx_lock);
1380 rc = oce_multiq_transmit(ifp, NULL, wq);
1381 if (rc) {
1382 device_printf(sc->dev,
1383 "TX[%d] restart failed\n", wq->queue_index);
1384 }
1385 UNLOCK(&wq->tx_lock);
1386 #else
1387 oce_start(ifp);
1388 #endif
1389
1390 }
1391
1392
1393 void
oce_start(struct ifnet * ifp)1394 oce_start(struct ifnet *ifp)
1395 {
1396 POCE_SOFTC sc = ifp->if_softc;
1397 struct mbuf *m;
1398 int rc = 0;
1399 int def_q = 0; /* Defualt tx queue is 0*/
1400
1401 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
1402 IFF_DRV_RUNNING)
1403 return;
1404
1405 if (!sc->link_status)
1406 return;
1407
1408 do {
1409 IF_DEQUEUE(&sc->ifp->if_snd, m);
1410 if (m == NULL)
1411 break;
1412
1413 LOCK(&sc->wq[def_q]->tx_lock);
1414 rc = oce_tx(sc, &m, def_q);
1415 UNLOCK(&sc->wq[def_q]->tx_lock);
1416 if (rc) {
1417 if (m != NULL) {
1418 sc->wq[def_q]->tx_stats.tx_stops ++;
1419 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1420 IFQ_DRV_PREPEND(&ifp->if_snd, m);
1421 m = NULL;
1422 }
1423 break;
1424 }
1425 if (m != NULL)
1426 ETHER_BPF_MTAP(ifp, m);
1427
1428 } while (TRUE);
1429
1430 return;
1431 }
1432
1433
1434 /* Handle the Completion Queue for transmit */
1435 uint16_t
oce_wq_handler(void * arg)1436 oce_wq_handler(void *arg)
1437 {
1438 struct oce_wq *wq = (struct oce_wq *)arg;
1439 POCE_SOFTC sc = wq->parent;
1440 struct oce_cq *cq = wq->cq;
1441 struct oce_nic_tx_cqe *cqe;
1442 int num_cqes = 0;
1443
1444 LOCK(&wq->tx_compl_lock);
1445 bus_dmamap_sync(cq->ring->dma.tag,
1446 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
1447 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
1448 while (cqe->u0.dw[3]) {
1449 DW_SWAP((uint32_t *) cqe, sizeof(oce_wq_cqe));
1450
1451 wq->ring->cidx = cqe->u0.s.wqe_index + 1;
1452 if (wq->ring->cidx >= wq->ring->num_items)
1453 wq->ring->cidx -= wq->ring->num_items;
1454
1455 oce_process_tx_completion(wq);
1456 wq->tx_stats.tx_compl++;
1457 cqe->u0.dw[3] = 0;
1458 RING_GET(cq->ring, 1);
1459 bus_dmamap_sync(cq->ring->dma.tag,
1460 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
1461 cqe =
1462 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
1463 num_cqes++;
1464 }
1465
1466 if (num_cqes)
1467 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE);
1468
1469 UNLOCK(&wq->tx_compl_lock);
1470 return num_cqes;
1471 }
1472
1473
1474 static int
oce_multiq_transmit(struct ifnet * ifp,struct mbuf * m,struct oce_wq * wq)1475 oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, struct oce_wq *wq)
1476 {
1477 POCE_SOFTC sc = ifp->if_softc;
1478 int status = 0, queue_index = 0;
1479 struct mbuf *next = NULL;
1480 struct buf_ring *br = NULL;
1481
1482 br = wq->br;
1483 queue_index = wq->queue_index;
1484
1485 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
1486 IFF_DRV_RUNNING) {
1487 if (m != NULL)
1488 status = drbr_enqueue(ifp, br, m);
1489 return status;
1490 }
1491
1492 if (m != NULL) {
1493 if ((status = drbr_enqueue(ifp, br, m)) != 0)
1494 return status;
1495 }
1496 while ((next = drbr_peek(ifp, br)) != NULL) {
1497 if (oce_tx(sc, &next, queue_index)) {
1498 if (next == NULL) {
1499 drbr_advance(ifp, br);
1500 } else {
1501 drbr_putback(ifp, br, next);
1502 wq->tx_stats.tx_stops ++;
1503 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1504 }
1505 break;
1506 }
1507 drbr_advance(ifp, br);
1508 if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len);
1509 if (next->m_flags & M_MCAST)
1510 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
1511 ETHER_BPF_MTAP(ifp, next);
1512 }
1513
1514 return 0;
1515 }
1516
1517
1518
1519
1520 /*****************************************************************************
1521 * Receive routines functions *
1522 *****************************************************************************/
1523
1524 static void
oce_correct_header(struct mbuf * m,struct nic_hwlro_cqe_part1 * cqe1,struct nic_hwlro_cqe_part2 * cqe2)1525 oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2)
1526 {
1527 uint32_t *p;
1528 struct ether_header *eh = NULL;
1529 struct tcphdr *tcp_hdr = NULL;
1530 struct ip *ip4_hdr = NULL;
1531 struct ip6_hdr *ip6 = NULL;
1532 uint32_t payload_len = 0;
1533
1534 eh = mtod(m, struct ether_header *);
1535 /* correct IP header */
1536 if(!cqe2->ipv6_frame) {
1537 ip4_hdr = (struct ip *)((char*)eh + sizeof(struct ether_header));
1538 ip4_hdr->ip_ttl = cqe2->frame_lifespan;
1539 ip4_hdr->ip_len = htons(cqe2->coalesced_size - sizeof(struct ether_header));
1540 tcp_hdr = (struct tcphdr *)((char*)ip4_hdr + sizeof(struct ip));
1541 }else {
1542 ip6 = (struct ip6_hdr *)((char*)eh + sizeof(struct ether_header));
1543 ip6->ip6_ctlun.ip6_un1.ip6_un1_hlim = cqe2->frame_lifespan;
1544 payload_len = cqe2->coalesced_size - sizeof(struct ether_header)
1545 - sizeof(struct ip6_hdr);
1546 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = htons(payload_len);
1547 tcp_hdr = (struct tcphdr *)((char*)ip6 + sizeof(struct ip6_hdr));
1548 }
1549
1550 /* correct tcp header */
1551 tcp_hdr->th_ack = htonl(cqe2->tcp_ack_num);
1552 if(cqe2->push) {
1553 tcp_hdr->th_flags |= TH_PUSH;
1554 }
1555 tcp_hdr->th_win = htons(cqe2->tcp_window);
1556 tcp_hdr->th_sum = 0xffff;
1557 if(cqe2->ts_opt) {
1558 p = (uint32_t *)((char*)tcp_hdr + sizeof(struct tcphdr) + 2);
1559 *p = cqe1->tcp_timestamp_val;
1560 *(p+1) = cqe1->tcp_timestamp_ecr;
1561 }
1562
1563 return;
1564 }
1565
1566 static void
oce_rx_mbuf_chain(struct oce_rq * rq,struct oce_common_cqe_info * cqe_info,struct mbuf ** m)1567 oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m)
1568 {
1569 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1570 uint32_t i = 0, frag_len = 0;
1571 uint32_t len = cqe_info->pkt_size;
1572 struct oce_packet_desc *pd;
1573 struct mbuf *tail = NULL;
1574
1575 for (i = 0; i < cqe_info->num_frags; i++) {
1576 if (rq->ring->cidx == rq->ring->pidx) {
1577 device_printf(sc->dev,
1578 "oce_rx_mbuf_chain: Invalid RX completion - Queue is empty\n");
1579 return;
1580 }
1581 pd = &rq->pckts[rq->ring->cidx];
1582
1583 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1584 bus_dmamap_unload(rq->tag, pd->map);
1585 RING_GET(rq->ring, 1);
1586 rq->pending--;
1587
1588 frag_len = (len > rq->cfg.frag_size) ? rq->cfg.frag_size : len;
1589 pd->mbuf->m_len = frag_len;
1590
1591 if (tail != NULL) {
1592 /* additional fragments */
1593 pd->mbuf->m_flags &= ~M_PKTHDR;
1594 tail->m_next = pd->mbuf;
1595 if(rq->islro)
1596 tail->m_nextpkt = NULL;
1597 tail = pd->mbuf;
1598 } else {
1599 /* first fragment, fill out much of the packet header */
1600 pd->mbuf->m_pkthdr.len = len;
1601 if(rq->islro)
1602 pd->mbuf->m_nextpkt = NULL;
1603 pd->mbuf->m_pkthdr.csum_flags = 0;
1604 if (IF_CSUM_ENABLED(sc)) {
1605 if (cqe_info->l4_cksum_pass) {
1606 if(!cqe_info->ipv6_frame) { /* IPV4 */
1607 pd->mbuf->m_pkthdr.csum_flags |=
1608 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1609 }else { /* IPV6 frame */
1610 if(rq->islro) {
1611 pd->mbuf->m_pkthdr.csum_flags |=
1612 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1613 }
1614 }
1615 pd->mbuf->m_pkthdr.csum_data = 0xffff;
1616 }
1617 if (cqe_info->ip_cksum_pass) {
1618 pd->mbuf->m_pkthdr.csum_flags |=
1619 (CSUM_IP_CHECKED|CSUM_IP_VALID);
1620 }
1621 }
1622 *m = tail = pd->mbuf;
1623 }
1624 pd->mbuf = NULL;
1625 len -= frag_len;
1626 }
1627
1628 return;
1629 }
1630
1631 static void
oce_rx_lro(struct oce_rq * rq,struct nic_hwlro_singleton_cqe * cqe,struct nic_hwlro_cqe_part2 * cqe2)1632 oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2)
1633 {
1634 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1635 struct nic_hwlro_cqe_part1 *cqe1 = NULL;
1636 struct mbuf *m = NULL;
1637 struct oce_common_cqe_info cq_info;
1638
1639 /* parse cqe */
1640 if(cqe2 == NULL) {
1641 cq_info.pkt_size = cqe->pkt_size;
1642 cq_info.vtag = cqe->vlan_tag;
1643 cq_info.l4_cksum_pass = cqe->l4_cksum_pass;
1644 cq_info.ip_cksum_pass = cqe->ip_cksum_pass;
1645 cq_info.ipv6_frame = cqe->ipv6_frame;
1646 cq_info.vtp = cqe->vtp;
1647 cq_info.qnq = cqe->qnq;
1648 }else {
1649 cqe1 = (struct nic_hwlro_cqe_part1 *)cqe;
1650 cq_info.pkt_size = cqe2->coalesced_size;
1651 cq_info.vtag = cqe2->vlan_tag;
1652 cq_info.l4_cksum_pass = cqe2->l4_cksum_pass;
1653 cq_info.ip_cksum_pass = cqe2->ip_cksum_pass;
1654 cq_info.ipv6_frame = cqe2->ipv6_frame;
1655 cq_info.vtp = cqe2->vtp;
1656 cq_info.qnq = cqe1->qnq;
1657 }
1658
1659 cq_info.vtag = BSWAP_16(cq_info.vtag);
1660
1661 cq_info.num_frags = cq_info.pkt_size / rq->cfg.frag_size;
1662 if(cq_info.pkt_size % rq->cfg.frag_size)
1663 cq_info.num_frags++;
1664
1665 oce_rx_mbuf_chain(rq, &cq_info, &m);
1666
1667 if (m) {
1668 if(cqe2) {
1669 //assert(cqe2->valid != 0);
1670
1671 //assert(cqe2->cqe_type != 2);
1672 oce_correct_header(m, cqe1, cqe2);
1673 }
1674
1675 m->m_pkthdr.rcvif = sc->ifp;
1676 #if __FreeBSD_version >= 800000
1677 if (rq->queue_index)
1678 m->m_pkthdr.flowid = (rq->queue_index - 1);
1679 else
1680 m->m_pkthdr.flowid = rq->queue_index;
1681 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
1682 #endif
1683 /* This deternies if vlan tag is Valid */
1684 if (cq_info.vtp) {
1685 if (sc->function_mode & FNM_FLEX10_MODE) {
1686 /* FLEX10. If QnQ is not set, neglect VLAN */
1687 if (cq_info.qnq) {
1688 m->m_pkthdr.ether_vtag = cq_info.vtag;
1689 m->m_flags |= M_VLANTAG;
1690 }
1691 } else if (sc->pvid != (cq_info.vtag & VLAN_VID_MASK)) {
1692 /* In UMC mode generally pvid will be striped by
1693 hw. But in some cases we have seen it comes
1694 with pvid. So if pvid == vlan, neglect vlan.
1695 */
1696 m->m_pkthdr.ether_vtag = cq_info.vtag;
1697 m->m_flags |= M_VLANTAG;
1698 }
1699 }
1700 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1);
1701
1702 (*sc->ifp->if_input) (sc->ifp, m);
1703
1704 /* Update rx stats per queue */
1705 rq->rx_stats.rx_pkts++;
1706 rq->rx_stats.rx_bytes += cq_info.pkt_size;
1707 rq->rx_stats.rx_frags += cq_info.num_frags;
1708 rq->rx_stats.rx_ucast_pkts++;
1709 }
1710 return;
1711 }
1712
1713 static void
oce_rx(struct oce_rq * rq,struct oce_nic_rx_cqe * cqe)1714 oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe)
1715 {
1716 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1717 int len;
1718 struct mbuf *m = NULL;
1719 struct oce_common_cqe_info cq_info;
1720 uint16_t vtag = 0;
1721
1722 /* Is it a flush compl that has no data */
1723 if(!cqe->u0.s.num_fragments)
1724 goto exit;
1725
1726 len = cqe->u0.s.pkt_size;
1727 if (!len) {
1728 /*partial DMA workaround for Lancer*/
1729 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments);
1730 goto exit;
1731 }
1732
1733 if (!oce_cqe_portid_valid(sc, cqe)) {
1734 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments);
1735 goto exit;
1736 }
1737
1738 /* Get vlan_tag value */
1739 if(IS_BE(sc) || IS_SH(sc))
1740 vtag = BSWAP_16(cqe->u0.s.vlan_tag);
1741 else
1742 vtag = cqe->u0.s.vlan_tag;
1743
1744 cq_info.l4_cksum_pass = cqe->u0.s.l4_cksum_pass;
1745 cq_info.ip_cksum_pass = cqe->u0.s.ip_cksum_pass;
1746 cq_info.ipv6_frame = cqe->u0.s.ip_ver;
1747 cq_info.num_frags = cqe->u0.s.num_fragments;
1748 cq_info.pkt_size = cqe->u0.s.pkt_size;
1749
1750 oce_rx_mbuf_chain(rq, &cq_info, &m);
1751
1752 if (m) {
1753 m->m_pkthdr.rcvif = sc->ifp;
1754 #if __FreeBSD_version >= 800000
1755 if (rq->queue_index)
1756 m->m_pkthdr.flowid = (rq->queue_index - 1);
1757 else
1758 m->m_pkthdr.flowid = rq->queue_index;
1759 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
1760 #endif
1761 /* This deternies if vlan tag is Valid */
1762 if (oce_cqe_vtp_valid(sc, cqe)) {
1763 if (sc->function_mode & FNM_FLEX10_MODE) {
1764 /* FLEX10. If QnQ is not set, neglect VLAN */
1765 if (cqe->u0.s.qnq) {
1766 m->m_pkthdr.ether_vtag = vtag;
1767 m->m_flags |= M_VLANTAG;
1768 }
1769 } else if (sc->pvid != (vtag & VLAN_VID_MASK)) {
1770 /* In UMC mode generally pvid will be striped by
1771 hw. But in some cases we have seen it comes
1772 with pvid. So if pvid == vlan, neglect vlan.
1773 */
1774 m->m_pkthdr.ether_vtag = vtag;
1775 m->m_flags |= M_VLANTAG;
1776 }
1777 }
1778
1779 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1);
1780 #if defined(INET6) || defined(INET)
1781 /* Try to queue to LRO */
1782 if (IF_LRO_ENABLED(sc) &&
1783 (cqe->u0.s.ip_cksum_pass) &&
1784 (cqe->u0.s.l4_cksum_pass) &&
1785 (!cqe->u0.s.ip_ver) &&
1786 (rq->lro.lro_cnt != 0)) {
1787
1788 if (tcp_lro_rx(&rq->lro, m, 0) == 0) {
1789 rq->lro_pkts_queued ++;
1790 goto post_done;
1791 }
1792 /* If LRO posting fails then try to post to STACK */
1793 }
1794 #endif
1795
1796 (*sc->ifp->if_input) (sc->ifp, m);
1797 #if defined(INET6) || defined(INET)
1798 post_done:
1799 #endif
1800 /* Update rx stats per queue */
1801 rq->rx_stats.rx_pkts++;
1802 rq->rx_stats.rx_bytes += cqe->u0.s.pkt_size;
1803 rq->rx_stats.rx_frags += cqe->u0.s.num_fragments;
1804 if (cqe->u0.s.pkt_type == OCE_MULTICAST_PACKET)
1805 rq->rx_stats.rx_mcast_pkts++;
1806 if (cqe->u0.s.pkt_type == OCE_UNICAST_PACKET)
1807 rq->rx_stats.rx_ucast_pkts++;
1808 }
1809 exit:
1810 return;
1811 }
1812
1813
1814 void
oce_discard_rx_comp(struct oce_rq * rq,int num_frags)1815 oce_discard_rx_comp(struct oce_rq *rq, int num_frags)
1816 {
1817 uint32_t i = 0;
1818 struct oce_packet_desc *pd;
1819 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1820
1821 for (i = 0; i < num_frags; i++) {
1822 if (rq->ring->cidx == rq->ring->pidx) {
1823 device_printf(sc->dev,
1824 "oce_discard_rx_comp: Invalid RX completion - Queue is empty\n");
1825 return;
1826 }
1827 pd = &rq->pckts[rq->ring->cidx];
1828 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1829 bus_dmamap_unload(rq->tag, pd->map);
1830 if (pd->mbuf != NULL) {
1831 m_freem(pd->mbuf);
1832 pd->mbuf = NULL;
1833 }
1834
1835 RING_GET(rq->ring, 1);
1836 rq->pending--;
1837 }
1838 }
1839
1840
1841 static int
oce_cqe_vtp_valid(POCE_SOFTC sc,struct oce_nic_rx_cqe * cqe)1842 oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe)
1843 {
1844 struct oce_nic_rx_cqe_v1 *cqe_v1;
1845 int vtp = 0;
1846
1847 if (sc->be3_native) {
1848 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe;
1849 vtp = cqe_v1->u0.s.vlan_tag_present;
1850 } else
1851 vtp = cqe->u0.s.vlan_tag_present;
1852
1853 return vtp;
1854
1855 }
1856
1857
1858 static int
oce_cqe_portid_valid(POCE_SOFTC sc,struct oce_nic_rx_cqe * cqe)1859 oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe)
1860 {
1861 struct oce_nic_rx_cqe_v1 *cqe_v1;
1862 int port_id = 0;
1863
1864 if (sc->be3_native && (IS_BE(sc) || IS_SH(sc))) {
1865 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe;
1866 port_id = cqe_v1->u0.s.port;
1867 if (sc->port_id != port_id)
1868 return 0;
1869 } else
1870 ;/* For BE3 legacy and Lancer this is dummy */
1871
1872 return 1;
1873
1874 }
1875
1876 #if defined(INET6) || defined(INET)
1877 void
oce_rx_flush_lro(struct oce_rq * rq)1878 oce_rx_flush_lro(struct oce_rq *rq)
1879 {
1880 struct lro_ctrl *lro = &rq->lro;
1881 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1882
1883 if (!IF_LRO_ENABLED(sc))
1884 return;
1885
1886 tcp_lro_flush_all(lro);
1887 rq->lro_pkts_queued = 0;
1888
1889 return;
1890 }
1891
1892
1893 static int
oce_init_lro(POCE_SOFTC sc)1894 oce_init_lro(POCE_SOFTC sc)
1895 {
1896 struct lro_ctrl *lro = NULL;
1897 int i = 0, rc = 0;
1898
1899 for (i = 0; i < sc->nrqs; i++) {
1900 lro = &sc->rq[i]->lro;
1901 rc = tcp_lro_init(lro);
1902 if (rc != 0) {
1903 device_printf(sc->dev, "LRO init failed\n");
1904 return rc;
1905 }
1906 lro->ifp = sc->ifp;
1907 }
1908
1909 return rc;
1910 }
1911
1912
1913 void
oce_free_lro(POCE_SOFTC sc)1914 oce_free_lro(POCE_SOFTC sc)
1915 {
1916 struct lro_ctrl *lro = NULL;
1917 int i = 0;
1918
1919 for (i = 0; i < sc->nrqs; i++) {
1920 lro = &sc->rq[i]->lro;
1921 if (lro)
1922 tcp_lro_free(lro);
1923 }
1924 }
1925 #endif
1926
1927 int
oce_alloc_rx_bufs(struct oce_rq * rq,int count)1928 oce_alloc_rx_bufs(struct oce_rq *rq, int count)
1929 {
1930 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1931 int i, in, rc;
1932 struct oce_packet_desc *pd;
1933 bus_dma_segment_t segs[6];
1934 int nsegs, added = 0;
1935 struct oce_nic_rqe *rqe;
1936 pd_rxulp_db_t rxdb_reg;
1937 uint32_t val = 0;
1938 uint32_t oce_max_rq_posts = 64;
1939
1940 bzero(&rxdb_reg, sizeof(pd_rxulp_db_t));
1941 for (i = 0; i < count; i++) {
1942 in = (rq->ring->pidx + 1) % OCE_RQ_PACKET_ARRAY_SIZE;
1943
1944 pd = &rq->pckts[rq->ring->pidx];
1945 pd->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, oce_rq_buf_size);
1946 if (pd->mbuf == NULL) {
1947 device_printf(sc->dev, "mbuf allocation failed, size = %d\n",oce_rq_buf_size);
1948 break;
1949 }
1950 pd->mbuf->m_nextpkt = NULL;
1951
1952 pd->mbuf->m_len = pd->mbuf->m_pkthdr.len = rq->cfg.frag_size;
1953
1954 rc = bus_dmamap_load_mbuf_sg(rq->tag,
1955 pd->map,
1956 pd->mbuf,
1957 segs, &nsegs, BUS_DMA_NOWAIT);
1958 if (rc) {
1959 m_free(pd->mbuf);
1960 device_printf(sc->dev, "bus_dmamap_load_mbuf_sg failed rc = %d\n", rc);
1961 break;
1962 }
1963
1964 if (nsegs != 1) {
1965 i--;
1966 continue;
1967 }
1968
1969 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_PREREAD);
1970
1971 rqe = RING_GET_PRODUCER_ITEM_VA(rq->ring, struct oce_nic_rqe);
1972 rqe->u0.s.frag_pa_hi = ADDR_HI(segs[0].ds_addr);
1973 rqe->u0.s.frag_pa_lo = ADDR_LO(segs[0].ds_addr);
1974 DW_SWAP(u32ptr(rqe), sizeof(struct oce_nic_rqe));
1975 RING_PUT(rq->ring, 1);
1976 added++;
1977 rq->pending++;
1978 }
1979 oce_max_rq_posts = sc->enable_hwlro ? OCE_HWLRO_MAX_RQ_POSTS : OCE_MAX_RQ_POSTS;
1980 if (added != 0) {
1981 for (i = added / oce_max_rq_posts; i > 0; i--) {
1982 rxdb_reg.bits.num_posted = oce_max_rq_posts;
1983 rxdb_reg.bits.qid = rq->rq_id;
1984 if(rq->islro) {
1985 val |= rq->rq_id & DB_LRO_RQ_ID_MASK;
1986 val |= oce_max_rq_posts << 16;
1987 OCE_WRITE_REG32(sc, db, DB_OFFSET, val);
1988 }else {
1989 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0);
1990 }
1991 added -= oce_max_rq_posts;
1992 }
1993 if (added > 0) {
1994 rxdb_reg.bits.qid = rq->rq_id;
1995 rxdb_reg.bits.num_posted = added;
1996 if(rq->islro) {
1997 val |= rq->rq_id & DB_LRO_RQ_ID_MASK;
1998 val |= added << 16;
1999 OCE_WRITE_REG32(sc, db, DB_OFFSET, val);
2000 }else {
2001 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0);
2002 }
2003 }
2004 }
2005
2006 return 0;
2007 }
2008
2009 static void
oce_check_rx_bufs(POCE_SOFTC sc,uint32_t num_cqes,struct oce_rq * rq)2010 oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq)
2011 {
2012 if (num_cqes) {
2013 oce_arm_cq(sc, rq->cq->cq_id, num_cqes, FALSE);
2014 if(!sc->enable_hwlro) {
2015 if((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) > 1)
2016 oce_alloc_rx_bufs(rq, ((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) - 1));
2017 }else {
2018 if ((OCE_RQ_PACKET_ARRAY_SIZE -1 - rq->pending) > 64)
2019 oce_alloc_rx_bufs(rq, 64);
2020 }
2021 }
2022
2023 return;
2024 }
2025
2026 uint16_t
oce_rq_handler_lro(void * arg)2027 oce_rq_handler_lro(void *arg)
2028 {
2029 struct oce_rq *rq = (struct oce_rq *)arg;
2030 struct oce_cq *cq = rq->cq;
2031 POCE_SOFTC sc = rq->parent;
2032 struct nic_hwlro_singleton_cqe *cqe;
2033 struct nic_hwlro_cqe_part2 *cqe2;
2034 int num_cqes = 0;
2035
2036 LOCK(&rq->rx_lock);
2037 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2038 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe);
2039 while (cqe->valid) {
2040 if(cqe->cqe_type == 0) { /* singleton cqe */
2041 /* we should not get singleton cqe after cqe1 on same rq */
2042 if(rq->cqe_firstpart != NULL) {
2043 device_printf(sc->dev, "Got singleton cqe after cqe1 \n");
2044 goto exit_rq_handler_lro;
2045 }
2046 if(cqe->error != 0) {
2047 rq->rx_stats.rxcp_err++;
2048 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
2049 }
2050 oce_rx_lro(rq, cqe, NULL);
2051 rq->rx_stats.rx_compl++;
2052 cqe->valid = 0;
2053 RING_GET(cq->ring, 1);
2054 num_cqes++;
2055 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
2056 break;
2057 }else if(cqe->cqe_type == 0x1) { /* first part */
2058 /* we should not get cqe1 after cqe1 on same rq */
2059 if(rq->cqe_firstpart != NULL) {
2060 device_printf(sc->dev, "Got cqe1 after cqe1 \n");
2061 goto exit_rq_handler_lro;
2062 }
2063 rq->cqe_firstpart = (struct nic_hwlro_cqe_part1 *)cqe;
2064 RING_GET(cq->ring, 1);
2065 }else if(cqe->cqe_type == 0x2) { /* second part */
2066 cqe2 = (struct nic_hwlro_cqe_part2 *)cqe;
2067 if(cqe2->error != 0) {
2068 rq->rx_stats.rxcp_err++;
2069 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
2070 }
2071 /* We should not get cqe2 without cqe1 */
2072 if(rq->cqe_firstpart == NULL) {
2073 device_printf(sc->dev, "Got cqe2 without cqe1 \n");
2074 goto exit_rq_handler_lro;
2075 }
2076 oce_rx_lro(rq, (struct nic_hwlro_singleton_cqe *)rq->cqe_firstpart, cqe2);
2077
2078 rq->rx_stats.rx_compl++;
2079 rq->cqe_firstpart->valid = 0;
2080 cqe2->valid = 0;
2081 rq->cqe_firstpart = NULL;
2082
2083 RING_GET(cq->ring, 1);
2084 num_cqes += 2;
2085 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
2086 break;
2087 }
2088
2089 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2090 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe);
2091 }
2092 oce_check_rx_bufs(sc, num_cqes, rq);
2093 exit_rq_handler_lro:
2094 UNLOCK(&rq->rx_lock);
2095 return 0;
2096 }
2097
2098 /* Handle the Completion Queue for receive */
2099 uint16_t
oce_rq_handler(void * arg)2100 oce_rq_handler(void *arg)
2101 {
2102 struct oce_rq *rq = (struct oce_rq *)arg;
2103 struct oce_cq *cq = rq->cq;
2104 POCE_SOFTC sc = rq->parent;
2105 struct oce_nic_rx_cqe *cqe;
2106 int num_cqes = 0;
2107
2108 if(rq->islro) {
2109 oce_rq_handler_lro(arg);
2110 return 0;
2111 }
2112 LOCK(&rq->rx_lock);
2113 bus_dmamap_sync(cq->ring->dma.tag,
2114 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2115 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe);
2116 while (cqe->u0.dw[2]) {
2117 DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe));
2118
2119 if (cqe->u0.s.error == 0) {
2120 oce_rx(rq, cqe);
2121 } else {
2122 rq->rx_stats.rxcp_err++;
2123 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
2124 /* Post L3/L4 errors to stack.*/
2125 oce_rx(rq, cqe);
2126 }
2127 rq->rx_stats.rx_compl++;
2128 cqe->u0.dw[2] = 0;
2129
2130 #if defined(INET6) || defined(INET)
2131 if (IF_LRO_ENABLED(sc) && rq->lro_pkts_queued >= 16) {
2132 oce_rx_flush_lro(rq);
2133 }
2134 #endif
2135
2136 RING_GET(cq->ring, 1);
2137 bus_dmamap_sync(cq->ring->dma.tag,
2138 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2139 cqe =
2140 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe);
2141 num_cqes++;
2142 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
2143 break;
2144 }
2145
2146 #if defined(INET6) || defined(INET)
2147 if (IF_LRO_ENABLED(sc))
2148 oce_rx_flush_lro(rq);
2149 #endif
2150
2151 oce_check_rx_bufs(sc, num_cqes, rq);
2152 UNLOCK(&rq->rx_lock);
2153 return 0;
2154
2155 }
2156
2157
2158
2159
2160 /*****************************************************************************
2161 * Helper function prototypes in this file *
2162 *****************************************************************************/
2163
2164 static int
oce_attach_ifp(POCE_SOFTC sc)2165 oce_attach_ifp(POCE_SOFTC sc)
2166 {
2167
2168 sc->ifp = if_alloc(IFT_ETHER);
2169 if (!sc->ifp)
2170 return ENOMEM;
2171
2172 ifmedia_init(&sc->media, IFM_IMASK, oce_media_change, oce_media_status);
2173 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2174 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO);
2175
2176 sc->ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST;
2177 sc->ifp->if_ioctl = oce_ioctl;
2178 sc->ifp->if_start = oce_start;
2179 sc->ifp->if_init = oce_init;
2180 sc->ifp->if_mtu = ETHERMTU;
2181 sc->ifp->if_softc = sc;
2182 #if __FreeBSD_version >= 800000
2183 sc->ifp->if_transmit = oce_multiq_start;
2184 sc->ifp->if_qflush = oce_multiq_flush;
2185 #endif
2186
2187 if_initname(sc->ifp,
2188 device_get_name(sc->dev), device_get_unit(sc->dev));
2189
2190 sc->ifp->if_snd.ifq_drv_maxlen = OCE_MAX_TX_DESC - 1;
2191 IFQ_SET_MAXLEN(&sc->ifp->if_snd, sc->ifp->if_snd.ifq_drv_maxlen);
2192 IFQ_SET_READY(&sc->ifp->if_snd);
2193
2194 sc->ifp->if_hwassist = OCE_IF_HWASSIST;
2195 sc->ifp->if_hwassist |= CSUM_TSO;
2196 sc->ifp->if_hwassist |= (CSUM_IP | CSUM_TCP | CSUM_UDP);
2197
2198 sc->ifp->if_capabilities = OCE_IF_CAPABILITIES;
2199 sc->ifp->if_capabilities |= IFCAP_HWCSUM;
2200 sc->ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2201
2202 #if defined(INET6) || defined(INET)
2203 sc->ifp->if_capabilities |= IFCAP_TSO;
2204 sc->ifp->if_capabilities |= IFCAP_LRO;
2205 sc->ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
2206 #endif
2207
2208 sc->ifp->if_capenable = sc->ifp->if_capabilities;
2209 sc->ifp->if_baudrate = IF_Gbps(10);
2210
2211 #if __FreeBSD_version >= 1000000
2212 sc->ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
2213 sc->ifp->if_hw_tsomaxsegcount = OCE_MAX_TX_ELEMENTS;
2214 sc->ifp->if_hw_tsomaxsegsize = 4096;
2215 #endif
2216
2217 ether_ifattach(sc->ifp, sc->macaddr.mac_addr);
2218
2219 return 0;
2220 }
2221
2222
2223 static void
oce_add_vlan(void * arg,struct ifnet * ifp,uint16_t vtag)2224 oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag)
2225 {
2226 POCE_SOFTC sc = ifp->if_softc;
2227
2228 if (ifp->if_softc != arg)
2229 return;
2230 if ((vtag == 0) || (vtag > 4095))
2231 return;
2232
2233 sc->vlan_tag[vtag] = 1;
2234 sc->vlans_added++;
2235 if (sc->vlans_added <= (sc->max_vlans + 1))
2236 oce_vid_config(sc);
2237 }
2238
2239
2240 static void
oce_del_vlan(void * arg,struct ifnet * ifp,uint16_t vtag)2241 oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag)
2242 {
2243 POCE_SOFTC sc = ifp->if_softc;
2244
2245 if (ifp->if_softc != arg)
2246 return;
2247 if ((vtag == 0) || (vtag > 4095))
2248 return;
2249
2250 sc->vlan_tag[vtag] = 0;
2251 sc->vlans_added--;
2252 oce_vid_config(sc);
2253 }
2254
2255
2256 /*
2257 * A max of 64 vlans can be configured in BE. If the user configures
2258 * more, place the card in vlan promiscuous mode.
2259 */
2260 static int
oce_vid_config(POCE_SOFTC sc)2261 oce_vid_config(POCE_SOFTC sc)
2262 {
2263 struct normal_vlan vtags[MAX_VLANFILTER_SIZE];
2264 uint16_t ntags = 0, i;
2265 int status = 0;
2266
2267 if ((sc->vlans_added <= MAX_VLANFILTER_SIZE) &&
2268 (sc->ifp->if_capenable & IFCAP_VLAN_HWFILTER)) {
2269 for (i = 0; i < MAX_VLANS; i++) {
2270 if (sc->vlan_tag[i]) {
2271 vtags[ntags].vtag = i;
2272 ntags++;
2273 }
2274 }
2275 if (ntags)
2276 status = oce_config_vlan(sc, (uint8_t) sc->if_id,
2277 vtags, ntags, 1, 0);
2278 } else
2279 status = oce_config_vlan(sc, (uint8_t) sc->if_id,
2280 NULL, 0, 1, 1);
2281 return status;
2282 }
2283
2284
2285 static void
oce_mac_addr_set(POCE_SOFTC sc)2286 oce_mac_addr_set(POCE_SOFTC sc)
2287 {
2288 uint32_t old_pmac_id = sc->pmac_id;
2289 int status = 0;
2290
2291
2292 status = bcmp((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr,
2293 sc->macaddr.size_of_struct);
2294 if (!status)
2295 return;
2296
2297 status = oce_mbox_macaddr_add(sc, (uint8_t *)(IF_LLADDR(sc->ifp)),
2298 sc->if_id, &sc->pmac_id);
2299 if (!status) {
2300 status = oce_mbox_macaddr_del(sc, sc->if_id, old_pmac_id);
2301 bcopy((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr,
2302 sc->macaddr.size_of_struct);
2303 }
2304 if (status)
2305 device_printf(sc->dev, "Failed update macaddress\n");
2306
2307 }
2308
2309
2310 static int
oce_handle_passthrough(struct ifnet * ifp,caddr_t data)2311 oce_handle_passthrough(struct ifnet *ifp, caddr_t data)
2312 {
2313 POCE_SOFTC sc = ifp->if_softc;
2314 struct ifreq *ifr = (struct ifreq *)data;
2315 int rc = ENXIO;
2316 char cookie[32] = {0};
2317 void *priv_data = ifr_data_get_ptr(ifr);
2318 void *ioctl_ptr;
2319 uint32_t req_size;
2320 struct mbx_hdr req;
2321 OCE_DMA_MEM dma_mem;
2322 struct mbx_common_get_cntl_attr *fw_cmd;
2323
2324 if (copyin(priv_data, cookie, strlen(IOCTL_COOKIE)))
2325 return EFAULT;
2326
2327 if (memcmp(cookie, IOCTL_COOKIE, strlen(IOCTL_COOKIE)))
2328 return EINVAL;
2329
2330 ioctl_ptr = (char *)priv_data + strlen(IOCTL_COOKIE);
2331 if (copyin(ioctl_ptr, &req, sizeof(struct mbx_hdr)))
2332 return EFAULT;
2333
2334 req_size = le32toh(req.u0.req.request_length);
2335 if (req_size > 65536)
2336 return EINVAL;
2337
2338 req_size += sizeof(struct mbx_hdr);
2339 rc = oce_dma_alloc(sc, req_size, &dma_mem, 0);
2340 if (rc)
2341 return ENOMEM;
2342
2343 if (copyin(ioctl_ptr, OCE_DMAPTR(&dma_mem,char), req_size)) {
2344 rc = EFAULT;
2345 goto dma_free;
2346 }
2347
2348 rc = oce_pass_through_mbox(sc, &dma_mem, req_size);
2349 if (rc) {
2350 rc = EIO;
2351 goto dma_free;
2352 }
2353
2354 if (copyout(OCE_DMAPTR(&dma_mem,char), ioctl_ptr, req_size))
2355 rc = EFAULT;
2356
2357 /*
2358 firmware is filling all the attributes for this ioctl except
2359 the driver version..so fill it
2360 */
2361 if(req.u0.rsp.opcode == OPCODE_COMMON_GET_CNTL_ATTRIBUTES) {
2362 fw_cmd = (struct mbx_common_get_cntl_attr *) ioctl_ptr;
2363 strncpy(fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str,
2364 COMPONENT_REVISION, strlen(COMPONENT_REVISION));
2365 }
2366
2367 dma_free:
2368 oce_dma_free(sc, &dma_mem);
2369 return rc;
2370
2371 }
2372
2373 static void
oce_eqd_set_periodic(POCE_SOFTC sc)2374 oce_eqd_set_periodic(POCE_SOFTC sc)
2375 {
2376 struct oce_set_eqd set_eqd[OCE_MAX_EQ];
2377 struct oce_aic_obj *aic;
2378 struct oce_eq *eqo;
2379 uint64_t now = 0, delta;
2380 int eqd, i, num = 0;
2381 uint32_t tx_reqs = 0, rxpkts = 0, pps;
2382 struct oce_wq *wq;
2383 struct oce_rq *rq;
2384
2385 #define ticks_to_msecs(t) (1000 * (t) / hz)
2386
2387 for (i = 0 ; i < sc->neqs; i++) {
2388 eqo = sc->eq[i];
2389 aic = &sc->aic_obj[i];
2390 /* When setting the static eq delay from the user space */
2391 if (!aic->enable) {
2392 if (aic->ticks)
2393 aic->ticks = 0;
2394 eqd = aic->et_eqd;
2395 goto modify_eqd;
2396 }
2397
2398 if (i == 0) {
2399 rq = sc->rq[0];
2400 rxpkts = rq->rx_stats.rx_pkts;
2401 } else
2402 rxpkts = 0;
2403 if (i + 1 < sc->nrqs) {
2404 rq = sc->rq[i + 1];
2405 rxpkts += rq->rx_stats.rx_pkts;
2406 }
2407 if (i < sc->nwqs) {
2408 wq = sc->wq[i];
2409 tx_reqs = wq->tx_stats.tx_reqs;
2410 } else
2411 tx_reqs = 0;
2412 now = ticks;
2413
2414 if (!aic->ticks || now < aic->ticks ||
2415 rxpkts < aic->prev_rxpkts || tx_reqs < aic->prev_txreqs) {
2416 aic->prev_rxpkts = rxpkts;
2417 aic->prev_txreqs = tx_reqs;
2418 aic->ticks = now;
2419 continue;
2420 }
2421
2422 delta = ticks_to_msecs(now - aic->ticks);
2423
2424 pps = (((uint32_t)(rxpkts - aic->prev_rxpkts) * 1000) / delta) +
2425 (((uint32_t)(tx_reqs - aic->prev_txreqs) * 1000) / delta);
2426 eqd = (pps / 15000) << 2;
2427 if (eqd < 8)
2428 eqd = 0;
2429
2430 /* Make sure that the eq delay is in the known range */
2431 eqd = min(eqd, aic->max_eqd);
2432 eqd = max(eqd, aic->min_eqd);
2433
2434 aic->prev_rxpkts = rxpkts;
2435 aic->prev_txreqs = tx_reqs;
2436 aic->ticks = now;
2437
2438 modify_eqd:
2439 if (eqd != aic->cur_eqd) {
2440 set_eqd[num].delay_multiplier = (eqd * 65)/100;
2441 set_eqd[num].eq_id = eqo->eq_id;
2442 aic->cur_eqd = eqd;
2443 num++;
2444 }
2445 }
2446
2447 /* Is there atleast one eq that needs to be modified? */
2448 for(i = 0; i < num; i += 8) {
2449 if((num - i) >=8 )
2450 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], 8);
2451 else
2452 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], (num - i));
2453 }
2454
2455 }
2456
oce_detect_hw_error(POCE_SOFTC sc)2457 static void oce_detect_hw_error(POCE_SOFTC sc)
2458 {
2459
2460 uint32_t ue_low = 0, ue_high = 0, ue_low_mask = 0, ue_high_mask = 0;
2461 uint32_t sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2462 uint32_t i;
2463
2464 if (sc->hw_error)
2465 return;
2466
2467 if (IS_XE201(sc)) {
2468 sliport_status = OCE_READ_REG32(sc, db, SLIPORT_STATUS_OFFSET);
2469 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2470 sliport_err1 = OCE_READ_REG32(sc, db, SLIPORT_ERROR1_OFFSET);
2471 sliport_err2 = OCE_READ_REG32(sc, db, SLIPORT_ERROR2_OFFSET);
2472 }
2473 } else {
2474 ue_low = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW);
2475 ue_high = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HIGH);
2476 ue_low_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW_MASK);
2477 ue_high_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HI_MASK);
2478
2479 ue_low = (ue_low & ~ue_low_mask);
2480 ue_high = (ue_high & ~ue_high_mask);
2481 }
2482
2483 /* On certain platforms BE hardware can indicate spurious UEs.
2484 * Allow the h/w to stop working completely in case of a real UE.
2485 * Hence not setting the hw_error for UE detection.
2486 */
2487 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2488 sc->hw_error = TRUE;
2489 device_printf(sc->dev, "Error detected in the card\n");
2490 }
2491
2492 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2493 device_printf(sc->dev,
2494 "ERR: sliport status 0x%x\n", sliport_status);
2495 device_printf(sc->dev,
2496 "ERR: sliport error1 0x%x\n", sliport_err1);
2497 device_printf(sc->dev,
2498 "ERR: sliport error2 0x%x\n", sliport_err2);
2499 }
2500
2501 if (ue_low) {
2502 for (i = 0; ue_low; ue_low >>= 1, i++) {
2503 if (ue_low & 1)
2504 device_printf(sc->dev, "UE: %s bit set\n",
2505 ue_status_low_desc[i]);
2506 }
2507 }
2508
2509 if (ue_high) {
2510 for (i = 0; ue_high; ue_high >>= 1, i++) {
2511 if (ue_high & 1)
2512 device_printf(sc->dev, "UE: %s bit set\n",
2513 ue_status_hi_desc[i]);
2514 }
2515 }
2516
2517 }
2518
2519
2520 static void
oce_local_timer(void * arg)2521 oce_local_timer(void *arg)
2522 {
2523 POCE_SOFTC sc = arg;
2524 int i = 0;
2525
2526 oce_detect_hw_error(sc);
2527 oce_refresh_nic_stats(sc);
2528 oce_refresh_queue_stats(sc);
2529 oce_mac_addr_set(sc);
2530
2531 /* TX Watch Dog*/
2532 for (i = 0; i < sc->nwqs; i++)
2533 oce_tx_restart(sc, sc->wq[i]);
2534
2535 /* calculate and set the eq delay for optimal interrupt rate */
2536 if (IS_BE(sc) || IS_SH(sc))
2537 oce_eqd_set_periodic(sc);
2538
2539 callout_reset(&sc->timer, hz, oce_local_timer, sc);
2540 }
2541
2542 static void
oce_tx_compl_clean(POCE_SOFTC sc)2543 oce_tx_compl_clean(POCE_SOFTC sc)
2544 {
2545 struct oce_wq *wq;
2546 int i = 0, timeo = 0, num_wqes = 0;
2547 int pending_txqs = sc->nwqs;
2548
2549 /* Stop polling for compls when HW has been silent for 10ms or
2550 * hw_error or no outstanding completions expected
2551 */
2552 do {
2553 pending_txqs = sc->nwqs;
2554
2555 for_all_wq_queues(sc, wq, i) {
2556 num_wqes = oce_wq_handler(wq);
2557
2558 if(num_wqes)
2559 timeo = 0;
2560
2561 if(!wq->ring->num_used)
2562 pending_txqs--;
2563 }
2564
2565 if (pending_txqs == 0 || ++timeo > 10 || sc->hw_error)
2566 break;
2567
2568 DELAY(1000);
2569 } while (TRUE);
2570
2571 for_all_wq_queues(sc, wq, i) {
2572 while(wq->ring->num_used) {
2573 LOCK(&wq->tx_compl_lock);
2574 oce_process_tx_completion(wq);
2575 UNLOCK(&wq->tx_compl_lock);
2576 }
2577 }
2578
2579 }
2580
2581 /* NOTE : This should only be called holding
2582 * DEVICE_LOCK.
2583 */
2584 static void
oce_if_deactivate(POCE_SOFTC sc)2585 oce_if_deactivate(POCE_SOFTC sc)
2586 {
2587 int i;
2588 struct oce_rq *rq;
2589 struct oce_wq *wq;
2590 struct oce_eq *eq;
2591
2592 sc->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
2593
2594 oce_tx_compl_clean(sc);
2595
2596 /* Stop intrs and finish any bottom halves pending */
2597 oce_hw_intr_disable(sc);
2598
2599 /* Since taskqueue_drain takes a Gaint Lock, We should not acquire
2600 any other lock. So unlock device lock and require after
2601 completing taskqueue_drain.
2602 */
2603 UNLOCK(&sc->dev_lock);
2604 for (i = 0; i < sc->intr_count; i++) {
2605 if (sc->intrs[i].tq != NULL) {
2606 taskqueue_drain(sc->intrs[i].tq, &sc->intrs[i].task);
2607 }
2608 }
2609 LOCK(&sc->dev_lock);
2610
2611 /* Delete RX queue in card with flush param */
2612 oce_stop_rx(sc);
2613
2614 /* Invalidate any pending cq and eq entries*/
2615 for_all_evnt_queues(sc, eq, i)
2616 oce_drain_eq(eq);
2617 for_all_rq_queues(sc, rq, i)
2618 oce_drain_rq_cq(rq);
2619 for_all_wq_queues(sc, wq, i)
2620 oce_drain_wq_cq(wq);
2621
2622 /* But still we need to get MCC aync events.
2623 So enable intrs and also arm first EQ
2624 */
2625 oce_hw_intr_enable(sc);
2626 oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE);
2627
2628 DELAY(10);
2629 }
2630
2631
2632 static void
oce_if_activate(POCE_SOFTC sc)2633 oce_if_activate(POCE_SOFTC sc)
2634 {
2635 struct oce_eq *eq;
2636 struct oce_rq *rq;
2637 struct oce_wq *wq;
2638 int i, rc = 0;
2639
2640 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING;
2641
2642 oce_hw_intr_disable(sc);
2643
2644 oce_start_rx(sc);
2645
2646 for_all_rq_queues(sc, rq, i) {
2647 rc = oce_start_rq(rq);
2648 if (rc)
2649 device_printf(sc->dev, "Unable to start RX\n");
2650 }
2651
2652 for_all_wq_queues(sc, wq, i) {
2653 rc = oce_start_wq(wq);
2654 if (rc)
2655 device_printf(sc->dev, "Unable to start TX\n");
2656 }
2657
2658
2659 for_all_evnt_queues(sc, eq, i)
2660 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE);
2661
2662 oce_hw_intr_enable(sc);
2663
2664 }
2665
2666 static void
process_link_state(POCE_SOFTC sc,struct oce_async_cqe_link_state * acqe)2667 process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe)
2668 {
2669 /* Update Link status */
2670 if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) ==
2671 ASYNC_EVENT_LINK_UP) {
2672 sc->link_status = ASYNC_EVENT_LINK_UP;
2673 if_link_state_change(sc->ifp, LINK_STATE_UP);
2674 } else {
2675 sc->link_status = ASYNC_EVENT_LINK_DOWN;
2676 if_link_state_change(sc->ifp, LINK_STATE_DOWN);
2677 }
2678 }
2679
2680
oce_async_grp5_osbmc_process(POCE_SOFTC sc,struct oce_async_evt_grp5_os2bmc * evt)2681 static void oce_async_grp5_osbmc_process(POCE_SOFTC sc,
2682 struct oce_async_evt_grp5_os2bmc *evt)
2683 {
2684 DW_SWAP(evt, sizeof(struct oce_async_evt_grp5_os2bmc));
2685 if (evt->u.s.mgmt_enable)
2686 sc->flags |= OCE_FLAGS_OS2BMC;
2687 else
2688 return;
2689
2690 sc->bmc_filt_mask = evt->u.s.arp_filter;
2691 sc->bmc_filt_mask |= (evt->u.s.dhcp_client_filt << 1);
2692 sc->bmc_filt_mask |= (evt->u.s.dhcp_server_filt << 2);
2693 sc->bmc_filt_mask |= (evt->u.s.net_bios_filt << 3);
2694 sc->bmc_filt_mask |= (evt->u.s.bcast_filt << 4);
2695 sc->bmc_filt_mask |= (evt->u.s.ipv6_nbr_filt << 5);
2696 sc->bmc_filt_mask |= (evt->u.s.ipv6_ra_filt << 6);
2697 sc->bmc_filt_mask |= (evt->u.s.ipv6_ras_filt << 7);
2698 sc->bmc_filt_mask |= (evt->u.s.mcast_filt << 8);
2699 }
2700
2701
oce_process_grp5_events(POCE_SOFTC sc,struct oce_mq_cqe * cqe)2702 static void oce_process_grp5_events(POCE_SOFTC sc, struct oce_mq_cqe *cqe)
2703 {
2704 struct oce_async_event_grp5_pvid_state *gcqe;
2705 struct oce_async_evt_grp5_os2bmc *bmccqe;
2706
2707 switch (cqe->u0.s.async_type) {
2708 case ASYNC_EVENT_PVID_STATE:
2709 /* GRP5 PVID */
2710 gcqe = (struct oce_async_event_grp5_pvid_state *)cqe;
2711 if (gcqe->enabled)
2712 sc->pvid = gcqe->tag & VLAN_VID_MASK;
2713 else
2714 sc->pvid = 0;
2715 break;
2716 case ASYNC_EVENT_OS2BMC:
2717 bmccqe = (struct oce_async_evt_grp5_os2bmc *)cqe;
2718 oce_async_grp5_osbmc_process(sc, bmccqe);
2719 break;
2720 default:
2721 break;
2722 }
2723 }
2724
2725 /* Handle the Completion Queue for the Mailbox/Async notifications */
2726 uint16_t
oce_mq_handler(void * arg)2727 oce_mq_handler(void *arg)
2728 {
2729 struct oce_mq *mq = (struct oce_mq *)arg;
2730 POCE_SOFTC sc = mq->parent;
2731 struct oce_cq *cq = mq->cq;
2732 int num_cqes = 0, evt_type = 0, optype = 0;
2733 struct oce_mq_cqe *cqe;
2734 struct oce_async_cqe_link_state *acqe;
2735 struct oce_async_event_qnq *dbgcqe;
2736
2737
2738 bus_dmamap_sync(cq->ring->dma.tag,
2739 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2740 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe);
2741
2742 while (cqe->u0.dw[3]) {
2743 DW_SWAP((uint32_t *) cqe, sizeof(oce_mq_cqe));
2744 if (cqe->u0.s.async_event) {
2745 evt_type = cqe->u0.s.event_type;
2746 optype = cqe->u0.s.async_type;
2747 if (evt_type == ASYNC_EVENT_CODE_LINK_STATE) {
2748 /* Link status evt */
2749 acqe = (struct oce_async_cqe_link_state *)cqe;
2750 process_link_state(sc, acqe);
2751 } else if (evt_type == ASYNC_EVENT_GRP5) {
2752 oce_process_grp5_events(sc, cqe);
2753 } else if (evt_type == ASYNC_EVENT_CODE_DEBUG &&
2754 optype == ASYNC_EVENT_DEBUG_QNQ) {
2755 dbgcqe = (struct oce_async_event_qnq *)cqe;
2756 if(dbgcqe->valid)
2757 sc->qnqid = dbgcqe->vlan_tag;
2758 sc->qnq_debug_event = TRUE;
2759 }
2760 }
2761 cqe->u0.dw[3] = 0;
2762 RING_GET(cq->ring, 1);
2763 bus_dmamap_sync(cq->ring->dma.tag,
2764 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2765 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe);
2766 num_cqes++;
2767 }
2768
2769 if (num_cqes)
2770 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE);
2771
2772 return 0;
2773 }
2774
2775
2776 static void
setup_max_queues_want(POCE_SOFTC sc)2777 setup_max_queues_want(POCE_SOFTC sc)
2778 {
2779 /* Check if it is FLEX machine. Is so dont use RSS */
2780 if ((sc->function_mode & FNM_FLEX10_MODE) ||
2781 (sc->function_mode & FNM_UMC_MODE) ||
2782 (sc->function_mode & FNM_VNIC_MODE) ||
2783 (!is_rss_enabled(sc)) ||
2784 IS_BE2(sc)) {
2785 sc->nrqs = 1;
2786 sc->nwqs = 1;
2787 } else {
2788 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1;
2789 sc->nwqs = MIN(OCE_NCPUS, sc->nrssqs);
2790 }
2791
2792 if (IS_BE2(sc) && is_rss_enabled(sc))
2793 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1;
2794 }
2795
2796
2797 static void
update_queues_got(POCE_SOFTC sc)2798 update_queues_got(POCE_SOFTC sc)
2799 {
2800 if (is_rss_enabled(sc)) {
2801 sc->nrqs = sc->intr_count + 1;
2802 sc->nwqs = sc->intr_count;
2803 } else {
2804 sc->nrqs = 1;
2805 sc->nwqs = 1;
2806 }
2807
2808 if (IS_BE2(sc))
2809 sc->nwqs = 1;
2810 }
2811
2812 static int
oce_check_ipv6_ext_hdr(struct mbuf * m)2813 oce_check_ipv6_ext_hdr(struct mbuf *m)
2814 {
2815 struct ether_header *eh = mtod(m, struct ether_header *);
2816 caddr_t m_datatemp = m->m_data;
2817
2818 if (eh->ether_type == htons(ETHERTYPE_IPV6)) {
2819 m->m_data += sizeof(struct ether_header);
2820 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
2821
2822 if((ip6->ip6_nxt != IPPROTO_TCP) && \
2823 (ip6->ip6_nxt != IPPROTO_UDP)){
2824 struct ip6_ext *ip6e = NULL;
2825 m->m_data += sizeof(struct ip6_hdr);
2826
2827 ip6e = (struct ip6_ext *) mtod(m, struct ip6_ext *);
2828 if(ip6e->ip6e_len == 0xff) {
2829 m->m_data = m_datatemp;
2830 return TRUE;
2831 }
2832 }
2833 m->m_data = m_datatemp;
2834 }
2835 return FALSE;
2836 }
2837
2838 static int
is_be3_a1(POCE_SOFTC sc)2839 is_be3_a1(POCE_SOFTC sc)
2840 {
2841 if((sc->flags & OCE_FLAGS_BE3) && ((sc->asic_revision & 0xFF) < 2)) {
2842 return TRUE;
2843 }
2844 return FALSE;
2845 }
2846
2847 static struct mbuf *
oce_insert_vlan_tag(POCE_SOFTC sc,struct mbuf * m,boolean_t * complete)2848 oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete)
2849 {
2850 uint16_t vlan_tag = 0;
2851
2852 if(!M_WRITABLE(m))
2853 return NULL;
2854
2855 /* Embed vlan tag in the packet if it is not part of it */
2856 if(m->m_flags & M_VLANTAG) {
2857 vlan_tag = EVL_VLANOFTAG(m->m_pkthdr.ether_vtag);
2858 m->m_flags &= ~M_VLANTAG;
2859 }
2860
2861 /* if UMC, ignore vlan tag insertion and instead insert pvid */
2862 if(sc->pvid) {
2863 if(!vlan_tag)
2864 vlan_tag = sc->pvid;
2865 if (complete)
2866 *complete = FALSE;
2867 }
2868
2869 if(vlan_tag) {
2870 m = ether_vlanencap(m, vlan_tag);
2871 }
2872
2873 if(sc->qnqid) {
2874 m = ether_vlanencap(m, sc->qnqid);
2875
2876 if (complete)
2877 *complete = FALSE;
2878 }
2879 return m;
2880 }
2881
2882 static int
oce_tx_asic_stall_verify(POCE_SOFTC sc,struct mbuf * m)2883 oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m)
2884 {
2885 if(is_be3_a1(sc) && IS_QNQ_OR_UMC(sc) && \
2886 oce_check_ipv6_ext_hdr(m)) {
2887 return TRUE;
2888 }
2889 return FALSE;
2890 }
2891
2892 static void
oce_get_config(POCE_SOFTC sc)2893 oce_get_config(POCE_SOFTC sc)
2894 {
2895 int rc = 0;
2896 uint32_t max_rss = 0;
2897
2898 if ((IS_BE(sc) || IS_SH(sc)) && (!sc->be3_native))
2899 max_rss = OCE_LEGACY_MODE_RSS;
2900 else
2901 max_rss = OCE_MAX_RSS;
2902
2903 if (!IS_BE(sc)) {
2904 rc = oce_get_profile_config(sc, max_rss);
2905 if (rc) {
2906 sc->nwqs = OCE_MAX_WQ;
2907 sc->nrssqs = max_rss;
2908 sc->nrqs = sc->nrssqs + 1;
2909 }
2910 }
2911 else { /* For BE3 don't rely on fw for determining the resources */
2912 sc->nrssqs = max_rss;
2913 sc->nrqs = sc->nrssqs + 1;
2914 sc->nwqs = OCE_MAX_WQ;
2915 sc->max_vlans = MAX_VLANFILTER_SIZE;
2916 }
2917 }
2918
2919 static void
oce_rdma_close(void)2920 oce_rdma_close(void)
2921 {
2922 if (oce_rdma_if != NULL) {
2923 oce_rdma_if = NULL;
2924 }
2925 }
2926
2927 static void
oce_get_mac_addr(POCE_SOFTC sc,uint8_t * macaddr)2928 oce_get_mac_addr(POCE_SOFTC sc, uint8_t *macaddr)
2929 {
2930 memcpy(macaddr, sc->macaddr.mac_addr, 6);
2931 }
2932
2933 int
oce_register_rdma(POCE_RDMA_INFO rdma_info,POCE_RDMA_IF rdma_if)2934 oce_register_rdma(POCE_RDMA_INFO rdma_info, POCE_RDMA_IF rdma_if)
2935 {
2936 POCE_SOFTC sc;
2937 struct oce_dev_info di;
2938 int i;
2939
2940 if ((rdma_info == NULL) || (rdma_if == NULL)) {
2941 return -EINVAL;
2942 }
2943
2944 if ((rdma_info->size != OCE_RDMA_INFO_SIZE) ||
2945 (rdma_if->size != OCE_RDMA_IF_SIZE)) {
2946 return -ENXIO;
2947 }
2948
2949 rdma_info->close = oce_rdma_close;
2950 rdma_info->mbox_post = oce_mbox_post;
2951 rdma_info->common_req_hdr_init = mbx_common_req_hdr_init;
2952 rdma_info->get_mac_addr = oce_get_mac_addr;
2953
2954 oce_rdma_if = rdma_if;
2955
2956 sc = softc_head;
2957 while (sc != NULL) {
2958 if (oce_rdma_if->announce != NULL) {
2959 memset(&di, 0, sizeof(di));
2960 di.dev = sc->dev;
2961 di.softc = sc;
2962 di.ifp = sc->ifp;
2963 di.db_bhandle = sc->db_bhandle;
2964 di.db_btag = sc->db_btag;
2965 di.db_page_size = 4096;
2966 if (sc->flags & OCE_FLAGS_USING_MSIX) {
2967 di.intr_mode = OCE_INTERRUPT_MODE_MSIX;
2968 } else if (sc->flags & OCE_FLAGS_USING_MSI) {
2969 di.intr_mode = OCE_INTERRUPT_MODE_MSI;
2970 } else {
2971 di.intr_mode = OCE_INTERRUPT_MODE_INTX;
2972 }
2973 di.dev_family = OCE_GEN2_FAMILY; // fixme: must detect skyhawk
2974 if (di.intr_mode != OCE_INTERRUPT_MODE_INTX) {
2975 di.msix.num_vectors = sc->intr_count + sc->roce_intr_count;
2976 di.msix.start_vector = sc->intr_count;
2977 for (i=0; i<di.msix.num_vectors; i++) {
2978 di.msix.vector_list[i] = sc->intrs[i].vector;
2979 }
2980 } else {
2981 }
2982 memcpy(di.mac_addr, sc->macaddr.mac_addr, 6);
2983 di.vendor_id = pci_get_vendor(sc->dev);
2984 di.dev_id = pci_get_device(sc->dev);
2985
2986 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
2987 di.flags |= OCE_RDMA_INFO_RDMA_SUPPORTED;
2988 }
2989
2990 rdma_if->announce(&di);
2991 sc = sc->next;
2992 }
2993 }
2994
2995 return 0;
2996 }
2997
2998 static void
oce_read_env_variables(POCE_SOFTC sc)2999 oce_read_env_variables( POCE_SOFTC sc )
3000 {
3001 char *value = NULL;
3002 int rc = 0;
3003
3004 /* read if user wants to enable hwlro or swlro */
3005 //value = getenv("oce_enable_hwlro");
3006 if(value && IS_SH(sc)) {
3007 sc->enable_hwlro = strtol(value, NULL, 10);
3008 if(sc->enable_hwlro) {
3009 rc = oce_mbox_nic_query_lro_capabilities(sc, NULL, NULL);
3010 if(rc) {
3011 device_printf(sc->dev, "no hardware lro support\n");
3012 device_printf(sc->dev, "software lro enabled\n");
3013 sc->enable_hwlro = 0;
3014 }else {
3015 device_printf(sc->dev, "hardware lro enabled\n");
3016 oce_max_rsp_handled = 32;
3017 }
3018 }else {
3019 device_printf(sc->dev, "software lro enabled\n");
3020 }
3021 }else {
3022 sc->enable_hwlro = 0;
3023 }
3024
3025 /* read mbuf size */
3026 //value = getenv("oce_rq_buf_size");
3027 if(value && IS_SH(sc)) {
3028 oce_rq_buf_size = strtol(value, NULL, 10);
3029 switch(oce_rq_buf_size) {
3030 case 2048:
3031 case 4096:
3032 case 9216:
3033 case 16384:
3034 break;
3035
3036 default:
3037 device_printf(sc->dev, " Supported oce_rq_buf_size values are 2K, 4K, 9K, 16K \n");
3038 oce_rq_buf_size = 2048;
3039 }
3040 }
3041
3042 return;
3043 }
3044