1 /*
2 * BSD LICENSE
3 *
4 * Copyright(c) 2017 Cavium, Inc.. All rights reserved.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * * Neither the name of Cavium, Inc. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 #include "lio_bsd.h"
35 #include "lio_common.h"
36
37 #include "lio_droq.h"
38 #include "lio_iq.h"
39 #include "lio_response_manager.h"
40 #include "lio_device.h"
41 #include "lio_ctrl.h"
42 #include "lio_main.h"
43 #include "lio_network.h"
44 #include "cn23xx_pf_device.h"
45 #include "lio_image.h"
46 #include "lio_ioctl.h"
47 #include "lio_rxtx.h"
48 #include "lio_rss.h"
49
50 /* Number of milliseconds to wait for DDR initialization */
51 #define LIO_DDR_TIMEOUT 10000
52 #define LIO_MAX_FW_TYPE_LEN 8
53
54 static char fw_type[LIO_MAX_FW_TYPE_LEN];
55 TUNABLE_STR("hw.lio.fw_type", fw_type, sizeof(fw_type));
56
57 /*
58 * Integers that specify number of queues per PF.
59 * Valid range is 0 to 64.
60 * Use 0 to derive from CPU count.
61 */
62 static int num_queues_per_pf0;
63 static int num_queues_per_pf1;
64 TUNABLE_INT("hw.lio.num_queues_per_pf0", &num_queues_per_pf0);
65 TUNABLE_INT("hw.lio.num_queues_per_pf1", &num_queues_per_pf1);
66
67 #ifdef RSS
68 static int lio_rss = 1;
69 TUNABLE_INT("hw.lio.rss", &lio_rss);
70 #endif /* RSS */
71
72 /* Hardware LRO */
73 unsigned int lio_hwlro = 0;
74 TUNABLE_INT("hw.lio.hwlro", &lio_hwlro);
75
76 /*
77 * Bitmask indicating which consoles have debug
78 * output redirected to syslog.
79 */
80 static unsigned long console_bitmask;
81 TUNABLE_ULONG("hw.lio.console_bitmask", &console_bitmask);
82
83 /*
84 * \brief determines if a given console has debug enabled.
85 * @param console console to check
86 * @returns 1 = enabled. 0 otherwise
87 */
88 int
lio_console_debug_enabled(uint32_t console)89 lio_console_debug_enabled(uint32_t console)
90 {
91
92 return (console_bitmask >> (console)) & 0x1;
93 }
94
95 static int lio_detach(device_t dev);
96
97 static int lio_device_init(struct octeon_device *octeon_dev);
98 static int lio_chip_specific_setup(struct octeon_device *oct);
99 static void lio_watchdog(void *param);
100 static int lio_load_firmware(struct octeon_device *oct);
101 static int lio_nic_starter(struct octeon_device *oct);
102 static int lio_init_nic_module(struct octeon_device *oct);
103 static int lio_setup_nic_devices(struct octeon_device *octeon_dev);
104 static int lio_link_info(struct lio_recv_info *recv_info, void *ptr);
105 static void lio_if_cfg_callback(struct octeon_device *oct, uint32_t status,
106 void *buf);
107 static int lio_set_rxcsum_command(struct ifnet *ifp, int command,
108 uint8_t rx_cmd);
109 static int lio_setup_glists(struct octeon_device *oct, struct lio *lio,
110 int num_iqs);
111 static void lio_destroy_nic_device(struct octeon_device *oct, int ifidx);
112 static inline void lio_update_link_status(struct ifnet *ifp,
113 union octeon_link_status *ls);
114 static void lio_send_rx_ctrl_cmd(struct lio *lio, int start_stop);
115 static int lio_stop_nic_module(struct octeon_device *oct);
116 static void lio_destroy_resources(struct octeon_device *oct);
117 static int lio_setup_rx_oom_poll_fn(struct ifnet *ifp);
118
119 static void lio_vlan_rx_add_vid(void *arg, struct ifnet *ifp, uint16_t vid);
120 static void lio_vlan_rx_kill_vid(void *arg, struct ifnet *ifp,
121 uint16_t vid);
122 static struct octeon_device *
123 lio_get_other_octeon_device(struct octeon_device *oct);
124
125 static int lio_wait_for_oq_pkts(struct octeon_device *oct);
126
127 int lio_send_rss_param(struct lio *lio);
128 static int lio_dbg_console_print(struct octeon_device *oct,
129 uint32_t console_num, char *prefix,
130 char *suffix);
131
132 /* Polling interval for determining when NIC application is alive */
133 #define LIO_STARTER_POLL_INTERVAL_MS 100
134
135 /*
136 * vendor_info_array.
137 * This array contains the list of IDs on which the driver should load.
138 */
139 struct lio_vendor_info {
140 uint16_t vendor_id;
141 uint16_t device_id;
142 uint16_t subdevice_id;
143 uint8_t revision_id;
144 uint8_t index;
145 };
146
147 static struct lio_vendor_info lio_pci_tbl[] = {
148 /* CN2350 10G */
149 {PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_PF_VID, LIO_CN2350_10G_SUBDEVICE,
150 0x02, 0},
151
152 /* CN2350 10G */
153 {PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_PF_VID, LIO_CN2350_10G_SUBDEVICE1,
154 0x02, 0},
155
156 /* CN2360 10G */
157 {PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_PF_VID, LIO_CN2360_10G_SUBDEVICE,
158 0x02, 1},
159
160 /* CN2350 25G */
161 {PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_PF_VID, LIO_CN2350_25G_SUBDEVICE,
162 0x02, 2},
163
164 /* CN2360 25G */
165 {PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_PF_VID, LIO_CN2360_25G_SUBDEVICE,
166 0x02, 3},
167
168 {0, 0, 0, 0, 0}
169 };
170
171 static char *lio_strings[] = {
172 "LiquidIO 2350 10GbE Server Adapter",
173 "LiquidIO 2360 10GbE Server Adapter",
174 "LiquidIO 2350 25GbE Server Adapter",
175 "LiquidIO 2360 25GbE Server Adapter",
176 };
177
178 struct lio_if_cfg_resp {
179 uint64_t rh;
180 struct octeon_if_cfg_info cfg_info;
181 uint64_t status;
182 };
183
184 struct lio_if_cfg_context {
185 int octeon_id;
186 volatile int cond;
187 };
188
189 struct lio_rx_ctl_context {
190 int octeon_id;
191 volatile int cond;
192 };
193
194 static int
lio_probe(device_t dev)195 lio_probe(device_t dev)
196 {
197 struct lio_vendor_info *tbl;
198
199 uint16_t vendor_id;
200 uint16_t device_id;
201 uint16_t subdevice_id;
202 uint8_t revision_id;
203 char device_ver[256];
204
205 vendor_id = pci_get_vendor(dev);
206 if (vendor_id != PCI_VENDOR_ID_CAVIUM)
207 return (ENXIO);
208
209 device_id = pci_get_device(dev);
210 subdevice_id = pci_get_subdevice(dev);
211 revision_id = pci_get_revid(dev);
212
213 tbl = lio_pci_tbl;
214 while (tbl->vendor_id) {
215 if ((vendor_id == tbl->vendor_id) &&
216 (device_id == tbl->device_id) &&
217 (subdevice_id == tbl->subdevice_id) &&
218 (revision_id == tbl->revision_id)) {
219 sprintf(device_ver, "%s, Version - %s",
220 lio_strings[tbl->index], LIO_VERSION);
221 device_set_desc_copy(dev, device_ver);
222 return (BUS_PROBE_DEFAULT);
223 }
224
225 tbl++;
226 }
227
228 return (ENXIO);
229 }
230
231 static int
lio_attach(device_t device)232 lio_attach(device_t device)
233 {
234 struct octeon_device *oct_dev = NULL;
235 uint64_t scratch1;
236 uint32_t error;
237 int timeout, ret = 1;
238 uint8_t bus, dev, function;
239
240 oct_dev = lio_allocate_device(device);
241 if (oct_dev == NULL) {
242 device_printf(device, "Error: Unable to allocate device\n");
243 return (-ENOMEM);
244 }
245
246 oct_dev->tx_budget = LIO_DEFAULT_TX_PKTS_PROCESS_BUDGET;
247 oct_dev->rx_budget = LIO_DEFAULT_RX_PKTS_PROCESS_BUDGET;
248 oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED;
249
250 oct_dev->device = device;
251 bus = pci_get_bus(device);
252 dev = pci_get_slot(device);
253 function = pci_get_function(device);
254
255 lio_dev_info(oct_dev, "Initializing device %x:%x %02x:%02x.%01x\n",
256 pci_get_vendor(device), pci_get_device(device), bus, dev,
257 function);
258
259 if (lio_device_init(oct_dev)) {
260 lio_dev_err(oct_dev, "Failed to init device\n");
261 lio_detach(device);
262 return (-ENOMEM);
263 }
264
265 scratch1 = lio_read_csr64(oct_dev, LIO_CN23XX_SLI_SCRATCH1);
266 if (!(scratch1 & 4ULL)) {
267 /*
268 * Bit 2 of SLI_SCRATCH_1 is a flag that indicates that
269 * the lio watchdog kernel thread is running for this
270 * NIC. Each NIC gets one watchdog kernel thread.
271 */
272 scratch1 |= 4ULL;
273 lio_write_csr64(oct_dev, LIO_CN23XX_SLI_SCRATCH1, scratch1);
274
275 error = kproc_create(lio_watchdog, oct_dev,
276 &oct_dev->watchdog_task, 0, 0,
277 "liowd/%02hhx:%02hhx.%hhx", bus,
278 dev, function);
279 if (!error) {
280 kproc_resume(oct_dev->watchdog_task);
281 } else {
282 oct_dev->watchdog_task = NULL;
283 lio_dev_err(oct_dev,
284 "failed to create kernel_thread\n");
285 lio_detach(device);
286 return (-1);
287 }
288 }
289 oct_dev->rx_pause = 1;
290 oct_dev->tx_pause = 1;
291
292 timeout = 0;
293 while (timeout < LIO_NIC_STARTER_TIMEOUT) {
294 lio_mdelay(LIO_STARTER_POLL_INTERVAL_MS);
295 timeout += LIO_STARTER_POLL_INTERVAL_MS;
296
297 /*
298 * During the boot process interrupts are not available.
299 * So polling for first control message from FW.
300 */
301 if (cold)
302 lio_droq_bh(oct_dev->droq[0], 0);
303
304 if (atomic_load_acq_int(&oct_dev->status) == LIO_DEV_CORE_OK) {
305 ret = lio_nic_starter(oct_dev);
306 break;
307 }
308 }
309
310 if (ret) {
311 lio_dev_err(oct_dev, "Firmware failed to start\n");
312 lio_detach(device);
313 return (-EIO);
314 }
315
316 lio_dev_dbg(oct_dev, "Device is ready\n");
317
318 return (0);
319 }
320
321 static int
lio_detach(device_t dev)322 lio_detach(device_t dev)
323 {
324 struct octeon_device *oct_dev = device_get_softc(dev);
325
326 lio_dev_dbg(oct_dev, "Stopping device\n");
327 if (oct_dev->watchdog_task) {
328 uint64_t scratch1;
329
330 kproc_suspend(oct_dev->watchdog_task, 0);
331
332 scratch1 = lio_read_csr64(oct_dev, LIO_CN23XX_SLI_SCRATCH1);
333 scratch1 &= ~4ULL;
334 lio_write_csr64(oct_dev, LIO_CN23XX_SLI_SCRATCH1, scratch1);
335 }
336
337 if (oct_dev->app_mode && (oct_dev->app_mode == LIO_DRV_NIC_APP))
338 lio_stop_nic_module(oct_dev);
339
340 /*
341 * Reset the octeon device and cleanup all memory allocated for
342 * the octeon device by driver.
343 */
344 lio_destroy_resources(oct_dev);
345
346 lio_dev_info(oct_dev, "Device removed\n");
347
348 /*
349 * This octeon device has been removed. Update the global
350 * data structure to reflect this. Free the device structure.
351 */
352 lio_free_device_mem(oct_dev);
353 return (0);
354 }
355
356 static int
lio_shutdown(device_t dev)357 lio_shutdown(device_t dev)
358 {
359 struct octeon_device *oct_dev = device_get_softc(dev);
360 struct lio *lio = if_getsoftc(oct_dev->props.ifp);
361
362 lio_send_rx_ctrl_cmd(lio, 0);
363
364 return (0);
365 }
366
367 static int
lio_suspend(device_t dev)368 lio_suspend(device_t dev)
369 {
370
371 return (ENXIO);
372 }
373
374 static int
lio_resume(device_t dev)375 lio_resume(device_t dev)
376 {
377
378 return (ENXIO);
379 }
380
381 static int
lio_event(struct module * mod,int event,void * junk)382 lio_event(struct module *mod, int event, void *junk)
383 {
384
385 switch (event) {
386 case MOD_LOAD:
387 lio_init_device_list(LIO_CFG_TYPE_DEFAULT);
388 break;
389 default:
390 break;
391 }
392
393 return (0);
394 }
395
396 /*********************************************************************
397 * FreeBSD Device Interface Entry Points
398 * *******************************************************************/
399 static device_method_t lio_methods[] = {
400 /* Device interface */
401 DEVMETHOD(device_probe, lio_probe),
402 DEVMETHOD(device_attach, lio_attach),
403 DEVMETHOD(device_detach, lio_detach),
404 DEVMETHOD(device_shutdown, lio_shutdown),
405 DEVMETHOD(device_suspend, lio_suspend),
406 DEVMETHOD(device_resume, lio_resume),
407 DEVMETHOD_END
408 };
409
410 static driver_t lio_driver = {
411 LIO_DRV_NAME, lio_methods, sizeof(struct octeon_device),
412 };
413
414 devclass_t lio_devclass;
415 DRIVER_MODULE(lio, pci, lio_driver, lio_devclass, lio_event, 0);
416
417 MODULE_DEPEND(lio, pci, 1, 1, 1);
418 MODULE_DEPEND(lio, ether, 1, 1, 1);
419 MODULE_DEPEND(lio, firmware, 1, 1, 1);
420
421 static bool
fw_type_is_none(void)422 fw_type_is_none(void)
423 {
424 return strncmp(fw_type, LIO_FW_NAME_TYPE_NONE,
425 sizeof(LIO_FW_NAME_TYPE_NONE)) == 0;
426 }
427
428 /*
429 * \brief Device initialization for each Octeon device that is probed
430 * @param octeon_dev octeon device
431 */
432 static int
lio_device_init(struct octeon_device * octeon_dev)433 lio_device_init(struct octeon_device *octeon_dev)
434 {
435 unsigned long ddr_timeout = LIO_DDR_TIMEOUT;
436 char *dbg_enb = NULL;
437 int fw_loaded = 0;
438 int i, j, ret;
439 uint8_t bus, dev, function;
440 char bootcmd[] = "\n";
441
442 bus = pci_get_bus(octeon_dev->device);
443 dev = pci_get_slot(octeon_dev->device);
444 function = pci_get_function(octeon_dev->device);
445
446 atomic_store_rel_int(&octeon_dev->status, LIO_DEV_BEGIN_STATE);
447
448 /* Enable access to the octeon device */
449 if (pci_enable_busmaster(octeon_dev->device)) {
450 lio_dev_err(octeon_dev, "pci_enable_device failed\n");
451 return (1);
452 }
453
454 atomic_store_rel_int(&octeon_dev->status, LIO_DEV_PCI_ENABLE_DONE);
455
456 /* Identify the Octeon type and map the BAR address space. */
457 if (lio_chip_specific_setup(octeon_dev)) {
458 lio_dev_err(octeon_dev, "Chip specific setup failed\n");
459 return (1);
460 }
461
462 atomic_store_rel_int(&octeon_dev->status, LIO_DEV_PCI_MAP_DONE);
463
464 /*
465 * Only add a reference after setting status 'OCT_DEV_PCI_MAP_DONE',
466 * since that is what is required for the reference to be removed
467 * during de-initialization (see 'octeon_destroy_resources').
468 */
469 lio_register_device(octeon_dev, bus, dev, function, true);
470
471
472 octeon_dev->app_mode = LIO_DRV_INVALID_APP;
473
474 if (!lio_cn23xx_pf_fw_loaded(octeon_dev) && !fw_type_is_none()) {
475 fw_loaded = 0;
476 /* Do a soft reset of the Octeon device. */
477 if (octeon_dev->fn_list.soft_reset(octeon_dev))
478 return (1);
479
480 /* things might have changed */
481 if (!lio_cn23xx_pf_fw_loaded(octeon_dev))
482 fw_loaded = 0;
483 else
484 fw_loaded = 1;
485 } else {
486 fw_loaded = 1;
487 }
488
489 /*
490 * Initialize the dispatch mechanism used to push packets arriving on
491 * Octeon Output queues.
492 */
493 if (lio_init_dispatch_list(octeon_dev))
494 return (1);
495
496 lio_register_dispatch_fn(octeon_dev, LIO_OPCODE_NIC,
497 LIO_OPCODE_NIC_CORE_DRV_ACTIVE,
498 lio_core_drv_init, octeon_dev);
499 atomic_store_rel_int(&octeon_dev->status, LIO_DEV_DISPATCH_INIT_DONE);
500
501 ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
502 if (ret) {
503 lio_dev_err(octeon_dev,
504 "Failed to configure device registers\n");
505 return (ret);
506 }
507
508 /* Initialize soft command buffer pool */
509 if (lio_setup_sc_buffer_pool(octeon_dev)) {
510 lio_dev_err(octeon_dev, "sc buffer pool allocation failed\n");
511 return (1);
512 }
513
514 atomic_store_rel_int(&octeon_dev->status,
515 LIO_DEV_SC_BUFF_POOL_INIT_DONE);
516
517 if (lio_allocate_ioq_vector(octeon_dev)) {
518 lio_dev_err(octeon_dev,
519 "IOQ vector allocation failed\n");
520 return (1);
521 }
522
523 atomic_store_rel_int(&octeon_dev->status,
524 LIO_DEV_MSIX_ALLOC_VECTOR_DONE);
525
526 for (i = 0; i < LIO_MAX_POSSIBLE_INSTR_QUEUES; i++) {
527 octeon_dev->instr_queue[i] =
528 malloc(sizeof(struct lio_instr_queue),
529 M_DEVBUF, M_NOWAIT | M_ZERO);
530 if (octeon_dev->instr_queue[i] == NULL)
531 return (1);
532 }
533
534 /* Setup the data structures that manage this Octeon's Input queues. */
535 if (lio_setup_instr_queue0(octeon_dev)) {
536 lio_dev_err(octeon_dev,
537 "Instruction queue initialization failed\n");
538 return (1);
539 }
540
541 atomic_store_rel_int(&octeon_dev->status,
542 LIO_DEV_INSTR_QUEUE_INIT_DONE);
543
544 /*
545 * Initialize lists to manage the requests of different types that
546 * arrive from user & kernel applications for this octeon device.
547 */
548
549 if (lio_setup_response_list(octeon_dev)) {
550 lio_dev_err(octeon_dev, "Response list allocation failed\n");
551 return (1);
552 }
553
554 atomic_store_rel_int(&octeon_dev->status, LIO_DEV_RESP_LIST_INIT_DONE);
555
556 for (i = 0; i < LIO_MAX_POSSIBLE_OUTPUT_QUEUES; i++) {
557 octeon_dev->droq[i] = malloc(sizeof(*octeon_dev->droq[i]),
558 M_DEVBUF, M_NOWAIT | M_ZERO);
559 if (octeon_dev->droq[i] == NULL)
560 return (1);
561 }
562
563 if (lio_setup_output_queue0(octeon_dev)) {
564 lio_dev_err(octeon_dev, "Output queue initialization failed\n");
565 return (1);
566 }
567
568 atomic_store_rel_int(&octeon_dev->status, LIO_DEV_DROQ_INIT_DONE);
569
570 /*
571 * Setup the interrupt handler and record the INT SUM register address
572 */
573 if (lio_setup_interrupt(octeon_dev,
574 octeon_dev->sriov_info.num_pf_rings))
575 return (1);
576
577 /* Enable Octeon device interrupts */
578 octeon_dev->fn_list.enable_interrupt(octeon_dev, OCTEON_ALL_INTR);
579
580 atomic_store_rel_int(&octeon_dev->status, LIO_DEV_INTR_SET_DONE);
581
582 /*
583 * Send Credit for Octeon Output queues. Credits are always sent BEFORE
584 * the output queue is enabled.
585 * This ensures that we'll receive the f/w CORE DRV_ACTIVE message in
586 * case we've configured CN23XX_SLI_GBL_CONTROL[NOPTR_D] = 0.
587 * Otherwise, it is possible that the DRV_ACTIVE message will be sent
588 * before any credits have been issued, causing the ring to be reset
589 * (and the f/w appear to never have started).
590 */
591 for (j = 0; j < octeon_dev->num_oqs; j++)
592 lio_write_csr32(octeon_dev,
593 octeon_dev->droq[j]->pkts_credit_reg,
594 octeon_dev->droq[j]->max_count);
595
596 /* Enable the input and output queues for this Octeon device */
597 ret = octeon_dev->fn_list.enable_io_queues(octeon_dev);
598 if (ret) {
599 lio_dev_err(octeon_dev, "Failed to enable input/output queues");
600 return (ret);
601 }
602
603 atomic_store_rel_int(&octeon_dev->status, LIO_DEV_IO_QUEUES_DONE);
604
605 if (!fw_loaded) {
606 lio_dev_dbg(octeon_dev, "Waiting for DDR initialization...\n");
607 if (!ddr_timeout) {
608 lio_dev_info(octeon_dev,
609 "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
610 }
611
612 lio_sleep_timeout(LIO_RESET_MSECS);
613
614 /*
615 * Wait for the octeon to initialize DDR after the
616 * soft-reset.
617 */
618 while (!ddr_timeout) {
619 if (pause("-", lio_ms_to_ticks(100))) {
620 /* user probably pressed Control-C */
621 return (1);
622 }
623 }
624
625 ret = lio_wait_for_ddr_init(octeon_dev, &ddr_timeout);
626 if (ret) {
627 lio_dev_err(octeon_dev,
628 "DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",
629 ret);
630 return (1);
631 }
632
633 if (lio_wait_for_bootloader(octeon_dev, 1100)) {
634 lio_dev_err(octeon_dev, "Board not responding\n");
635 return (1);
636 }
637
638 /* Divert uboot to take commands from host instead. */
639 ret = lio_console_send_cmd(octeon_dev, bootcmd, 50);
640
641 lio_dev_dbg(octeon_dev, "Initializing consoles\n");
642 ret = lio_init_consoles(octeon_dev);
643 if (ret) {
644 lio_dev_err(octeon_dev, "Could not access board consoles\n");
645 return (1);
646 }
647
648 /*
649 * If console debug enabled, specify empty string to
650 * use default enablement ELSE specify NULL string for
651 * 'disabled'.
652 */
653 dbg_enb = lio_console_debug_enabled(0) ? "" : NULL;
654 ret = lio_add_console(octeon_dev, 0, dbg_enb);
655
656 if (ret) {
657 lio_dev_err(octeon_dev, "Could not access board console\n");
658 return (1);
659 } else if (lio_console_debug_enabled(0)) {
660 /*
661 * If console was added AND we're logging console output
662 * then set our console print function.
663 */
664 octeon_dev->console[0].print = lio_dbg_console_print;
665 }
666
667 atomic_store_rel_int(&octeon_dev->status,
668 LIO_DEV_CONSOLE_INIT_DONE);
669
670 lio_dev_dbg(octeon_dev, "Loading firmware\n");
671
672 ret = lio_load_firmware(octeon_dev);
673 if (ret) {
674 lio_dev_err(octeon_dev, "Could not load firmware to board\n");
675 return (1);
676 }
677 }
678
679 atomic_store_rel_int(&octeon_dev->status, LIO_DEV_HOST_OK);
680
681 return (0);
682 }
683
684 /*
685 * \brief PCI FLR for each Octeon device.
686 * @param oct octeon device
687 */
688 static void
lio_pci_flr(struct octeon_device * oct)689 lio_pci_flr(struct octeon_device *oct)
690 {
691 uint32_t exppos, status;
692
693 pci_find_cap(oct->device, PCIY_EXPRESS, &exppos);
694
695 pci_save_state(oct->device);
696
697 /* Quiesce the device completely */
698 pci_write_config(oct->device, PCIR_COMMAND, PCIM_CMD_INTxDIS, 2);
699
700 /* Wait for Transaction Pending bit clean */
701 lio_mdelay(100);
702
703 status = pci_read_config(oct->device, exppos + PCIER_DEVICE_STA, 2);
704 if (status & PCIEM_STA_TRANSACTION_PND) {
705 lio_dev_info(oct, "Function reset incomplete after 100ms, sleeping for 5 seconds\n");
706 lio_mdelay(5);
707
708 status = pci_read_config(oct->device, exppos + PCIER_DEVICE_STA, 2);
709 if (status & PCIEM_STA_TRANSACTION_PND)
710 lio_dev_info(oct, "Function reset still incomplete after 5s, reset anyway\n");
711 }
712
713 pci_write_config(oct->device, exppos + PCIER_DEVICE_CTL, PCIEM_CTL_INITIATE_FLR, 2);
714 lio_mdelay(100);
715
716 pci_restore_state(oct->device);
717 }
718
719 /*
720 * \brief Debug console print function
721 * @param octeon_dev octeon device
722 * @param console_num console number
723 * @param prefix first portion of line to display
724 * @param suffix second portion of line to display
725 *
726 * The OCTEON debug console outputs entire lines (excluding '\n').
727 * Normally, the line will be passed in the 'prefix' parameter.
728 * However, due to buffering, it is possible for a line to be split into two
729 * parts, in which case they will be passed as the 'prefix' parameter and
730 * 'suffix' parameter.
731 */
732 static int
lio_dbg_console_print(struct octeon_device * oct,uint32_t console_num,char * prefix,char * suffix)733 lio_dbg_console_print(struct octeon_device *oct, uint32_t console_num,
734 char *prefix, char *suffix)
735 {
736
737 if (prefix != NULL && suffix != NULL)
738 lio_dev_info(oct, "%u: %s%s\n", console_num, prefix, suffix);
739 else if (prefix != NULL)
740 lio_dev_info(oct, "%u: %s\n", console_num, prefix);
741 else if (suffix != NULL)
742 lio_dev_info(oct, "%u: %s\n", console_num, suffix);
743
744 return (0);
745 }
746
747 static void
lio_watchdog(void * param)748 lio_watchdog(void *param)
749 {
750 int core_num;
751 uint16_t mask_of_crashed_or_stuck_cores = 0;
752 struct octeon_device *oct = param;
753 bool err_msg_was_printed[12];
754
755 bzero(err_msg_was_printed, sizeof(err_msg_was_printed));
756
757 while (1) {
758 kproc_suspend_check(oct->watchdog_task);
759 mask_of_crashed_or_stuck_cores =
760 (uint16_t)lio_read_csr64(oct, LIO_CN23XX_SLI_SCRATCH2);
761
762 if (mask_of_crashed_or_stuck_cores) {
763 struct octeon_device *other_oct;
764
765 oct->cores_crashed = true;
766 other_oct = lio_get_other_octeon_device(oct);
767 if (other_oct != NULL)
768 other_oct->cores_crashed = true;
769
770 for (core_num = 0; core_num < LIO_MAX_CORES;
771 core_num++) {
772 bool core_crashed_or_got_stuck;
773
774 core_crashed_or_got_stuck =
775 (mask_of_crashed_or_stuck_cores >>
776 core_num) & 1;
777 if (core_crashed_or_got_stuck &&
778 !err_msg_was_printed[core_num]) {
779 lio_dev_err(oct,
780 "ERROR: Octeon core %d crashed or got stuck! See oct-fwdump for details.\n",
781 core_num);
782 err_msg_was_printed[core_num] = true;
783 }
784 }
785
786 }
787
788 /* sleep for two seconds */
789 pause("-", lio_ms_to_ticks(2000));
790 }
791 }
792
793 static int
lio_chip_specific_setup(struct octeon_device * oct)794 lio_chip_specific_setup(struct octeon_device *oct)
795 {
796 char *s;
797 uint32_t dev_id, rev_id;
798 int ret = 1;
799
800 dev_id = lio_read_pci_cfg(oct, 0);
801 rev_id = pci_get_revid(oct->device);
802 oct->subdevice_id = pci_get_subdevice(oct->device);
803
804 switch (dev_id) {
805 case LIO_CN23XX_PF_PCIID:
806 oct->chip_id = LIO_CN23XX_PF_VID;
807 if (pci_get_function(oct->device) == 0) {
808 if (num_queues_per_pf0 < 0) {
809 lio_dev_info(oct, "Invalid num_queues_per_pf0: %d, Setting it to default\n",
810 num_queues_per_pf0);
811 num_queues_per_pf0 = 0;
812 }
813
814 oct->sriov_info.num_pf_rings = num_queues_per_pf0;
815 } else {
816 if (num_queues_per_pf1 < 0) {
817 lio_dev_info(oct, "Invalid num_queues_per_pf1: %d, Setting it to default\n",
818 num_queues_per_pf1);
819 num_queues_per_pf1 = 0;
820 }
821
822 oct->sriov_info.num_pf_rings = num_queues_per_pf1;
823 }
824
825 ret = lio_cn23xx_pf_setup_device(oct);
826 s = "CN23XX";
827 break;
828
829 default:
830 s = "?";
831 lio_dev_err(oct, "Unknown device found (dev_id: %x)\n", dev_id);
832 }
833
834 if (!ret)
835 lio_dev_info(oct, "%s PASS%d.%d %s Version: %s\n", s,
836 OCTEON_MAJOR_REV(oct), OCTEON_MINOR_REV(oct),
837 lio_get_conf(oct)->card_name, LIO_VERSION);
838
839 return (ret);
840 }
841
842 static struct octeon_device *
lio_get_other_octeon_device(struct octeon_device * oct)843 lio_get_other_octeon_device(struct octeon_device *oct)
844 {
845 struct octeon_device *other_oct;
846
847 other_oct = lio_get_device(oct->octeon_id + 1);
848
849 if ((other_oct != NULL) && other_oct->device) {
850 int oct_busnum, other_oct_busnum;
851
852 oct_busnum = pci_get_bus(oct->device);
853 other_oct_busnum = pci_get_bus(other_oct->device);
854
855 if (oct_busnum == other_oct_busnum) {
856 int oct_slot, other_oct_slot;
857
858 oct_slot = pci_get_slot(oct->device);
859 other_oct_slot = pci_get_slot(other_oct->device);
860
861 if (oct_slot == other_oct_slot)
862 return (other_oct);
863 }
864 }
865 return (NULL);
866 }
867
868 /*
869 * \brief Load firmware to device
870 * @param oct octeon device
871 *
872 * Maps device to firmware filename, requests firmware, and downloads it
873 */
874 static int
lio_load_firmware(struct octeon_device * oct)875 lio_load_firmware(struct octeon_device *oct)
876 {
877 const struct firmware *fw;
878 char *tmp_fw_type = NULL;
879 int ret = 0;
880 char fw_name[LIO_MAX_FW_FILENAME_LEN];
881
882 if (fw_type[0] == '\0')
883 tmp_fw_type = LIO_FW_NAME_TYPE_NIC;
884 else
885 tmp_fw_type = fw_type;
886
887 sprintf(fw_name, "%s%s_%s%s", LIO_FW_BASE_NAME,
888 lio_get_conf(oct)->card_name, tmp_fw_type, LIO_FW_NAME_SUFFIX);
889
890 fw = firmware_get(fw_name);
891 if (fw == NULL) {
892 lio_dev_err(oct, "Request firmware failed. Could not find file %s.\n",
893 fw_name);
894 return (EINVAL);
895 }
896
897 ret = lio_download_firmware(oct, fw->data, fw->datasize);
898
899 firmware_put(fw, FIRMWARE_UNLOAD);
900
901 return (ret);
902 }
903
904 static int
lio_nic_starter(struct octeon_device * oct)905 lio_nic_starter(struct octeon_device *oct)
906 {
907 int ret = 0;
908
909 atomic_store_rel_int(&oct->status, LIO_DEV_RUNNING);
910
911 if (oct->app_mode && oct->app_mode == LIO_DRV_NIC_APP) {
912 if (lio_init_nic_module(oct)) {
913 lio_dev_err(oct, "NIC initialization failed\n");
914 ret = -1;
915 #ifdef CAVIUM_ONiLY_23XX_VF
916 } else {
917 if (octeon_enable_sriov(oct) < 0)
918 ret = -1;
919 #endif
920 }
921 } else {
922 lio_dev_err(oct,
923 "Unexpected application running on NIC (%d). Check firmware.\n",
924 oct->app_mode);
925 ret = -1;
926 }
927
928 return (ret);
929 }
930
931 static int
lio_init_nic_module(struct octeon_device * oct)932 lio_init_nic_module(struct octeon_device *oct)
933 {
934 int num_nic_ports = LIO_GET_NUM_NIC_PORTS_CFG(lio_get_conf(oct));
935 int retval = 0;
936
937 lio_dev_dbg(oct, "Initializing network interfaces\n");
938
939 /*
940 * only default iq and oq were initialized
941 * initialize the rest as well
942 */
943
944 /* run port_config command for each port */
945 oct->ifcount = num_nic_ports;
946
947 bzero(&oct->props, sizeof(struct lio_if_props));
948
949 oct->props.gmxport = -1;
950
951 retval = lio_setup_nic_devices(oct);
952 if (retval) {
953 lio_dev_err(oct, "Setup NIC devices failed\n");
954 goto lio_init_failure;
955 }
956
957 lio_dev_dbg(oct, "Network interfaces ready\n");
958
959 return (retval);
960
961 lio_init_failure:
962
963 oct->ifcount = 0;
964
965 return (retval);
966 }
967
968 static int
lio_ifmedia_update(struct ifnet * ifp)969 lio_ifmedia_update(struct ifnet *ifp)
970 {
971 struct lio *lio = if_getsoftc(ifp);
972 struct ifmedia *ifm;
973
974 ifm = &lio->ifmedia;
975
976 /* We only support Ethernet media type. */
977 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
978 return (EINVAL);
979
980 switch (IFM_SUBTYPE(ifm->ifm_media)) {
981 case IFM_AUTO:
982 break;
983 case IFM_10G_CX4:
984 case IFM_10G_SR:
985 case IFM_10G_T:
986 case IFM_10G_TWINAX:
987 default:
988 /* We don't support changing the media type. */
989 lio_dev_err(lio->oct_dev, "Invalid media type (%d)\n",
990 IFM_SUBTYPE(ifm->ifm_media));
991 return (EINVAL);
992 }
993
994 return (0);
995 }
996
997 static int
lio_get_media_subtype(struct octeon_device * oct)998 lio_get_media_subtype(struct octeon_device *oct)
999 {
1000
1001 switch(oct->subdevice_id) {
1002 case LIO_CN2350_10G_SUBDEVICE:
1003 case LIO_CN2350_10G_SUBDEVICE1:
1004 case LIO_CN2360_10G_SUBDEVICE:
1005 return (IFM_10G_SR);
1006
1007 case LIO_CN2350_25G_SUBDEVICE:
1008 case LIO_CN2360_25G_SUBDEVICE:
1009 return (IFM_25G_SR);
1010 }
1011
1012 return (IFM_10G_SR);
1013 }
1014
1015 static uint64_t
lio_get_baudrate(struct octeon_device * oct)1016 lio_get_baudrate(struct octeon_device *oct)
1017 {
1018
1019 switch(oct->subdevice_id) {
1020 case LIO_CN2350_10G_SUBDEVICE:
1021 case LIO_CN2350_10G_SUBDEVICE1:
1022 case LIO_CN2360_10G_SUBDEVICE:
1023 return (IF_Gbps(10));
1024
1025 case LIO_CN2350_25G_SUBDEVICE:
1026 case LIO_CN2360_25G_SUBDEVICE:
1027 return (IF_Gbps(25));
1028 }
1029
1030 return (IF_Gbps(10));
1031 }
1032
1033 static void
lio_ifmedia_status(struct ifnet * ifp,struct ifmediareq * ifmr)1034 lio_ifmedia_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1035 {
1036 struct lio *lio = if_getsoftc(ifp);
1037
1038 /* Report link down if the driver isn't running. */
1039 if (!lio_ifstate_check(lio, LIO_IFSTATE_RUNNING)) {
1040 ifmr->ifm_active |= IFM_NONE;
1041 return;
1042 }
1043
1044 /* Setup the default interface info. */
1045 ifmr->ifm_status = IFM_AVALID;
1046 ifmr->ifm_active = IFM_ETHER;
1047
1048 if (lio->linfo.link.s.link_up) {
1049 ifmr->ifm_status |= IFM_ACTIVE;
1050 } else {
1051 ifmr->ifm_active |= IFM_NONE;
1052 return;
1053 }
1054
1055 ifmr->ifm_active |= lio_get_media_subtype(lio->oct_dev);
1056
1057 if (lio->linfo.link.s.duplex)
1058 ifmr->ifm_active |= IFM_FDX;
1059 else
1060 ifmr->ifm_active |= IFM_HDX;
1061 }
1062
1063 static uint64_t
lio_get_counter(if_t ifp,ift_counter cnt)1064 lio_get_counter(if_t ifp, ift_counter cnt)
1065 {
1066 struct lio *lio = if_getsoftc(ifp);
1067 struct octeon_device *oct = lio->oct_dev;
1068 uint64_t counter = 0;
1069 int i, q_no;
1070
1071 switch (cnt) {
1072 case IFCOUNTER_IPACKETS:
1073 for (i = 0; i < oct->num_oqs; i++) {
1074 q_no = lio->linfo.rxpciq[i].s.q_no;
1075 counter += oct->droq[q_no]->stats.rx_pkts_received;
1076 }
1077 break;
1078 case IFCOUNTER_OPACKETS:
1079 for (i = 0; i < oct->num_iqs; i++) {
1080 q_no = lio->linfo.txpciq[i].s.q_no;
1081 counter += oct->instr_queue[q_no]->stats.tx_done;
1082 }
1083 break;
1084 case IFCOUNTER_IBYTES:
1085 for (i = 0; i < oct->num_oqs; i++) {
1086 q_no = lio->linfo.rxpciq[i].s.q_no;
1087 counter += oct->droq[q_no]->stats.rx_bytes_received;
1088 }
1089 break;
1090 case IFCOUNTER_OBYTES:
1091 for (i = 0; i < oct->num_iqs; i++) {
1092 q_no = lio->linfo.txpciq[i].s.q_no;
1093 counter += oct->instr_queue[q_no]->stats.tx_tot_bytes;
1094 }
1095 break;
1096 case IFCOUNTER_IQDROPS:
1097 for (i = 0; i < oct->num_oqs; i++) {
1098 q_no = lio->linfo.rxpciq[i].s.q_no;
1099 counter += oct->droq[q_no]->stats.rx_dropped;
1100 }
1101 break;
1102 case IFCOUNTER_OQDROPS:
1103 for (i = 0; i < oct->num_iqs; i++) {
1104 q_no = lio->linfo.txpciq[i].s.q_no;
1105 counter += oct->instr_queue[q_no]->stats.tx_dropped;
1106 }
1107 break;
1108 case IFCOUNTER_IMCASTS:
1109 counter = oct->link_stats.fromwire.total_mcst;
1110 break;
1111 case IFCOUNTER_OMCASTS:
1112 counter = oct->link_stats.fromhost.mcast_pkts_sent;
1113 break;
1114 case IFCOUNTER_COLLISIONS:
1115 counter = oct->link_stats.fromhost.total_collisions;
1116 break;
1117 case IFCOUNTER_IERRORS:
1118 counter = oct->link_stats.fromwire.fcs_err +
1119 oct->link_stats.fromwire.l2_err +
1120 oct->link_stats.fromwire.frame_err;
1121 break;
1122 default:
1123 return (if_get_counter_default(ifp, cnt));
1124 }
1125
1126 return (counter);
1127 }
1128
1129 static int
lio_init_ifnet(struct lio * lio)1130 lio_init_ifnet(struct lio *lio)
1131 {
1132 struct octeon_device *oct = lio->oct_dev;
1133 if_t ifp = lio->ifp;
1134
1135 /* ifconfig entrypoint for media type/status reporting */
1136 ifmedia_init(&lio->ifmedia, IFM_IMASK, lio_ifmedia_update,
1137 lio_ifmedia_status);
1138
1139 /* set the default interface values */
1140 ifmedia_add(&lio->ifmedia,
1141 (IFM_ETHER | IFM_FDX | lio_get_media_subtype(oct)),
1142 0, NULL);
1143 ifmedia_add(&lio->ifmedia, (IFM_ETHER | IFM_AUTO), 0, NULL);
1144 ifmedia_set(&lio->ifmedia, (IFM_ETHER | IFM_AUTO));
1145
1146 lio->ifmedia.ifm_media = lio->ifmedia.ifm_cur->ifm_media;
1147 lio_dev_dbg(oct, "IFMEDIA flags : %x\n", lio->ifmedia.ifm_media);
1148
1149 if_initname(ifp, device_get_name(oct->device),
1150 device_get_unit(oct->device));
1151 if_setflags(ifp, (IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST));
1152 if_setioctlfn(ifp, lio_ioctl);
1153 if_setgetcounterfn(ifp, lio_get_counter);
1154 if_settransmitfn(ifp, lio_mq_start);
1155 if_setqflushfn(ifp, lio_qflush);
1156 if_setinitfn(ifp, lio_open);
1157 if_setmtu(ifp, lio->linfo.link.s.mtu);
1158 lio->mtu = lio->linfo.link.s.mtu;
1159 if_sethwassist(ifp, (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_TSO |
1160 CSUM_TCP_IPV6 | CSUM_UDP_IPV6));
1161
1162 if_setcapabilitiesbit(ifp, (IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 |
1163 IFCAP_TSO | IFCAP_LRO |
1164 IFCAP_JUMBO_MTU | IFCAP_HWSTATS |
1165 IFCAP_LINKSTATE | IFCAP_VLAN_HWFILTER |
1166 IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWTAGGING |
1167 IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU), 0);
1168
1169 if_setcapenable(ifp, if_getcapabilities(ifp));
1170 if_setbaudrate(ifp, lio_get_baudrate(oct));
1171
1172 return (0);
1173 }
1174
1175 static void
lio_tcp_lro_free(struct octeon_device * octeon_dev,struct ifnet * ifp)1176 lio_tcp_lro_free(struct octeon_device *octeon_dev, struct ifnet *ifp)
1177 {
1178 struct lio *lio = if_getsoftc(ifp);
1179 struct lio_droq *droq;
1180 int q_no;
1181 int i;
1182
1183 for (i = 0; i < octeon_dev->num_oqs; i++) {
1184 q_no = lio->linfo.rxpciq[i].s.q_no;
1185 droq = octeon_dev->droq[q_no];
1186 if (droq->lro.ifp) {
1187 tcp_lro_free(&droq->lro);
1188 droq->lro.ifp = NULL;
1189 }
1190 }
1191 }
1192
1193 static int
lio_tcp_lro_init(struct octeon_device * octeon_dev,struct ifnet * ifp)1194 lio_tcp_lro_init(struct octeon_device *octeon_dev, struct ifnet *ifp)
1195 {
1196 struct lio *lio = if_getsoftc(ifp);
1197 struct lio_droq *droq;
1198 struct lro_ctrl *lro;
1199 int i, q_no, ret = 0;
1200
1201 for (i = 0; i < octeon_dev->num_oqs; i++) {
1202 q_no = lio->linfo.rxpciq[i].s.q_no;
1203 droq = octeon_dev->droq[q_no];
1204 lro = &droq->lro;
1205 ret = tcp_lro_init(lro);
1206 if (ret) {
1207 lio_dev_err(octeon_dev, "LRO Initialization failed ret %d\n",
1208 ret);
1209 goto lro_init_failed;
1210 }
1211
1212 lro->ifp = ifp;
1213 }
1214
1215 return (ret);
1216
1217 lro_init_failed:
1218 lio_tcp_lro_free(octeon_dev, ifp);
1219
1220 return (ret);
1221 }
1222
1223 static int
lio_setup_nic_devices(struct octeon_device * octeon_dev)1224 lio_setup_nic_devices(struct octeon_device *octeon_dev)
1225 {
1226 union octeon_if_cfg if_cfg;
1227 struct lio *lio = NULL;
1228 struct ifnet *ifp = NULL;
1229 struct lio_version *vdata;
1230 struct lio_soft_command *sc;
1231 struct lio_if_cfg_context *ctx;
1232 struct lio_if_cfg_resp *resp;
1233 struct lio_if_props *props;
1234 int num_iqueues, num_oqueues, retval;
1235 unsigned int base_queue;
1236 unsigned int gmx_port_id;
1237 uint32_t ctx_size, data_size;
1238 uint32_t ifidx_or_pfnum, resp_size;
1239 uint8_t mac[ETHER_HDR_LEN], i, j;
1240
1241 /* This is to handle link status changes */
1242 lio_register_dispatch_fn(octeon_dev, LIO_OPCODE_NIC,
1243 LIO_OPCODE_NIC_INFO,
1244 lio_link_info, octeon_dev);
1245
1246 for (i = 0; i < octeon_dev->ifcount; i++) {
1247 resp_size = sizeof(struct lio_if_cfg_resp);
1248 ctx_size = sizeof(struct lio_if_cfg_context);
1249 data_size = sizeof(struct lio_version);
1250 sc = lio_alloc_soft_command(octeon_dev, data_size, resp_size,
1251 ctx_size);
1252 if (sc == NULL)
1253 return (ENOMEM);
1254
1255 resp = (struct lio_if_cfg_resp *)sc->virtrptr;
1256 ctx = (struct lio_if_cfg_context *)sc->ctxptr;
1257 vdata = (struct lio_version *)sc->virtdptr;
1258
1259 *((uint64_t *)vdata) = 0;
1260 vdata->major = htobe16(LIO_BASE_MAJOR_VERSION);
1261 vdata->minor = htobe16(LIO_BASE_MINOR_VERSION);
1262 vdata->micro = htobe16(LIO_BASE_MICRO_VERSION);
1263
1264 num_iqueues = octeon_dev->sriov_info.num_pf_rings;
1265 num_oqueues = octeon_dev->sriov_info.num_pf_rings;
1266 base_queue = octeon_dev->sriov_info.pf_srn;
1267
1268 gmx_port_id = octeon_dev->pf_num;
1269 ifidx_or_pfnum = octeon_dev->pf_num;
1270
1271 lio_dev_dbg(octeon_dev, "requesting config for interface %d, iqs %d, oqs %d\n",
1272 ifidx_or_pfnum, num_iqueues, num_oqueues);
1273 ctx->cond = 0;
1274 ctx->octeon_id = lio_get_device_id(octeon_dev);
1275
1276 if_cfg.if_cfg64 = 0;
1277 if_cfg.s.num_iqueues = num_iqueues;
1278 if_cfg.s.num_oqueues = num_oqueues;
1279 if_cfg.s.base_queue = base_queue;
1280 if_cfg.s.gmx_port_id = gmx_port_id;
1281
1282 sc->iq_no = 0;
1283
1284 lio_prepare_soft_command(octeon_dev, sc, LIO_OPCODE_NIC,
1285 LIO_OPCODE_NIC_IF_CFG, 0,
1286 if_cfg.if_cfg64, 0);
1287
1288 sc->callback = lio_if_cfg_callback;
1289 sc->callback_arg = sc;
1290 sc->wait_time = 3000;
1291
1292 retval = lio_send_soft_command(octeon_dev, sc);
1293 if (retval == LIO_IQ_SEND_FAILED) {
1294 lio_dev_err(octeon_dev, "iq/oq config failed status: %x\n",
1295 retval);
1296 /* Soft instr is freed by driver in case of failure. */
1297 goto setup_nic_dev_fail;
1298 }
1299
1300 /*
1301 * Sleep on a wait queue till the cond flag indicates that the
1302 * response arrived or timed-out.
1303 */
1304 lio_sleep_cond(octeon_dev, &ctx->cond);
1305
1306 retval = resp->status;
1307 if (retval) {
1308 lio_dev_err(octeon_dev, "iq/oq config failed\n");
1309 goto setup_nic_dev_fail;
1310 }
1311
1312 lio_swap_8B_data((uint64_t *)(&resp->cfg_info),
1313 (sizeof(struct octeon_if_cfg_info)) >> 3);
1314
1315 num_iqueues = bitcount64(resp->cfg_info.iqmask);
1316 num_oqueues = bitcount64(resp->cfg_info.oqmask);
1317
1318 if (!(num_iqueues) || !(num_oqueues)) {
1319 lio_dev_err(octeon_dev,
1320 "Got bad iqueues (%016llX) or oqueues (%016llX) from firmware.\n",
1321 LIO_CAST64(resp->cfg_info.iqmask),
1322 LIO_CAST64(resp->cfg_info.oqmask));
1323 goto setup_nic_dev_fail;
1324 }
1325
1326 lio_dev_dbg(octeon_dev,
1327 "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n",
1328 i, LIO_CAST64(resp->cfg_info.iqmask),
1329 LIO_CAST64(resp->cfg_info.oqmask),
1330 num_iqueues, num_oqueues);
1331
1332 ifp = if_alloc(IFT_ETHER);
1333
1334 lio = malloc(sizeof(struct lio), M_DEVBUF, M_NOWAIT | M_ZERO);
1335
1336 if (lio == NULL) {
1337 lio_dev_err(octeon_dev, "Lio allocation failed\n");
1338 goto setup_nic_dev_fail;
1339 }
1340
1341 if_setsoftc(ifp, lio);
1342
1343 ifp->if_hw_tsomax = LIO_MAX_FRAME_SIZE;
1344 ifp->if_hw_tsomaxsegcount = LIO_MAX_SG;
1345 ifp->if_hw_tsomaxsegsize = PAGE_SIZE;
1346
1347 lio->ifidx = ifidx_or_pfnum;
1348
1349 props = &octeon_dev->props;
1350 props->gmxport = resp->cfg_info.linfo.gmxport;
1351 props->ifp = ifp;
1352
1353 lio->linfo.num_rxpciq = num_oqueues;
1354 lio->linfo.num_txpciq = num_iqueues;
1355 for (j = 0; j < num_oqueues; j++) {
1356 lio->linfo.rxpciq[j].rxpciq64 =
1357 resp->cfg_info.linfo.rxpciq[j].rxpciq64;
1358 }
1359
1360 for (j = 0; j < num_iqueues; j++) {
1361 lio->linfo.txpciq[j].txpciq64 =
1362 resp->cfg_info.linfo.txpciq[j].txpciq64;
1363 }
1364
1365 lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;
1366 lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;
1367 lio->linfo.link.link_status64 =
1368 resp->cfg_info.linfo.link.link_status64;
1369
1370 /*
1371 * Point to the properties for octeon device to which this
1372 * interface belongs.
1373 */
1374 lio->oct_dev = octeon_dev;
1375 lio->ifp = ifp;
1376
1377 lio_dev_dbg(octeon_dev, "if%d gmx: %d hw_addr: 0x%llx\n", i,
1378 lio->linfo.gmxport, LIO_CAST64(lio->linfo.hw_addr));
1379 lio_init_ifnet(lio);
1380 /* 64-bit swap required on LE machines */
1381 lio_swap_8B_data(&lio->linfo.hw_addr, 1);
1382 for (j = 0; j < 6; j++)
1383 mac[j] = *((uint8_t *)(
1384 ((uint8_t *)&lio->linfo.hw_addr) + 2 + j));
1385
1386 ether_ifattach(ifp, mac);
1387
1388 /*
1389 * By default all interfaces on a single Octeon uses the same
1390 * tx and rx queues
1391 */
1392 lio->txq = lio->linfo.txpciq[0].s.q_no;
1393 lio->rxq = lio->linfo.rxpciq[0].s.q_no;
1394 if (lio_setup_io_queues(octeon_dev, i, lio->linfo.num_txpciq,
1395 lio->linfo.num_rxpciq)) {
1396 lio_dev_err(octeon_dev, "I/O queues creation failed\n");
1397 goto setup_nic_dev_fail;
1398 }
1399
1400 lio_ifstate_set(lio, LIO_IFSTATE_DROQ_OPS);
1401
1402 lio->tx_qsize = lio_get_tx_qsize(octeon_dev, lio->txq);
1403 lio->rx_qsize = lio_get_rx_qsize(octeon_dev, lio->rxq);
1404
1405 if (lio_setup_glists(octeon_dev, lio, num_iqueues)) {
1406 lio_dev_err(octeon_dev, "Gather list allocation failed\n");
1407 goto setup_nic_dev_fail;
1408 }
1409
1410 if ((lio_hwlro == 0) && lio_tcp_lro_init(octeon_dev, ifp))
1411 goto setup_nic_dev_fail;
1412
1413 if (lio_hwlro &&
1414 (if_getcapenable(ifp) & IFCAP_LRO) &&
1415 (if_getcapenable(ifp) & IFCAP_RXCSUM) &&
1416 (if_getcapenable(ifp) & IFCAP_RXCSUM_IPV6))
1417 lio_set_feature(ifp, LIO_CMD_LRO_ENABLE,
1418 LIO_LROIPV4 | LIO_LROIPV6);
1419
1420 if ((if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER))
1421 lio_set_feature(ifp, LIO_CMD_VLAN_FILTER_CTL, 1);
1422 else
1423 lio_set_feature(ifp, LIO_CMD_VLAN_FILTER_CTL, 0);
1424
1425 if (lio_setup_rx_oom_poll_fn(ifp))
1426 goto setup_nic_dev_fail;
1427
1428 lio_dev_dbg(octeon_dev, "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
1429 i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
1430 lio->link_changes++;
1431
1432 lio_ifstate_set(lio, LIO_IFSTATE_REGISTERED);
1433
1434 /*
1435 * Sending command to firmware to enable Rx checksum offload
1436 * by default at the time of setup of Liquidio driver for
1437 * this device
1438 */
1439 lio_set_rxcsum_command(ifp, LIO_CMD_TNL_RX_CSUM_CTL,
1440 LIO_CMD_RXCSUM_ENABLE);
1441 lio_set_feature(ifp, LIO_CMD_TNL_TX_CSUM_CTL,
1442 LIO_CMD_TXCSUM_ENABLE);
1443
1444 #ifdef RSS
1445 if (lio_rss) {
1446 if (lio_send_rss_param(lio))
1447 goto setup_nic_dev_fail;
1448 } else
1449 #endif /* RSS */
1450
1451 lio_set_feature(ifp, LIO_CMD_SET_FNV,
1452 LIO_CMD_FNV_ENABLE);
1453
1454 lio_dev_dbg(octeon_dev, "NIC ifidx:%d Setup successful\n", i);
1455
1456 lio_free_soft_command(octeon_dev, sc);
1457 lio->vlan_attach =
1458 EVENTHANDLER_REGISTER(vlan_config,
1459 lio_vlan_rx_add_vid, lio,
1460 EVENTHANDLER_PRI_FIRST);
1461 lio->vlan_detach =
1462 EVENTHANDLER_REGISTER(vlan_unconfig,
1463 lio_vlan_rx_kill_vid, lio,
1464 EVENTHANDLER_PRI_FIRST);
1465
1466 /* Update stats periodically */
1467 callout_init(&lio->stats_timer, 0);
1468 lio->stats_interval = LIO_DEFAULT_STATS_INTERVAL;
1469
1470 lio_add_hw_stats(lio);
1471 }
1472
1473 return (0);
1474
1475 setup_nic_dev_fail:
1476
1477 lio_free_soft_command(octeon_dev, sc);
1478
1479 while (i--) {
1480 lio_dev_err(octeon_dev, "NIC ifidx:%d Setup failed\n", i);
1481 lio_destroy_nic_device(octeon_dev, i);
1482 }
1483
1484 return (ENODEV);
1485 }
1486
1487 static int
lio_link_info(struct lio_recv_info * recv_info,void * ptr)1488 lio_link_info(struct lio_recv_info *recv_info, void *ptr)
1489 {
1490 struct octeon_device *oct = (struct octeon_device *)ptr;
1491 struct lio_recv_pkt *recv_pkt = recv_info->recv_pkt;
1492 union octeon_link_status *ls;
1493 int gmxport = 0, i;
1494
1495 lio_dev_dbg(oct, "%s Called\n", __func__);
1496 if (recv_pkt->buffer_size[0] != (sizeof(*ls) + LIO_DROQ_INFO_SIZE)) {
1497 lio_dev_err(oct, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
1498 recv_pkt->buffer_size[0],
1499 recv_pkt->rh.r_nic_info.gmxport);
1500 goto nic_info_err;
1501 }
1502 gmxport = recv_pkt->rh.r_nic_info.gmxport;
1503 ls = (union octeon_link_status *)(recv_pkt->buffer_ptr[0]->m_data +
1504 LIO_DROQ_INFO_SIZE);
1505 lio_swap_8B_data((uint64_t *)ls,
1506 (sizeof(union octeon_link_status)) >> 3);
1507
1508 if (oct->props.gmxport == gmxport)
1509 lio_update_link_status(oct->props.ifp, ls);
1510
1511 nic_info_err:
1512 for (i = 0; i < recv_pkt->buffer_count; i++)
1513 lio_recv_buffer_free(recv_pkt->buffer_ptr[i]);
1514
1515 lio_free_recv_info(recv_info);
1516 return (0);
1517 }
1518
1519 void
lio_free_mbuf(struct lio_instr_queue * iq,struct lio_mbuf_free_info * finfo)1520 lio_free_mbuf(struct lio_instr_queue *iq, struct lio_mbuf_free_info *finfo)
1521 {
1522
1523 bus_dmamap_sync(iq->txtag, finfo->map, BUS_DMASYNC_POSTWRITE);
1524 bus_dmamap_unload(iq->txtag, finfo->map);
1525 m_freem(finfo->mb);
1526 }
1527
1528 void
lio_free_sgmbuf(struct lio_instr_queue * iq,struct lio_mbuf_free_info * finfo)1529 lio_free_sgmbuf(struct lio_instr_queue *iq, struct lio_mbuf_free_info *finfo)
1530 {
1531 struct lio_gather *g;
1532 struct octeon_device *oct;
1533 struct lio *lio;
1534 int iq_no;
1535
1536 g = finfo->g;
1537 iq_no = iq->txpciq.s.q_no;
1538 oct = iq->oct_dev;
1539 lio = if_getsoftc(oct->props.ifp);
1540
1541 mtx_lock(&lio->glist_lock[iq_no]);
1542 STAILQ_INSERT_TAIL(&lio->ghead[iq_no], &g->node, entries);
1543 mtx_unlock(&lio->glist_lock[iq_no]);
1544
1545 bus_dmamap_sync(iq->txtag, finfo->map, BUS_DMASYNC_POSTWRITE);
1546 bus_dmamap_unload(iq->txtag, finfo->map);
1547 m_freem(finfo->mb);
1548 }
1549
1550 static void
lio_if_cfg_callback(struct octeon_device * oct,uint32_t status,void * buf)1551 lio_if_cfg_callback(struct octeon_device *oct, uint32_t status, void *buf)
1552 {
1553 struct lio_soft_command *sc = (struct lio_soft_command *)buf;
1554 struct lio_if_cfg_resp *resp;
1555 struct lio_if_cfg_context *ctx;
1556
1557 resp = (struct lio_if_cfg_resp *)sc->virtrptr;
1558 ctx = (struct lio_if_cfg_context *)sc->ctxptr;
1559
1560 oct = lio_get_device(ctx->octeon_id);
1561 if (resp->status)
1562 lio_dev_err(oct, "nic if cfg instruction failed. Status: %llx (0x%08x)\n",
1563 LIO_CAST64(resp->status), status);
1564 ctx->cond = 1;
1565
1566 snprintf(oct->fw_info.lio_firmware_version, 32, "%s",
1567 resp->cfg_info.lio_firmware_version);
1568
1569 /*
1570 * This barrier is required to be sure that the response has been
1571 * written fully before waking up the handler
1572 */
1573 wmb();
1574 }
1575
1576 static int
lio_is_mac_changed(uint8_t * new,uint8_t * old)1577 lio_is_mac_changed(uint8_t *new, uint8_t *old)
1578 {
1579
1580 return ((new[0] != old[0]) || (new[1] != old[1]) ||
1581 (new[2] != old[2]) || (new[3] != old[3]) ||
1582 (new[4] != old[4]) || (new[5] != old[5]));
1583 }
1584
1585 void
lio_open(void * arg)1586 lio_open(void *arg)
1587 {
1588 struct lio *lio = arg;
1589 struct ifnet *ifp = lio->ifp;
1590 struct octeon_device *oct = lio->oct_dev;
1591 uint8_t *mac_new, mac_old[ETHER_HDR_LEN];
1592 int ret = 0;
1593
1594 lio_ifstate_set(lio, LIO_IFSTATE_RUNNING);
1595
1596 /* Ready for link status updates */
1597 lio->intf_open = 1;
1598
1599 lio_dev_info(oct, "Interface Open, ready for traffic\n");
1600
1601 /* tell Octeon to start forwarding packets to host */
1602 lio_send_rx_ctrl_cmd(lio, 1);
1603
1604 mac_new = IF_LLADDR(ifp);
1605 memcpy(mac_old, ((uint8_t *)&lio->linfo.hw_addr) + 2, ETHER_HDR_LEN);
1606
1607 if (lio_is_mac_changed(mac_new, mac_old)) {
1608 ret = lio_set_mac(ifp, mac_new);
1609 if (ret)
1610 lio_dev_err(oct, "MAC change failed, error: %d\n", ret);
1611 }
1612
1613 /* Now inform the stack we're ready */
1614 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0);
1615
1616 lio_dev_info(oct, "Interface is opened\n");
1617 }
1618
1619 static int
lio_set_rxcsum_command(struct ifnet * ifp,int command,uint8_t rx_cmd)1620 lio_set_rxcsum_command(struct ifnet *ifp, int command, uint8_t rx_cmd)
1621 {
1622 struct lio_ctrl_pkt nctrl;
1623 struct lio *lio = if_getsoftc(ifp);
1624 struct octeon_device *oct = lio->oct_dev;
1625 int ret = 0;
1626
1627 nctrl.ncmd.cmd64 = 0;
1628 nctrl.ncmd.s.cmd = command;
1629 nctrl.ncmd.s.param1 = rx_cmd;
1630 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1631 nctrl.wait_time = 100;
1632 nctrl.lio = lio;
1633 nctrl.cb_fn = lio_ctrl_cmd_completion;
1634
1635 ret = lio_send_ctrl_pkt(lio->oct_dev, &nctrl);
1636 if (ret < 0) {
1637 lio_dev_err(oct, "DEVFLAGS RXCSUM change failed in core(ret:0x%x)\n",
1638 ret);
1639 }
1640
1641 return (ret);
1642 }
1643
1644 static int
lio_stop_nic_module(struct octeon_device * oct)1645 lio_stop_nic_module(struct octeon_device *oct)
1646 {
1647 int i, j;
1648 struct lio *lio;
1649
1650 lio_dev_dbg(oct, "Stopping network interfaces\n");
1651 if (!oct->ifcount) {
1652 lio_dev_err(oct, "Init for Octeon was not completed\n");
1653 return (1);
1654 }
1655
1656 mtx_lock(&oct->cmd_resp_wqlock);
1657 oct->cmd_resp_state = LIO_DRV_OFFLINE;
1658 mtx_unlock(&oct->cmd_resp_wqlock);
1659
1660 for (i = 0; i < oct->ifcount; i++) {
1661 lio = if_getsoftc(oct->props.ifp);
1662 for (j = 0; j < oct->num_oqs; j++)
1663 lio_unregister_droq_ops(oct,
1664 lio->linfo.rxpciq[j].s.q_no);
1665 }
1666
1667 callout_drain(&lio->stats_timer);
1668
1669 for (i = 0; i < oct->ifcount; i++)
1670 lio_destroy_nic_device(oct, i);
1671
1672 lio_dev_dbg(oct, "Network interface stopped\n");
1673
1674 return (0);
1675 }
1676
1677 static void
lio_delete_glists(struct octeon_device * oct,struct lio * lio)1678 lio_delete_glists(struct octeon_device *oct, struct lio *lio)
1679 {
1680 struct lio_gather *g;
1681 int i;
1682
1683 if (lio->glist_lock != NULL) {
1684 free((void *)lio->glist_lock, M_DEVBUF);
1685 lio->glist_lock = NULL;
1686 }
1687
1688 if (lio->ghead == NULL)
1689 return;
1690
1691 for (i = 0; i < lio->linfo.num_txpciq; i++) {
1692 do {
1693 g = (struct lio_gather *)
1694 lio_delete_first_node(&lio->ghead[i]);
1695 free(g, M_DEVBUF);
1696 } while (g);
1697
1698 if ((lio->glists_virt_base != NULL) &&
1699 (lio->glists_virt_base[i] != NULL)) {
1700 lio_dma_free(lio->glist_entry_size * lio->tx_qsize,
1701 lio->glists_virt_base[i]);
1702 }
1703 }
1704
1705 free(lio->glists_virt_base, M_DEVBUF);
1706 lio->glists_virt_base = NULL;
1707
1708 free(lio->glists_dma_base, M_DEVBUF);
1709 lio->glists_dma_base = NULL;
1710
1711 free(lio->ghead, M_DEVBUF);
1712 lio->ghead = NULL;
1713 }
1714
1715 static int
lio_setup_glists(struct octeon_device * oct,struct lio * lio,int num_iqs)1716 lio_setup_glists(struct octeon_device *oct, struct lio *lio, int num_iqs)
1717 {
1718 struct lio_gather *g;
1719 int i, j;
1720
1721 lio->glist_lock = malloc(num_iqs * sizeof(*lio->glist_lock), M_DEVBUF,
1722 M_NOWAIT | M_ZERO);
1723 if (lio->glist_lock == NULL)
1724 return (1);
1725
1726 lio->ghead = malloc(num_iqs * sizeof(*lio->ghead), M_DEVBUF,
1727 M_NOWAIT | M_ZERO);
1728 if (lio->ghead == NULL) {
1729 free((void *)lio->glist_lock, M_DEVBUF);
1730 lio->glist_lock = NULL;
1731 return (1);
1732 }
1733
1734 lio->glist_entry_size = ROUNDUP8((ROUNDUP4(LIO_MAX_SG) >> 2) *
1735 LIO_SG_ENTRY_SIZE);
1736 /*
1737 * allocate memory to store virtual and dma base address of
1738 * per glist consistent memory
1739 */
1740 lio->glists_virt_base = malloc(num_iqs * sizeof(void *), M_DEVBUF,
1741 M_NOWAIT | M_ZERO);
1742 lio->glists_dma_base = malloc(num_iqs * sizeof(vm_paddr_t), M_DEVBUF,
1743 M_NOWAIT | M_ZERO);
1744 if ((lio->glists_virt_base == NULL) || (lio->glists_dma_base == NULL)) {
1745 lio_delete_glists(oct, lio);
1746 return (1);
1747 }
1748
1749 for (i = 0; i < num_iqs; i++) {
1750 mtx_init(&lio->glist_lock[i], "glist_lock", NULL, MTX_DEF);
1751
1752 STAILQ_INIT(&lio->ghead[i]);
1753
1754 lio->glists_virt_base[i] =
1755 lio_dma_alloc(lio->glist_entry_size * lio->tx_qsize,
1756 (vm_paddr_t *)&lio->glists_dma_base[i]);
1757 if (lio->glists_virt_base[i] == NULL) {
1758 lio_delete_glists(oct, lio);
1759 return (1);
1760 }
1761
1762 for (j = 0; j < lio->tx_qsize; j++) {
1763 g = malloc(sizeof(*g), M_DEVBUF, M_NOWAIT | M_ZERO);
1764 if (g == NULL)
1765 break;
1766
1767 g->sg = (struct lio_sg_entry *)(uintptr_t)
1768 ((uint64_t)(uintptr_t)lio->glists_virt_base[i] +
1769 (j * lio->glist_entry_size));
1770 g->sg_dma_ptr = (uint64_t)lio->glists_dma_base[i] +
1771 (j * lio->glist_entry_size);
1772 STAILQ_INSERT_TAIL(&lio->ghead[i], &g->node, entries);
1773 }
1774
1775 if (j != lio->tx_qsize) {
1776 lio_delete_glists(oct, lio);
1777 return (1);
1778 }
1779 }
1780
1781 return (0);
1782 }
1783
1784 void
lio_stop(struct ifnet * ifp)1785 lio_stop(struct ifnet *ifp)
1786 {
1787 struct lio *lio = if_getsoftc(ifp);
1788 struct octeon_device *oct = lio->oct_dev;
1789
1790 lio_ifstate_reset(lio, LIO_IFSTATE_RUNNING);
1791 if_link_state_change(ifp, LINK_STATE_DOWN);
1792
1793 lio->intf_open = 0;
1794 lio->linfo.link.s.link_up = 0;
1795 lio->link_changes++;
1796
1797 lio_send_rx_ctrl_cmd(lio, 0);
1798
1799 /* Tell the stack that the interface is no longer active */
1800 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
1801
1802 lio_dev_info(oct, "Interface is stopped\n");
1803 }
1804
1805 static void
lio_check_rx_oom_status(struct lio * lio)1806 lio_check_rx_oom_status(struct lio *lio)
1807 {
1808 struct lio_droq *droq;
1809 struct octeon_device *oct = lio->oct_dev;
1810 int desc_refilled;
1811 int q, q_no = 0;
1812
1813 for (q = 0; q < oct->num_oqs; q++) {
1814 q_no = lio->linfo.rxpciq[q].s.q_no;
1815 droq = oct->droq[q_no];
1816 if (droq == NULL)
1817 continue;
1818 if (lio_read_csr32(oct, droq->pkts_credit_reg) <= 0x40) {
1819 mtx_lock(&droq->lock);
1820 desc_refilled = lio_droq_refill(oct, droq);
1821 /*
1822 * Flush the droq descriptor data to memory to be sure
1823 * that when we update the credits the data in memory
1824 * is accurate.
1825 */
1826 wmb();
1827 lio_write_csr32(oct, droq->pkts_credit_reg,
1828 desc_refilled);
1829 /* make sure mmio write completes */
1830 __compiler_membar();
1831 mtx_unlock(&droq->lock);
1832 }
1833 }
1834 }
1835
1836 static void
lio_poll_check_rx_oom_status(void * arg,int pending __unused)1837 lio_poll_check_rx_oom_status(void *arg, int pending __unused)
1838 {
1839 struct lio_tq *rx_status_tq = arg;
1840 struct lio *lio = rx_status_tq->ctxptr;
1841
1842 if (lio_ifstate_check(lio, LIO_IFSTATE_RUNNING))
1843 lio_check_rx_oom_status(lio);
1844
1845 taskqueue_enqueue_timeout(rx_status_tq->tq, &rx_status_tq->work,
1846 lio_ms_to_ticks(50));
1847 }
1848
1849 static int
lio_setup_rx_oom_poll_fn(struct ifnet * ifp)1850 lio_setup_rx_oom_poll_fn(struct ifnet *ifp)
1851 {
1852 struct lio *lio = if_getsoftc(ifp);
1853 struct octeon_device *oct = lio->oct_dev;
1854 struct lio_tq *rx_status_tq;
1855
1856 rx_status_tq = &lio->rx_status_tq;
1857
1858 rx_status_tq->tq = taskqueue_create("lio_rx_oom_status", M_WAITOK,
1859 taskqueue_thread_enqueue,
1860 &rx_status_tq->tq);
1861
1862 TIMEOUT_TASK_INIT(rx_status_tq->tq, &rx_status_tq->work, 0,
1863 lio_poll_check_rx_oom_status, (void *)rx_status_tq);
1864
1865 rx_status_tq->ctxptr = lio;
1866
1867 taskqueue_start_threads(&rx_status_tq->tq, 1, PI_NET,
1868 "lio%d_rx_oom_status",
1869 oct->octeon_id);
1870
1871 taskqueue_enqueue_timeout(rx_status_tq->tq, &rx_status_tq->work,
1872 lio_ms_to_ticks(50));
1873
1874 return (0);
1875 }
1876
1877 static void
lio_cleanup_rx_oom_poll_fn(struct ifnet * ifp)1878 lio_cleanup_rx_oom_poll_fn(struct ifnet *ifp)
1879 {
1880 struct lio *lio = if_getsoftc(ifp);
1881
1882 if (lio->rx_status_tq.tq != NULL) {
1883 while (taskqueue_cancel_timeout(lio->rx_status_tq.tq,
1884 &lio->rx_status_tq.work, NULL))
1885 taskqueue_drain_timeout(lio->rx_status_tq.tq,
1886 &lio->rx_status_tq.work);
1887
1888 taskqueue_free(lio->rx_status_tq.tq);
1889
1890 lio->rx_status_tq.tq = NULL;
1891 }
1892 }
1893
1894 static void
lio_destroy_nic_device(struct octeon_device * oct,int ifidx)1895 lio_destroy_nic_device(struct octeon_device *oct, int ifidx)
1896 {
1897 struct ifnet *ifp = oct->props.ifp;
1898 struct lio *lio;
1899
1900 if (ifp == NULL) {
1901 lio_dev_err(oct, "%s No ifp ptr for index %d\n",
1902 __func__, ifidx);
1903 return;
1904 }
1905
1906 lio = if_getsoftc(ifp);
1907
1908 lio_ifstate_set(lio, LIO_IFSTATE_DETACH);
1909
1910 lio_dev_dbg(oct, "NIC device cleanup\n");
1911
1912 if (atomic_load_acq_int(&lio->ifstate) & LIO_IFSTATE_RUNNING)
1913 lio_stop(ifp);
1914
1915 if (lio_wait_for_pending_requests(oct))
1916 lio_dev_err(oct, "There were pending requests\n");
1917
1918 if (lio_wait_for_instr_fetch(oct))
1919 lio_dev_err(oct, "IQ had pending instructions\n");
1920
1921 if (lio_wait_for_oq_pkts(oct))
1922 lio_dev_err(oct, "OQ had pending packets\n");
1923
1924 if (atomic_load_acq_int(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
1925 ether_ifdetach(ifp);
1926
1927 lio_tcp_lro_free(oct, ifp);
1928
1929 lio_cleanup_rx_oom_poll_fn(ifp);
1930
1931 lio_delete_glists(oct, lio);
1932
1933 EVENTHANDLER_DEREGISTER(vlan_config, lio->vlan_attach);
1934 EVENTHANDLER_DEREGISTER(vlan_unconfig, lio->vlan_detach);
1935
1936 free(lio, M_DEVBUF);
1937
1938 if_free(ifp);
1939
1940 oct->props.gmxport = -1;
1941
1942 oct->props.ifp = NULL;
1943 }
1944
1945 static void
print_link_info(struct ifnet * ifp)1946 print_link_info(struct ifnet *ifp)
1947 {
1948 struct lio *lio = if_getsoftc(ifp);
1949
1950 if (!lio_ifstate_check(lio, LIO_IFSTATE_RESETTING) &&
1951 lio_ifstate_check(lio, LIO_IFSTATE_REGISTERED)) {
1952 struct octeon_link_info *linfo = &lio->linfo;
1953
1954 if (linfo->link.s.link_up) {
1955 lio_dev_info(lio->oct_dev, "%d Mbps %s Duplex UP\n",
1956 linfo->link.s.speed,
1957 (linfo->link.s.duplex) ? "Full" : "Half");
1958 } else {
1959 lio_dev_info(lio->oct_dev, "Link Down\n");
1960 }
1961 }
1962 }
1963
1964 static inline void
lio_update_link_status(struct ifnet * ifp,union octeon_link_status * ls)1965 lio_update_link_status(struct ifnet *ifp, union octeon_link_status *ls)
1966 {
1967 struct lio *lio = if_getsoftc(ifp);
1968 int changed = (lio->linfo.link.link_status64 != ls->link_status64);
1969
1970 lio->linfo.link.link_status64 = ls->link_status64;
1971
1972 if ((lio->intf_open) && (changed)) {
1973 print_link_info(ifp);
1974 lio->link_changes++;
1975 if (lio->linfo.link.s.link_up)
1976 if_link_state_change(ifp, LINK_STATE_UP);
1977 else
1978 if_link_state_change(ifp, LINK_STATE_DOWN);
1979 }
1980 }
1981
1982 /*
1983 * \brief Callback for rx ctrl
1984 * @param status status of request
1985 * @param buf pointer to resp structure
1986 */
1987 static void
lio_rx_ctl_callback(struct octeon_device * oct,uint32_t status,void * buf)1988 lio_rx_ctl_callback(struct octeon_device *oct, uint32_t status, void *buf)
1989 {
1990 struct lio_soft_command *sc = (struct lio_soft_command *)buf;
1991 struct lio_rx_ctl_context *ctx;
1992
1993 ctx = (struct lio_rx_ctl_context *)sc->ctxptr;
1994
1995 oct = lio_get_device(ctx->octeon_id);
1996 if (status)
1997 lio_dev_err(oct, "rx ctl instruction failed. Status: %llx\n",
1998 LIO_CAST64(status));
1999 ctx->cond = 1;
2000
2001 /*
2002 * This barrier is required to be sure that the response has been
2003 * written fully before waking up the handler
2004 */
2005 wmb();
2006 }
2007
2008 static void
lio_send_rx_ctrl_cmd(struct lio * lio,int start_stop)2009 lio_send_rx_ctrl_cmd(struct lio *lio, int start_stop)
2010 {
2011 struct lio_soft_command *sc;
2012 struct lio_rx_ctl_context *ctx;
2013 union octeon_cmd *ncmd;
2014 struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
2015 int ctx_size = sizeof(struct lio_rx_ctl_context);
2016 int retval;
2017
2018 if (oct->props.rx_on == start_stop)
2019 return;
2020
2021 sc = lio_alloc_soft_command(oct, OCTEON_CMD_SIZE, 16, ctx_size);
2022 if (sc == NULL)
2023 return;
2024
2025 ncmd = (union octeon_cmd *)sc->virtdptr;
2026 ctx = (struct lio_rx_ctl_context *)sc->ctxptr;
2027
2028 ctx->cond = 0;
2029 ctx->octeon_id = lio_get_device_id(oct);
2030 ncmd->cmd64 = 0;
2031 ncmd->s.cmd = LIO_CMD_RX_CTL;
2032 ncmd->s.param1 = start_stop;
2033
2034 lio_swap_8B_data((uint64_t *)ncmd, (OCTEON_CMD_SIZE >> 3));
2035
2036 sc->iq_no = lio->linfo.txpciq[0].s.q_no;
2037
2038 lio_prepare_soft_command(oct, sc, LIO_OPCODE_NIC, LIO_OPCODE_NIC_CMD, 0,
2039 0, 0);
2040
2041 sc->callback = lio_rx_ctl_callback;
2042 sc->callback_arg = sc;
2043 sc->wait_time = 5000;
2044
2045 retval = lio_send_soft_command(oct, sc);
2046 if (retval == LIO_IQ_SEND_FAILED) {
2047 lio_dev_err(oct, "Failed to send RX Control message\n");
2048 } else {
2049 /*
2050 * Sleep on a wait queue till the cond flag indicates that the
2051 * response arrived or timed-out.
2052 */
2053 lio_sleep_cond(oct, &ctx->cond);
2054 oct->props.rx_on = start_stop;
2055 }
2056
2057 lio_free_soft_command(oct, sc);
2058 }
2059
2060 static void
lio_vlan_rx_add_vid(void * arg,struct ifnet * ifp,uint16_t vid)2061 lio_vlan_rx_add_vid(void *arg, struct ifnet *ifp, uint16_t vid)
2062 {
2063 struct lio_ctrl_pkt nctrl;
2064 struct lio *lio = if_getsoftc(ifp);
2065 struct octeon_device *oct = lio->oct_dev;
2066 int ret = 0;
2067
2068 if (if_getsoftc(ifp) != arg) /* Not our event */
2069 return;
2070
2071 if ((vid == 0) || (vid > 4095)) /* Invalid */
2072 return;
2073
2074 bzero(&nctrl, sizeof(struct lio_ctrl_pkt));
2075
2076 nctrl.ncmd.cmd64 = 0;
2077 nctrl.ncmd.s.cmd = LIO_CMD_ADD_VLAN_FILTER;
2078 nctrl.ncmd.s.param1 = vid;
2079 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2080 nctrl.wait_time = 100;
2081 nctrl.lio = lio;
2082 nctrl.cb_fn = lio_ctrl_cmd_completion;
2083
2084 ret = lio_send_ctrl_pkt(lio->oct_dev, &nctrl);
2085 if (ret < 0) {
2086 lio_dev_err(oct, "Add VLAN filter failed in core (ret: 0x%x)\n",
2087 ret);
2088 }
2089 }
2090
2091 static void
lio_vlan_rx_kill_vid(void * arg,struct ifnet * ifp,uint16_t vid)2092 lio_vlan_rx_kill_vid(void *arg, struct ifnet *ifp, uint16_t vid)
2093 {
2094 struct lio_ctrl_pkt nctrl;
2095 struct lio *lio = if_getsoftc(ifp);
2096 struct octeon_device *oct = lio->oct_dev;
2097 int ret = 0;
2098
2099 if (if_getsoftc(ifp) != arg) /* Not our event */
2100 return;
2101
2102 if ((vid == 0) || (vid > 4095)) /* Invalid */
2103 return;
2104
2105 bzero(&nctrl, sizeof(struct lio_ctrl_pkt));
2106
2107 nctrl.ncmd.cmd64 = 0;
2108 nctrl.ncmd.s.cmd = LIO_CMD_DEL_VLAN_FILTER;
2109 nctrl.ncmd.s.param1 = vid;
2110 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2111 nctrl.wait_time = 100;
2112 nctrl.lio = lio;
2113 nctrl.cb_fn = lio_ctrl_cmd_completion;
2114
2115 ret = lio_send_ctrl_pkt(lio->oct_dev, &nctrl);
2116 if (ret < 0) {
2117 lio_dev_err(oct,
2118 "Kill VLAN filter failed in core (ret: 0x%x)\n",
2119 ret);
2120 }
2121 }
2122
2123 static int
lio_wait_for_oq_pkts(struct octeon_device * oct)2124 lio_wait_for_oq_pkts(struct octeon_device *oct)
2125 {
2126 int i, pending_pkts, pkt_cnt = 0, retry = 100;
2127
2128 do {
2129 pending_pkts = 0;
2130
2131 for (i = 0; i < LIO_MAX_OUTPUT_QUEUES(oct); i++) {
2132 if (!(oct->io_qmask.oq & BIT_ULL(i)))
2133 continue;
2134
2135 pkt_cnt = lio_droq_check_hw_for_pkts(oct->droq[i]);
2136 if (pkt_cnt > 0) {
2137 pending_pkts += pkt_cnt;
2138 taskqueue_enqueue(oct->droq[i]->droq_taskqueue,
2139 &oct->droq[i]->droq_task);
2140 }
2141 }
2142
2143 pkt_cnt = 0;
2144 lio_sleep_timeout(1);
2145 } while (retry-- && pending_pkts);
2146
2147 return (pkt_cnt);
2148 }
2149
2150 static void
lio_destroy_resources(struct octeon_device * oct)2151 lio_destroy_resources(struct octeon_device *oct)
2152 {
2153 int i, refcount;
2154
2155 switch (atomic_load_acq_int(&oct->status)) {
2156 case LIO_DEV_RUNNING:
2157 case LIO_DEV_CORE_OK:
2158 /* No more instructions will be forwarded. */
2159 atomic_store_rel_int(&oct->status, LIO_DEV_IN_RESET);
2160
2161 oct->app_mode = LIO_DRV_INVALID_APP;
2162 lio_dev_dbg(oct, "Device state is now %s\n",
2163 lio_get_state_string(&oct->status));
2164
2165 lio_sleep_timeout(100);
2166
2167 /* fallthrough */
2168 case LIO_DEV_HOST_OK:
2169
2170 /* fallthrough */
2171 case LIO_DEV_CONSOLE_INIT_DONE:
2172 /* Remove any consoles */
2173 lio_remove_consoles(oct);
2174
2175 /* fallthrough */
2176 case LIO_DEV_IO_QUEUES_DONE:
2177 if (lio_wait_for_pending_requests(oct))
2178 lio_dev_err(oct, "There were pending requests\n");
2179
2180 if (lio_wait_for_instr_fetch(oct))
2181 lio_dev_err(oct, "IQ had pending instructions\n");
2182
2183 /*
2184 * Disable the input and output queues now. No more packets will
2185 * arrive from Octeon, but we should wait for all packet
2186 * processing to finish.
2187 */
2188 oct->fn_list.disable_io_queues(oct);
2189
2190 if (lio_wait_for_oq_pkts(oct))
2191 lio_dev_err(oct, "OQ had pending packets\n");
2192
2193 /* fallthrough */
2194 case LIO_DEV_INTR_SET_DONE:
2195 /* Disable interrupts */
2196 oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
2197
2198 if (oct->msix_on) {
2199 for (i = 0; i < oct->num_msix_irqs - 1; i++) {
2200 if (oct->ioq_vector[i].tag != NULL) {
2201 bus_teardown_intr(oct->device,
2202 oct->ioq_vector[i].msix_res,
2203 oct->ioq_vector[i].tag);
2204 oct->ioq_vector[i].tag = NULL;
2205 }
2206 if (oct->ioq_vector[i].msix_res != NULL) {
2207 bus_release_resource(oct->device,
2208 SYS_RES_IRQ,
2209 oct->ioq_vector[i].vector,
2210 oct->ioq_vector[i].msix_res);
2211 oct->ioq_vector[i].msix_res = NULL;
2212 }
2213 }
2214 /* non-iov vector's argument is oct struct */
2215 if (oct->tag != NULL) {
2216 bus_teardown_intr(oct->device, oct->msix_res,
2217 oct->tag);
2218 oct->tag = NULL;
2219 }
2220
2221 if (oct->msix_res != NULL) {
2222 bus_release_resource(oct->device, SYS_RES_IRQ,
2223 oct->aux_vector,
2224 oct->msix_res);
2225 oct->msix_res = NULL;
2226 }
2227
2228 pci_release_msi(oct->device);
2229 }
2230 /* fallthrough */
2231 case LIO_DEV_IN_RESET:
2232 case LIO_DEV_DROQ_INIT_DONE:
2233 /* Wait for any pending operations */
2234 lio_mdelay(100);
2235 for (i = 0; i < LIO_MAX_OUTPUT_QUEUES(oct); i++) {
2236 if (!(oct->io_qmask.oq & BIT_ULL(i)))
2237 continue;
2238 lio_delete_droq(oct, i);
2239 }
2240
2241 /* fallthrough */
2242 case LIO_DEV_RESP_LIST_INIT_DONE:
2243 for (i = 0; i < LIO_MAX_POSSIBLE_OUTPUT_QUEUES; i++) {
2244 if (oct->droq[i] != NULL) {
2245 free(oct->droq[i], M_DEVBUF);
2246 oct->droq[i] = NULL;
2247 }
2248 }
2249 lio_delete_response_list(oct);
2250
2251 /* fallthrough */
2252 case LIO_DEV_INSTR_QUEUE_INIT_DONE:
2253 for (i = 0; i < LIO_MAX_INSTR_QUEUES(oct); i++) {
2254 if (!(oct->io_qmask.iq & BIT_ULL(i)))
2255 continue;
2256
2257 lio_delete_instr_queue(oct, i);
2258 }
2259
2260 /* fallthrough */
2261 case LIO_DEV_MSIX_ALLOC_VECTOR_DONE:
2262 for (i = 0; i < LIO_MAX_POSSIBLE_INSTR_QUEUES; i++) {
2263 if (oct->instr_queue[i] != NULL) {
2264 free(oct->instr_queue[i], M_DEVBUF);
2265 oct->instr_queue[i] = NULL;
2266 }
2267 }
2268 lio_free_ioq_vector(oct);
2269
2270 /* fallthrough */
2271 case LIO_DEV_SC_BUFF_POOL_INIT_DONE:
2272 lio_free_sc_buffer_pool(oct);
2273
2274 /* fallthrough */
2275 case LIO_DEV_DISPATCH_INIT_DONE:
2276 lio_delete_dispatch_list(oct);
2277
2278 /* fallthrough */
2279 case LIO_DEV_PCI_MAP_DONE:
2280 refcount = lio_deregister_device(oct);
2281
2282 if (fw_type_is_none())
2283 lio_pci_flr(oct);
2284
2285 if (!refcount)
2286 oct->fn_list.soft_reset(oct);
2287
2288 lio_unmap_pci_barx(oct, 0);
2289 lio_unmap_pci_barx(oct, 1);
2290
2291 /* fallthrough */
2292 case LIO_DEV_PCI_ENABLE_DONE:
2293 /* Disable the device, releasing the PCI INT */
2294 pci_disable_busmaster(oct->device);
2295
2296 /* fallthrough */
2297 case LIO_DEV_BEGIN_STATE:
2298 break;
2299 } /* end switch (oct->status) */
2300 }
2301