1 /*-
2 * Copyright (c) 2011 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30
31 #include "opt_inet.h"
32 #include "opt_inet6.h"
33
34 #include <sys/param.h>
35 #include <sys/conf.h>
36 #include <sys/priv.h>
37 #include <sys/kernel.h>
38 #include <sys/bus.h>
39 #include <sys/systm.h>
40 #include <sys/counter.h>
41 #include <sys/module.h>
42 #include <sys/malloc.h>
43 #include <sys/queue.h>
44 #include <sys/taskqueue.h>
45 #include <sys/pciio.h>
46 #include <dev/pci/pcireg.h>
47 #include <dev/pci/pcivar.h>
48 #include <dev/pci/pci_private.h>
49 #include <sys/firmware.h>
50 #include <sys/sbuf.h>
51 #include <sys/smp.h>
52 #include <sys/socket.h>
53 #include <sys/sockio.h>
54 #include <sys/sysctl.h>
55 #include <net/ethernet.h>
56 #include <net/if.h>
57 #include <net/if_types.h>
58 #include <net/if_dl.h>
59 #include <net/if_vlan_var.h>
60 #if defined(__i386__) || defined(__amd64__)
61 #include <vm/vm.h>
62 #include <vm/pmap.h>
63 #endif
64
65 #include "common/common.h"
66 #include "common/t4_msg.h"
67 #include "common/t4_regs.h"
68 #include "common/t4_regs_values.h"
69 #include "t4_ioctl.h"
70 #include "t4_l2t.h"
71 #include "t4_mp_ring.h"
72
73 /* T4 bus driver interface */
74 static int t4_probe(device_t);
75 static int t4_attach(device_t);
76 static int t4_detach(device_t);
77 static device_method_t t4_methods[] = {
78 DEVMETHOD(device_probe, t4_probe),
79 DEVMETHOD(device_attach, t4_attach),
80 DEVMETHOD(device_detach, t4_detach),
81
82 DEVMETHOD_END
83 };
84 static driver_t t4_driver = {
85 "t4nex",
86 t4_methods,
87 sizeof(struct adapter)
88 };
89
90
91 /* T4 port (cxgbe) interface */
92 static int cxgbe_probe(device_t);
93 static int cxgbe_attach(device_t);
94 static int cxgbe_detach(device_t);
95 static device_method_t cxgbe_methods[] = {
96 DEVMETHOD(device_probe, cxgbe_probe),
97 DEVMETHOD(device_attach, cxgbe_attach),
98 DEVMETHOD(device_detach, cxgbe_detach),
99 { 0, 0 }
100 };
101 static driver_t cxgbe_driver = {
102 "cxgbe",
103 cxgbe_methods,
104 sizeof(struct port_info)
105 };
106
107 static d_ioctl_t t4_ioctl;
108 static d_open_t t4_open;
109 static d_close_t t4_close;
110
111 static struct cdevsw t4_cdevsw = {
112 .d_version = D_VERSION,
113 .d_flags = 0,
114 .d_open = t4_open,
115 .d_close = t4_close,
116 .d_ioctl = t4_ioctl,
117 .d_name = "t4nex",
118 };
119
120 /* T5 bus driver interface */
121 static int t5_probe(device_t);
122 static device_method_t t5_methods[] = {
123 DEVMETHOD(device_probe, t5_probe),
124 DEVMETHOD(device_attach, t4_attach),
125 DEVMETHOD(device_detach, t4_detach),
126
127 DEVMETHOD_END
128 };
129 static driver_t t5_driver = {
130 "t5nex",
131 t5_methods,
132 sizeof(struct adapter)
133 };
134
135
136 /* T5 port (cxl) interface */
137 static driver_t cxl_driver = {
138 "cxl",
139 cxgbe_methods,
140 sizeof(struct port_info)
141 };
142
143 static struct cdevsw t5_cdevsw = {
144 .d_version = D_VERSION,
145 .d_flags = 0,
146 .d_open = t4_open,
147 .d_close = t4_close,
148 .d_ioctl = t4_ioctl,
149 .d_name = "t5nex",
150 };
151
152 /* ifnet + media interface */
153 static void cxgbe_init(void *);
154 static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
155 static int cxgbe_transmit(struct ifnet *, struct mbuf *);
156 static void cxgbe_qflush(struct ifnet *);
157 static int cxgbe_media_change(struct ifnet *);
158 static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
159
160 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
161
162 /*
163 * Correct lock order when you need to acquire multiple locks is t4_list_lock,
164 * then ADAPTER_LOCK, then t4_uld_list_lock.
165 */
166 static struct sx t4_list_lock;
167 SLIST_HEAD(, adapter) t4_list;
168 #ifdef TCP_OFFLOAD
169 static struct sx t4_uld_list_lock;
170 SLIST_HEAD(, uld_info) t4_uld_list;
171 #endif
172
173 /*
174 * Tunables. See tweak_tunables() too.
175 *
176 * Each tunable is set to a default value here if it's known at compile-time.
177 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should
178 * provide a reasonable default when the driver is loaded.
179 *
180 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to
181 * T5 are under hw.cxl.
182 */
183
184 /*
185 * Number of queues for tx and rx, 10G and 1G, NIC and offload.
186 */
187 #define NTXQ_10G 16
188 static int t4_ntxq10g = -1;
189 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
190
191 #define NRXQ_10G 8
192 static int t4_nrxq10g = -1;
193 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
194
195 #define NTXQ_1G 4
196 static int t4_ntxq1g = -1;
197 TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
198
199 #define NRXQ_1G 2
200 static int t4_nrxq1g = -1;
201 TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
202
203 static int t4_rsrv_noflowq = 0;
204 TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq);
205
206 #ifdef TCP_OFFLOAD
207 #define NOFLDTXQ_10G 8
208 static int t4_nofldtxq10g = -1;
209 TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
210
211 #define NOFLDRXQ_10G 2
212 static int t4_nofldrxq10g = -1;
213 TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
214
215 #define NOFLDTXQ_1G 2
216 static int t4_nofldtxq1g = -1;
217 TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
218
219 #define NOFLDRXQ_1G 1
220 static int t4_nofldrxq1g = -1;
221 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
222 #endif
223
224 #ifdef DEV_NETMAP
225 #define NNMTXQ_10G 2
226 static int t4_nnmtxq10g = -1;
227 TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
228
229 #define NNMRXQ_10G 2
230 static int t4_nnmrxq10g = -1;
231 TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
232
233 #define NNMTXQ_1G 1
234 static int t4_nnmtxq1g = -1;
235 TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
236
237 #define NNMRXQ_1G 1
238 static int t4_nnmrxq1g = -1;
239 TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
240 #endif
241
242 /*
243 * Holdoff parameters for 10G and 1G ports.
244 */
245 #define TMR_IDX_10G 1
246 static int t4_tmr_idx_10g = TMR_IDX_10G;
247 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
248
249 #define PKTC_IDX_10G (-1)
250 static int t4_pktc_idx_10g = PKTC_IDX_10G;
251 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
252
253 #define TMR_IDX_1G 1
254 static int t4_tmr_idx_1g = TMR_IDX_1G;
255 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
256
257 #define PKTC_IDX_1G (-1)
258 static int t4_pktc_idx_1g = PKTC_IDX_1G;
259 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
260
261 /*
262 * Size (# of entries) of each tx and rx queue.
263 */
264 static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
265 TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
266
267 static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
268 TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
269
270 /*
271 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
272 */
273 static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
274 TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
275
276 /*
277 * Configuration file.
278 */
279 #define DEFAULT_CF "default"
280 #define FLASH_CF "flash"
281 #define UWIRE_CF "uwire"
282 #define FPGA_CF "fpga"
283 static char t4_cfg_file[32] = DEFAULT_CF;
284 TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
285
286 /*
287 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively).
288 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
289 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
290 * mark or when signalled to do so, 0 to never emit PAUSE.
291 */
292 static int t4_pause_settings = PAUSE_TX | PAUSE_RX;
293 TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings);
294
295 /*
296 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
297 * encouraged respectively).
298 */
299 static unsigned int t4_fw_install = 1;
300 TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install);
301
302 /*
303 * ASIC features that will be used. Disable the ones you don't want so that the
304 * chip resources aren't wasted on features that will not be used.
305 */
306 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */
307 TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
308
309 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
310 TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
311
312 static int t4_toecaps_allowed = -1;
313 TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
314
315 static int t4_rdmacaps_allowed = 0;
316 TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
317
318 static int t4_iscsicaps_allowed = 0;
319 TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
320
321 static int t4_fcoecaps_allowed = 0;
322 TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
323
324 static int t5_write_combine = 0;
325 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
326
327 struct intrs_and_queues {
328 uint16_t intr_type; /* INTx, MSI, or MSI-X */
329 uint16_t nirq; /* Total # of vectors */
330 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
331 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */
332 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */
333 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */
334 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */
335 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */
336 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */
337 #ifdef TCP_OFFLOAD
338 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */
339 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */
340 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */
341 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */
342 #endif
343 #ifdef DEV_NETMAP
344 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */
345 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */
346 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */
347 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */
348 #endif
349 };
350
351 struct filter_entry {
352 uint32_t valid:1; /* filter allocated and valid */
353 uint32_t locked:1; /* filter is administratively locked */
354 uint32_t pending:1; /* filter action is pending firmware reply */
355 uint32_t smtidx:8; /* Source MAC Table index for smac */
356 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */
357
358 struct t4_filter_specification fs;
359 };
360
361 static int map_bars_0_and_4(struct adapter *);
362 static int map_bar_2(struct adapter *);
363 static void setup_memwin(struct adapter *);
364 static int validate_mem_range(struct adapter *, uint32_t, int);
365 static int fwmtype_to_hwmtype(int);
366 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
367 uint32_t *);
368 static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
369 static uint32_t position_memwin(struct adapter *, int, uint32_t);
370 static int cfg_itype_and_nqueues(struct adapter *, int, int,
371 struct intrs_and_queues *);
372 static int prep_firmware(struct adapter *);
373 static int partition_resources(struct adapter *, const struct firmware *,
374 const char *);
375 static int get_params__pre_init(struct adapter *);
376 static int get_params__post_init(struct adapter *);
377 static int set_params__post_init(struct adapter *);
378 static void t4_set_desc(struct adapter *);
379 static void build_medialist(struct port_info *, struct ifmedia *);
380 static int cxgbe_init_synchronized(struct port_info *);
381 static int cxgbe_uninit_synchronized(struct port_info *);
382 static int setup_intr_handlers(struct adapter *);
383 static void quiesce_txq(struct adapter *, struct sge_txq *);
384 static void quiesce_wrq(struct adapter *, struct sge_wrq *);
385 static void quiesce_iq(struct adapter *, struct sge_iq *);
386 static void quiesce_fl(struct adapter *, struct sge_fl *);
387 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
388 driver_intr_t *, void *, char *);
389 static int t4_free_irq(struct adapter *, struct irq *);
390 static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
391 unsigned int);
392 static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
393 static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
394 static void cxgbe_tick(void *);
395 static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
396 static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
397 struct mbuf *);
398 static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
399 static int fw_msg_not_handled(struct adapter *, const __be64 *);
400 static int t4_sysctls(struct adapter *);
401 static int cxgbe_sysctls(struct port_info *);
402 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
403 static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
404 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
405 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
406 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
407 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
408 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
409 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
410 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
411 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
412 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
413 #ifdef SBUF_DRAIN
414 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
415 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
416 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
417 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
418 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
419 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
420 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
421 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
422 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
423 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
424 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
425 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
426 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
427 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
428 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
429 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
430 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
431 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
432 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
433 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
434 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
435 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
436 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
437 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
438 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
439 #endif
440 static uint32_t fconf_to_mode(uint32_t);
441 static uint32_t mode_to_fconf(uint32_t);
442 static uint32_t fspec_to_fconf(struct t4_filter_specification *);
443 static int get_filter_mode(struct adapter *, uint32_t *);
444 static int set_filter_mode(struct adapter *, uint32_t);
445 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
446 static int get_filter(struct adapter *, struct t4_filter *);
447 static int set_filter(struct adapter *, struct t4_filter *);
448 static int del_filter(struct adapter *, struct t4_filter *);
449 static void clear_filter(struct filter_entry *);
450 static int set_filter_wr(struct adapter *, int);
451 static int del_filter_wr(struct adapter *, int);
452 static int get_sge_context(struct adapter *, struct t4_sge_context *);
453 static int load_fw(struct adapter *, struct t4_data *);
454 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
455 static int read_i2c(struct adapter *, struct t4_i2c_data *);
456 static int set_sched_class(struct adapter *, struct t4_sched_params *);
457 static int set_sched_queue(struct adapter *, struct t4_sched_queue *);
458 #ifdef TCP_OFFLOAD
459 static int toe_capability(struct port_info *, int);
460 #endif
461 static int mod_event(module_t, int, void *);
462
463 struct {
464 uint16_t device;
465 char *desc;
466 } t4_pciids[] = {
467 {0xa000, "Chelsio Terminator 4 FPGA"},
468 {0x4400, "Chelsio T440-dbg"},
469 {0x4401, "Chelsio T420-CR"},
470 {0x4402, "Chelsio T422-CR"},
471 {0x4403, "Chelsio T440-CR"},
472 {0x4404, "Chelsio T420-BCH"},
473 {0x4405, "Chelsio T440-BCH"},
474 {0x4406, "Chelsio T440-CH"},
475 {0x4407, "Chelsio T420-SO"},
476 {0x4408, "Chelsio T420-CX"},
477 {0x4409, "Chelsio T420-BT"},
478 {0x440a, "Chelsio T404-BT"},
479 {0x440e, "Chelsio T440-LP-CR"},
480 }, t5_pciids[] = {
481 {0xb000, "Chelsio Terminator 5 FPGA"},
482 {0x5400, "Chelsio T580-dbg"},
483 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */
484 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */
485 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */
486 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */
487 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */
488 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */
489 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */
490 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */
491 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */
492 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */
493 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */
494 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */
495 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */
496 #ifdef notyet
497 {0x5404, "Chelsio T520-BCH"},
498 {0x5405, "Chelsio T540-BCH"},
499 {0x5406, "Chelsio T540-CH"},
500 {0x5408, "Chelsio T520-CX"},
501 {0x540b, "Chelsio B520-SR"},
502 {0x540c, "Chelsio B504-BT"},
503 {0x540f, "Chelsio Amsterdam"},
504 {0x5413, "Chelsio T580-CHR"},
505 #endif
506 };
507
508 #ifdef TCP_OFFLOAD
509 /*
510 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be
511 * exactly the same for both rxq and ofld_rxq.
512 */
513 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
514 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
515 #endif
516
517 /* No easy way to include t4_msg.h before adapter.h so we check this way */
518 CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
519 CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
520
521 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
522
523 static int
t4_probe(device_t dev)524 t4_probe(device_t dev)
525 {
526 int i;
527 uint16_t v = pci_get_vendor(dev);
528 uint16_t d = pci_get_device(dev);
529 uint8_t f = pci_get_function(dev);
530
531 if (v != PCI_VENDOR_ID_CHELSIO)
532 return (ENXIO);
533
534 /* Attach only to PF0 of the FPGA */
535 if (d == 0xa000 && f != 0)
536 return (ENXIO);
537
538 for (i = 0; i < nitems(t4_pciids); i++) {
539 if (d == t4_pciids[i].device) {
540 device_set_desc(dev, t4_pciids[i].desc);
541 return (BUS_PROBE_DEFAULT);
542 }
543 }
544
545 return (ENXIO);
546 }
547
548 static int
t5_probe(device_t dev)549 t5_probe(device_t dev)
550 {
551 int i;
552 uint16_t v = pci_get_vendor(dev);
553 uint16_t d = pci_get_device(dev);
554 uint8_t f = pci_get_function(dev);
555
556 if (v != PCI_VENDOR_ID_CHELSIO)
557 return (ENXIO);
558
559 /* Attach only to PF0 of the FPGA */
560 if (d == 0xb000 && f != 0)
561 return (ENXIO);
562
563 for (i = 0; i < nitems(t5_pciids); i++) {
564 if (d == t5_pciids[i].device) {
565 device_set_desc(dev, t5_pciids[i].desc);
566 return (BUS_PROBE_DEFAULT);
567 }
568 }
569
570 return (ENXIO);
571 }
572
573 static int
t4_attach(device_t dev)574 t4_attach(device_t dev)
575 {
576 struct adapter *sc;
577 int rc = 0, i, n10g, n1g, rqidx, tqidx;
578 struct intrs_and_queues iaq;
579 struct sge *s;
580 #ifdef TCP_OFFLOAD
581 int ofld_rqidx, ofld_tqidx;
582 #endif
583 #ifdef DEV_NETMAP
584 int nm_rqidx, nm_tqidx;
585 #endif
586
587 sc = device_get_softc(dev);
588 sc->dev = dev;
589
590 pci_enable_busmaster(dev);
591 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
592 uint32_t v;
593
594 pci_set_max_read_req(dev, 4096);
595 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
596 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
597 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
598
599 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
600 }
601
602 sc->traceq = -1;
603 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
604 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
605 device_get_nameunit(dev));
606
607 snprintf(sc->lockname, sizeof(sc->lockname), "%s",
608 device_get_nameunit(dev));
609 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
610 sx_xlock(&t4_list_lock);
611 SLIST_INSERT_HEAD(&t4_list, sc, link);
612 sx_xunlock(&t4_list_lock);
613
614 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
615 TAILQ_INIT(&sc->sfl);
616 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
617
618 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF);
619
620 rc = map_bars_0_and_4(sc);
621 if (rc != 0)
622 goto done; /* error message displayed already */
623
624 /*
625 * This is the real PF# to which we're attaching. Works from within PCI
626 * passthrough environments too, where pci_get_function() could return a
627 * different PF# depending on the passthrough configuration. We need to
628 * use the real PF# in all our communication with the firmware.
629 */
630 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
631 sc->mbox = sc->pf;
632
633 memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
634 sc->an_handler = an_not_handled;
635 for (i = 0; i < nitems(sc->cpl_handler); i++)
636 sc->cpl_handler[i] = cpl_not_handled;
637 for (i = 0; i < nitems(sc->fw_msg_handler); i++)
638 sc->fw_msg_handler[i] = fw_msg_not_handled;
639 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
640 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt);
641 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt);
642 t4_init_sge_cpl_handlers(sc);
643
644 /* Prepare the adapter for operation */
645 rc = -t4_prep_adapter(sc);
646 if (rc != 0) {
647 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
648 goto done;
649 }
650
651 /*
652 * Do this really early, with the memory windows set up even before the
653 * character device. The userland tool's register i/o and mem read
654 * will work even in "recovery mode".
655 */
656 setup_memwin(sc);
657 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw,
658 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s",
659 device_get_nameunit(dev));
660 if (sc->cdev == NULL)
661 device_printf(dev, "failed to create nexus char device.\n");
662 else
663 sc->cdev->si_drv1 = sc;
664
665 /* Go no further if recovery mode has been requested. */
666 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
667 device_printf(dev, "recovery mode.\n");
668 goto done;
669 }
670
671 #if defined(__i386__)
672 if ((cpu_feature & CPUID_CX8) == 0) {
673 device_printf(dev, "64 bit atomics not available.\n");
674 rc = ENOTSUP;
675 goto done;
676 }
677 #endif
678
679 /* Prepare the firmware for operation */
680 rc = prep_firmware(sc);
681 if (rc != 0)
682 goto done; /* error message displayed already */
683
684 rc = get_params__post_init(sc);
685 if (rc != 0)
686 goto done; /* error message displayed already */
687
688 rc = set_params__post_init(sc);
689 if (rc != 0)
690 goto done; /* error message displayed already */
691
692 rc = map_bar_2(sc);
693 if (rc != 0)
694 goto done; /* error message displayed already */
695
696 rc = t4_create_dma_tag(sc);
697 if (rc != 0)
698 goto done; /* error message displayed already */
699
700 /*
701 * First pass over all the ports - allocate VIs and initialize some
702 * basic parameters like mac address, port type, etc. We also figure
703 * out whether a port is 10G or 1G and use that information when
704 * calculating how many interrupts to attempt to allocate.
705 */
706 n10g = n1g = 0;
707 for_each_port(sc, i) {
708 struct port_info *pi;
709
710 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
711 sc->port[i] = pi;
712
713 /* These must be set before t4_port_init */
714 pi->adapter = sc;
715 pi->port_id = i;
716
717 /* Allocate the vi and initialize parameters like mac addr */
718 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
719 if (rc != 0) {
720 device_printf(dev, "unable to initialize port %d: %d\n",
721 i, rc);
722 free(pi, M_CXGBE);
723 sc->port[i] = NULL;
724 goto done;
725 }
726
727 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX);
728 pi->link_cfg.requested_fc |= t4_pause_settings;
729 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX);
730 pi->link_cfg.fc |= t4_pause_settings;
731
732 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
733 if (rc != 0) {
734 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
735 free(pi, M_CXGBE);
736 sc->port[i] = NULL;
737 goto done;
738 }
739
740 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
741 device_get_nameunit(dev), i);
742 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
743 sc->chan_map[pi->tx_chan] = i;
744
745 if (is_10G_port(pi) || is_40G_port(pi)) {
746 n10g++;
747 pi->tmr_idx = t4_tmr_idx_10g;
748 pi->pktc_idx = t4_pktc_idx_10g;
749 } else {
750 n1g++;
751 pi->tmr_idx = t4_tmr_idx_1g;
752 pi->pktc_idx = t4_pktc_idx_1g;
753 }
754
755 pi->xact_addr_filt = -1;
756 pi->linkdnrc = -1;
757
758 pi->qsize_rxq = t4_qsize_rxq;
759 pi->qsize_txq = t4_qsize_txq;
760
761 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1);
762 if (pi->dev == NULL) {
763 device_printf(dev,
764 "failed to add device for port %d.\n", i);
765 rc = ENXIO;
766 goto done;
767 }
768 device_set_softc(pi->dev, pi);
769 }
770
771 /*
772 * Interrupt type, # of interrupts, # of rx/tx queues, etc.
773 */
774 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
775 if (rc != 0)
776 goto done; /* error message displayed already */
777
778 sc->intr_type = iaq.intr_type;
779 sc->intr_count = iaq.nirq;
780
781 s = &sc->sge;
782 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
783 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
784 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */
785 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
786 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */
787 #ifdef TCP_OFFLOAD
788 if (is_offload(sc)) {
789 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
790 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
791 s->neq += s->nofldtxq + s->nofldrxq;
792 s->niq += s->nofldrxq;
793
794 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
795 M_CXGBE, M_ZERO | M_WAITOK);
796 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
797 M_CXGBE, M_ZERO | M_WAITOK);
798 }
799 #endif
800 #ifdef DEV_NETMAP
801 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
802 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
803 s->neq += s->nnmtxq + s->nnmrxq;
804 s->niq += s->nnmrxq;
805
806 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
807 M_CXGBE, M_ZERO | M_WAITOK);
808 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
809 M_CXGBE, M_ZERO | M_WAITOK);
810 #endif
811
812 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
813 M_ZERO | M_WAITOK);
814 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
815 M_ZERO | M_WAITOK);
816 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
817 M_ZERO | M_WAITOK);
818 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
819 M_ZERO | M_WAITOK);
820 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
821 M_ZERO | M_WAITOK);
822
823 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
824 M_ZERO | M_WAITOK);
825
826 t4_init_l2t(sc, M_WAITOK);
827
828 /*
829 * Second pass over the ports. This time we know the number of rx and
830 * tx queues that each port should get.
831 */
832 rqidx = tqidx = 0;
833 #ifdef TCP_OFFLOAD
834 ofld_rqidx = ofld_tqidx = 0;
835 #endif
836 #ifdef DEV_NETMAP
837 nm_rqidx = nm_tqidx = 0;
838 #endif
839 for_each_port(sc, i) {
840 struct port_info *pi = sc->port[i];
841
842 if (pi == NULL)
843 continue;
844
845 pi->first_rxq = rqidx;
846 pi->first_txq = tqidx;
847 if (is_10G_port(pi) || is_40G_port(pi)) {
848 pi->flags |= iaq.intr_flags_10g;
849 pi->nrxq = iaq.nrxq10g;
850 pi->ntxq = iaq.ntxq10g;
851 } else {
852 pi->flags |= iaq.intr_flags_1g;
853 pi->nrxq = iaq.nrxq1g;
854 pi->ntxq = iaq.ntxq1g;
855 }
856
857 if (pi->ntxq > 1)
858 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0;
859 else
860 pi->rsrv_noflowq = 0;
861
862 rqidx += pi->nrxq;
863 tqidx += pi->ntxq;
864 #ifdef TCP_OFFLOAD
865 if (is_offload(sc)) {
866 pi->first_ofld_rxq = ofld_rqidx;
867 pi->first_ofld_txq = ofld_tqidx;
868 if (is_10G_port(pi) || is_40G_port(pi)) {
869 pi->nofldrxq = iaq.nofldrxq10g;
870 pi->nofldtxq = iaq.nofldtxq10g;
871 } else {
872 pi->nofldrxq = iaq.nofldrxq1g;
873 pi->nofldtxq = iaq.nofldtxq1g;
874 }
875 ofld_rqidx += pi->nofldrxq;
876 ofld_tqidx += pi->nofldtxq;
877 }
878 #endif
879 #ifdef DEV_NETMAP
880 pi->first_nm_rxq = nm_rqidx;
881 pi->first_nm_txq = nm_tqidx;
882 if (is_10G_port(pi) || is_40G_port(pi)) {
883 pi->nnmrxq = iaq.nnmrxq10g;
884 pi->nnmtxq = iaq.nnmtxq10g;
885 } else {
886 pi->nnmrxq = iaq.nnmrxq1g;
887 pi->nnmtxq = iaq.nnmtxq1g;
888 }
889 nm_rqidx += pi->nnmrxq;
890 nm_tqidx += pi->nnmtxq;
891 #endif
892 }
893
894 rc = setup_intr_handlers(sc);
895 if (rc != 0) {
896 device_printf(dev,
897 "failed to setup interrupt handlers: %d\n", rc);
898 goto done;
899 }
900
901 rc = bus_generic_attach(dev);
902 if (rc != 0) {
903 device_printf(dev,
904 "failed to attach all child ports: %d\n", rc);
905 goto done;
906 }
907
908 device_printf(dev,
909 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
910 sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
911 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
912 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
913 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
914
915 t4_set_desc(sc);
916
917 done:
918 if (rc != 0 && sc->cdev) {
919 /* cdev was created and so cxgbetool works; recover that way. */
920 device_printf(dev,
921 "error during attach, adapter is now in recovery mode.\n");
922 rc = 0;
923 }
924
925 if (rc != 0)
926 t4_detach(dev);
927 else
928 t4_sysctls(sc);
929
930 return (rc);
931 }
932
933 /*
934 * Idempotent
935 */
936 static int
t4_detach(device_t dev)937 t4_detach(device_t dev)
938 {
939 struct adapter *sc;
940 struct port_info *pi;
941 int i, rc;
942
943 sc = device_get_softc(dev);
944
945 if (sc->flags & FULL_INIT_DONE)
946 t4_intr_disable(sc);
947
948 if (sc->cdev) {
949 destroy_dev(sc->cdev);
950 sc->cdev = NULL;
951 }
952
953 rc = bus_generic_detach(dev);
954 if (rc) {
955 device_printf(dev,
956 "failed to detach child devices: %d\n", rc);
957 return (rc);
958 }
959
960 for (i = 0; i < sc->intr_count; i++)
961 t4_free_irq(sc, &sc->irq[i]);
962
963 for (i = 0; i < MAX_NPORTS; i++) {
964 pi = sc->port[i];
965 if (pi) {
966 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
967 if (pi->dev)
968 device_delete_child(dev, pi->dev);
969
970 mtx_destroy(&pi->pi_lock);
971 free(pi, M_CXGBE);
972 }
973 }
974
975 if (sc->flags & FULL_INIT_DONE)
976 adapter_full_uninit(sc);
977
978 if (sc->flags & FW_OK)
979 t4_fw_bye(sc, sc->mbox);
980
981 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
982 pci_release_msi(dev);
983
984 if (sc->regs_res)
985 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
986 sc->regs_res);
987
988 if (sc->udbs_res)
989 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
990 sc->udbs_res);
991
992 if (sc->msix_res)
993 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
994 sc->msix_res);
995
996 if (sc->l2t)
997 t4_free_l2t(sc->l2t);
998
999 #ifdef TCP_OFFLOAD
1000 free(sc->sge.ofld_rxq, M_CXGBE);
1001 free(sc->sge.ofld_txq, M_CXGBE);
1002 #endif
1003 #ifdef DEV_NETMAP
1004 free(sc->sge.nm_rxq, M_CXGBE);
1005 free(sc->sge.nm_txq, M_CXGBE);
1006 #endif
1007 free(sc->irq, M_CXGBE);
1008 free(sc->sge.rxq, M_CXGBE);
1009 free(sc->sge.txq, M_CXGBE);
1010 free(sc->sge.ctrlq, M_CXGBE);
1011 free(sc->sge.iqmap, M_CXGBE);
1012 free(sc->sge.eqmap, M_CXGBE);
1013 free(sc->tids.ftid_tab, M_CXGBE);
1014 t4_destroy_dma_tag(sc);
1015 if (mtx_initialized(&sc->sc_lock)) {
1016 sx_xlock(&t4_list_lock);
1017 SLIST_REMOVE(&t4_list, sc, adapter, link);
1018 sx_xunlock(&t4_list_lock);
1019 mtx_destroy(&sc->sc_lock);
1020 }
1021
1022 if (mtx_initialized(&sc->tids.ftid_lock))
1023 mtx_destroy(&sc->tids.ftid_lock);
1024 if (mtx_initialized(&sc->sfl_lock))
1025 mtx_destroy(&sc->sfl_lock);
1026 if (mtx_initialized(&sc->ifp_lock))
1027 mtx_destroy(&sc->ifp_lock);
1028 if (mtx_initialized(&sc->regwin_lock))
1029 mtx_destroy(&sc->regwin_lock);
1030
1031 bzero(sc, sizeof(*sc));
1032
1033 return (0);
1034 }
1035
1036 static int
cxgbe_probe(device_t dev)1037 cxgbe_probe(device_t dev)
1038 {
1039 char buf[128];
1040 struct port_info *pi = device_get_softc(dev);
1041
1042 snprintf(buf, sizeof(buf), "port %d", pi->port_id);
1043 device_set_desc_copy(dev, buf);
1044
1045 return (BUS_PROBE_DEFAULT);
1046 }
1047
1048 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
1049 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
1050 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS)
1051 #define T4_CAP_ENABLE (T4_CAP)
1052
1053 static int
cxgbe_attach(device_t dev)1054 cxgbe_attach(device_t dev)
1055 {
1056 struct port_info *pi = device_get_softc(dev);
1057 struct ifnet *ifp;
1058 char *s;
1059 int n, o;
1060
1061 /* Allocate an ifnet and set it up */
1062 ifp = if_alloc(IFT_ETHER);
1063 if (ifp == NULL) {
1064 device_printf(dev, "Cannot allocate ifnet\n");
1065 return (ENOMEM);
1066 }
1067 pi->ifp = ifp;
1068 ifp->if_softc = pi;
1069
1070 callout_init(&pi->tick, CALLOUT_MPSAFE);
1071
1072 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1073 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1074
1075 ifp->if_init = cxgbe_init;
1076 ifp->if_ioctl = cxgbe_ioctl;
1077 ifp->if_transmit = cxgbe_transmit;
1078 ifp->if_qflush = cxgbe_qflush;
1079
1080 ifp->if_capabilities = T4_CAP;
1081 #ifdef TCP_OFFLOAD
1082 if (is_offload(pi->adapter))
1083 ifp->if_capabilities |= IFCAP_TOE;
1084 #endif
1085 ifp->if_capenable = T4_CAP_ENABLE;
1086 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1087 CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1088
1089 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
1090 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS;
1091 ifp->if_hw_tsomaxsegsize = 65536;
1092
1093 /* Initialize ifmedia for this port */
1094 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1095 cxgbe_media_status);
1096 build_medialist(pi, &pi->media);
1097
1098 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
1099 EVENTHANDLER_PRI_ANY);
1100
1101 ether_ifattach(ifp, pi->hw_addr);
1102
1103 n = 128;
1104 s = malloc(n, M_CXGBE, M_WAITOK);
1105 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
1106 MPASS(n > o);
1107 #ifdef TCP_OFFLOAD
1108 if (is_offload(pi->adapter)) {
1109 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
1110 pi->nofldtxq, pi->nofldrxq);
1111 MPASS(n > o);
1112 }
1113 #endif
1114 #ifdef DEV_NETMAP
1115 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
1116 pi->nnmrxq);
1117 MPASS(n > o);
1118 #endif
1119 device_printf(dev, "%s\n", s);
1120 free(s, M_CXGBE);
1121
1122 #ifdef DEV_NETMAP
1123 /* nm_media handled here to keep implementation private to this file */
1124 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
1125 cxgbe_media_status);
1126 build_medialist(pi, &pi->nm_media);
1127 create_netmap_ifnet(pi); /* logs errors it something fails */
1128 #endif
1129 cxgbe_sysctls(pi);
1130
1131 return (0);
1132 }
1133
1134 static int
cxgbe_detach(device_t dev)1135 cxgbe_detach(device_t dev)
1136 {
1137 struct port_info *pi = device_get_softc(dev);
1138 struct adapter *sc = pi->adapter;
1139 struct ifnet *ifp = pi->ifp;
1140
1141 /* Tell if_ioctl and if_init that the port is going away */
1142 ADAPTER_LOCK(sc);
1143 SET_DOOMED(pi);
1144 wakeup(&sc->flags);
1145 while (IS_BUSY(sc))
1146 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
1147 SET_BUSY(sc);
1148 #ifdef INVARIANTS
1149 sc->last_op = "t4detach";
1150 sc->last_op_thr = curthread;
1151 #endif
1152 ADAPTER_UNLOCK(sc);
1153
1154 if (pi->flags & HAS_TRACEQ) {
1155 sc->traceq = -1; /* cloner should not create ifnet */
1156 t4_tracer_port_detach(sc);
1157 }
1158
1159 if (pi->vlan_c)
1160 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
1161
1162 PORT_LOCK(pi);
1163 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1164 callout_stop(&pi->tick);
1165 PORT_UNLOCK(pi);
1166 callout_drain(&pi->tick);
1167
1168 /* Let detach proceed even if these fail. */
1169 cxgbe_uninit_synchronized(pi);
1170 port_full_uninit(pi);
1171
1172 ifmedia_removeall(&pi->media);
1173 ether_ifdetach(pi->ifp);
1174 if_free(pi->ifp);
1175
1176 #ifdef DEV_NETMAP
1177 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
1178 destroy_netmap_ifnet(pi);
1179 #endif
1180
1181 ADAPTER_LOCK(sc);
1182 CLR_BUSY(sc);
1183 wakeup(&sc->flags);
1184 ADAPTER_UNLOCK(sc);
1185
1186 return (0);
1187 }
1188
1189 static void
cxgbe_init(void * arg)1190 cxgbe_init(void *arg)
1191 {
1192 struct port_info *pi = arg;
1193 struct adapter *sc = pi->adapter;
1194
1195 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
1196 return;
1197 cxgbe_init_synchronized(pi);
1198 end_synchronized_op(sc, 0);
1199 }
1200
1201 static int
cxgbe_ioctl(struct ifnet * ifp,unsigned long cmd,caddr_t data)1202 cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
1203 {
1204 int rc = 0, mtu, flags, can_sleep;
1205 struct port_info *pi = ifp->if_softc;
1206 struct adapter *sc = pi->adapter;
1207 struct ifreq *ifr = (struct ifreq *)data;
1208 uint32_t mask;
1209
1210 switch (cmd) {
1211 case SIOCSIFMTU:
1212 mtu = ifr->ifr_mtu;
1213 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
1214 return (EINVAL);
1215
1216 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
1217 if (rc)
1218 return (rc);
1219 ifp->if_mtu = mtu;
1220 if (pi->flags & PORT_INIT_DONE) {
1221 t4_update_fl_bufsize(ifp);
1222 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1223 rc = update_mac_settings(ifp, XGMAC_MTU);
1224 }
1225 end_synchronized_op(sc, 0);
1226 break;
1227
1228 case SIOCSIFFLAGS:
1229 can_sleep = 0;
1230 redo_sifflags:
1231 rc = begin_synchronized_op(sc, pi,
1232 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg");
1233 if (rc)
1234 return (rc);
1235
1236 if (ifp->if_flags & IFF_UP) {
1237 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1238 flags = pi->if_flags;
1239 if ((ifp->if_flags ^ flags) &
1240 (IFF_PROMISC | IFF_ALLMULTI)) {
1241 if (can_sleep == 1) {
1242 end_synchronized_op(sc, 0);
1243 can_sleep = 0;
1244 goto redo_sifflags;
1245 }
1246 rc = update_mac_settings(ifp,
1247 XGMAC_PROMISC | XGMAC_ALLMULTI);
1248 }
1249 } else {
1250 if (can_sleep == 0) {
1251 end_synchronized_op(sc, LOCK_HELD);
1252 can_sleep = 1;
1253 goto redo_sifflags;
1254 }
1255 rc = cxgbe_init_synchronized(pi);
1256 }
1257 pi->if_flags = ifp->if_flags;
1258 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1259 if (can_sleep == 0) {
1260 end_synchronized_op(sc, LOCK_HELD);
1261 can_sleep = 1;
1262 goto redo_sifflags;
1263 }
1264 rc = cxgbe_uninit_synchronized(pi);
1265 }
1266 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD);
1267 break;
1268
1269 case SIOCADDMULTI:
1270 case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1271 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1272 if (rc)
1273 return (rc);
1274 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1275 rc = update_mac_settings(ifp, XGMAC_MCADDRS);
1276 end_synchronized_op(sc, LOCK_HELD);
1277 break;
1278
1279 case SIOCSIFCAP:
1280 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1281 if (rc)
1282 return (rc);
1283
1284 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1285 if (mask & IFCAP_TXCSUM) {
1286 ifp->if_capenable ^= IFCAP_TXCSUM;
1287 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1288
1289 if (IFCAP_TSO4 & ifp->if_capenable &&
1290 !(IFCAP_TXCSUM & ifp->if_capenable)) {
1291 ifp->if_capenable &= ~IFCAP_TSO4;
1292 if_printf(ifp,
1293 "tso4 disabled due to -txcsum.\n");
1294 }
1295 }
1296 if (mask & IFCAP_TXCSUM_IPV6) {
1297 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1298 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1299
1300 if (IFCAP_TSO6 & ifp->if_capenable &&
1301 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1302 ifp->if_capenable &= ~IFCAP_TSO6;
1303 if_printf(ifp,
1304 "tso6 disabled due to -txcsum6.\n");
1305 }
1306 }
1307 if (mask & IFCAP_RXCSUM)
1308 ifp->if_capenable ^= IFCAP_RXCSUM;
1309 if (mask & IFCAP_RXCSUM_IPV6)
1310 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1311
1312 /*
1313 * Note that we leave CSUM_TSO alone (it is always set). The
1314 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1315 * sending a TSO request our way, so it's sufficient to toggle
1316 * IFCAP_TSOx only.
1317 */
1318 if (mask & IFCAP_TSO4) {
1319 if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1320 !(IFCAP_TXCSUM & ifp->if_capenable)) {
1321 if_printf(ifp, "enable txcsum first.\n");
1322 rc = EAGAIN;
1323 goto fail;
1324 }
1325 ifp->if_capenable ^= IFCAP_TSO4;
1326 }
1327 if (mask & IFCAP_TSO6) {
1328 if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1329 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1330 if_printf(ifp, "enable txcsum6 first.\n");
1331 rc = EAGAIN;
1332 goto fail;
1333 }
1334 ifp->if_capenable ^= IFCAP_TSO6;
1335 }
1336 if (mask & IFCAP_LRO) {
1337 #if defined(INET) || defined(INET6)
1338 int i;
1339 struct sge_rxq *rxq;
1340
1341 ifp->if_capenable ^= IFCAP_LRO;
1342 for_each_rxq(pi, i, rxq) {
1343 if (ifp->if_capenable & IFCAP_LRO)
1344 rxq->iq.flags |= IQ_LRO_ENABLED;
1345 else
1346 rxq->iq.flags &= ~IQ_LRO_ENABLED;
1347 }
1348 #endif
1349 }
1350 #ifdef TCP_OFFLOAD
1351 if (mask & IFCAP_TOE) {
1352 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1353
1354 rc = toe_capability(pi, enable);
1355 if (rc != 0)
1356 goto fail;
1357
1358 ifp->if_capenable ^= mask;
1359 }
1360 #endif
1361 if (mask & IFCAP_VLAN_HWTAGGING) {
1362 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1363 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1364 rc = update_mac_settings(ifp, XGMAC_VLANEX);
1365 }
1366 if (mask & IFCAP_VLAN_MTU) {
1367 ifp->if_capenable ^= IFCAP_VLAN_MTU;
1368
1369 /* Need to find out how to disable auto-mtu-inflation */
1370 }
1371 if (mask & IFCAP_VLAN_HWTSO)
1372 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1373 if (mask & IFCAP_VLAN_HWCSUM)
1374 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1375
1376 #ifdef VLAN_CAPABILITIES
1377 VLAN_CAPABILITIES(ifp);
1378 #endif
1379 fail:
1380 end_synchronized_op(sc, 0);
1381 break;
1382
1383 case SIOCSIFMEDIA:
1384 case SIOCGIFMEDIA:
1385 ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1386 break;
1387
1388 default:
1389 rc = ether_ioctl(ifp, cmd, data);
1390 }
1391
1392 return (rc);
1393 }
1394
1395 static int
cxgbe_transmit(struct ifnet * ifp,struct mbuf * m)1396 cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1397 {
1398 struct port_info *pi = ifp->if_softc;
1399 struct adapter *sc = pi->adapter;
1400 struct sge_txq *txq;
1401 void *items[1];
1402 int rc;
1403
1404 M_ASSERTPKTHDR(m);
1405 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */
1406
1407 if (__predict_false(pi->link_cfg.link_ok == 0)) {
1408 m_freem(m);
1409 return (ENETDOWN);
1410 }
1411
1412 rc = parse_pkt(&m);
1413 if (__predict_false(rc != 0)) {
1414 MPASS(m == NULL); /* was freed already */
1415 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
1416 return (rc);
1417 }
1418
1419 /* Select a txq. */
1420 txq = &sc->sge.txq[pi->first_txq];
1421 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
1422 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) +
1423 pi->rsrv_noflowq);
1424
1425 items[0] = m;
1426 rc = mp_ring_enqueue(txq->r, items, 1, 4096);
1427 if (__predict_false(rc != 0))
1428 m_freem(m);
1429
1430 return (rc);
1431 }
1432
1433 static void
cxgbe_qflush(struct ifnet * ifp)1434 cxgbe_qflush(struct ifnet *ifp)
1435 {
1436 struct port_info *pi = ifp->if_softc;
1437 struct sge_txq *txq;
1438 int i;
1439
1440 /* queues do not exist if !PORT_INIT_DONE. */
1441 if (pi->flags & PORT_INIT_DONE) {
1442 for_each_txq(pi, i, txq) {
1443 TXQ_LOCK(txq);
1444 txq->eq.flags &= ~EQ_ENABLED;
1445 TXQ_UNLOCK(txq);
1446 while (!mp_ring_is_idle(txq->r)) {
1447 mp_ring_check_drainage(txq->r, 0);
1448 pause("qflush", 1);
1449 }
1450 }
1451 }
1452 if_qflush(ifp);
1453 }
1454
1455 static int
cxgbe_media_change(struct ifnet * ifp)1456 cxgbe_media_change(struct ifnet *ifp)
1457 {
1458 struct port_info *pi = ifp->if_softc;
1459
1460 device_printf(pi->dev, "%s unimplemented.\n", __func__);
1461
1462 return (EOPNOTSUPP);
1463 }
1464
1465 static void
cxgbe_media_status(struct ifnet * ifp,struct ifmediareq * ifmr)1466 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1467 {
1468 struct port_info *pi = ifp->if_softc;
1469 struct ifmedia *media = NULL;
1470 struct ifmedia_entry *cur;
1471 int speed = pi->link_cfg.speed;
1472 #ifdef INVARIANTS
1473 int data = (pi->port_type << 8) | pi->mod_type;
1474 #endif
1475
1476 if (ifp == pi->ifp)
1477 media = &pi->media;
1478 #ifdef DEV_NETMAP
1479 else if (ifp == pi->nm_ifp)
1480 media = &pi->nm_media;
1481 #endif
1482 MPASS(media != NULL);
1483
1484 cur = media->ifm_cur;
1485 MPASS(cur->ifm_data == data);
1486
1487 ifmr->ifm_status = IFM_AVALID;
1488 if (!pi->link_cfg.link_ok)
1489 return;
1490
1491 ifmr->ifm_status |= IFM_ACTIVE;
1492
1493 /* active and current will differ iff current media is autoselect. */
1494 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1495 return;
1496
1497 ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1498 if (speed == SPEED_10000)
1499 ifmr->ifm_active |= IFM_10G_T;
1500 else if (speed == SPEED_1000)
1501 ifmr->ifm_active |= IFM_1000_T;
1502 else if (speed == SPEED_100)
1503 ifmr->ifm_active |= IFM_100_TX;
1504 else if (speed == SPEED_10)
1505 ifmr->ifm_active |= IFM_10_T;
1506 else
1507 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1508 speed));
1509 }
1510
1511 void
t4_fatal_err(struct adapter * sc)1512 t4_fatal_err(struct adapter *sc)
1513 {
1514 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1515 t4_intr_disable(sc);
1516 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1517 device_get_nameunit(sc->dev));
1518 }
1519
1520 static int
map_bars_0_and_4(struct adapter * sc)1521 map_bars_0_and_4(struct adapter *sc)
1522 {
1523 sc->regs_rid = PCIR_BAR(0);
1524 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1525 &sc->regs_rid, RF_ACTIVE);
1526 if (sc->regs_res == NULL) {
1527 device_printf(sc->dev, "cannot map registers.\n");
1528 return (ENXIO);
1529 }
1530 sc->bt = rman_get_bustag(sc->regs_res);
1531 sc->bh = rman_get_bushandle(sc->regs_res);
1532 sc->mmio_len = rman_get_size(sc->regs_res);
1533 setbit(&sc->doorbells, DOORBELL_KDB);
1534
1535 sc->msix_rid = PCIR_BAR(4);
1536 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1537 &sc->msix_rid, RF_ACTIVE);
1538 if (sc->msix_res == NULL) {
1539 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1540 return (ENXIO);
1541 }
1542
1543 return (0);
1544 }
1545
1546 static int
map_bar_2(struct adapter * sc)1547 map_bar_2(struct adapter *sc)
1548 {
1549
1550 /*
1551 * T4: only iWARP driver uses the userspace doorbells. There is no need
1552 * to map it if RDMA is disabled.
1553 */
1554 if (is_t4(sc) && sc->rdmacaps == 0)
1555 return (0);
1556
1557 sc->udbs_rid = PCIR_BAR(2);
1558 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1559 &sc->udbs_rid, RF_ACTIVE);
1560 if (sc->udbs_res == NULL) {
1561 device_printf(sc->dev, "cannot map doorbell BAR.\n");
1562 return (ENXIO);
1563 }
1564 sc->udbs_base = rman_get_virtual(sc->udbs_res);
1565
1566 if (is_t5(sc)) {
1567 setbit(&sc->doorbells, DOORBELL_UDB);
1568 #if defined(__i386__) || defined(__amd64__)
1569 if (t5_write_combine) {
1570 int rc;
1571
1572 /*
1573 * Enable write combining on BAR2. This is the
1574 * userspace doorbell BAR and is split into 128B
1575 * (UDBS_SEG_SIZE) doorbell regions, each associated
1576 * with an egress queue. The first 64B has the doorbell
1577 * and the second 64B can be used to submit a tx work
1578 * request with an implicit doorbell.
1579 */
1580
1581 rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
1582 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
1583 if (rc == 0) {
1584 clrbit(&sc->doorbells, DOORBELL_UDB);
1585 setbit(&sc->doorbells, DOORBELL_WCWR);
1586 setbit(&sc->doorbells, DOORBELL_UDBWC);
1587 } else {
1588 device_printf(sc->dev,
1589 "couldn't enable write combining: %d\n",
1590 rc);
1591 }
1592
1593 t4_write_reg(sc, A_SGE_STAT_CFG,
1594 V_STATSOURCE_T5(7) | V_STATMODE(0));
1595 }
1596 #endif
1597 }
1598
1599 return (0);
1600 }
1601
1602 static const struct memwin t4_memwin[] = {
1603 { MEMWIN0_BASE, MEMWIN0_APERTURE },
1604 { MEMWIN1_BASE, MEMWIN1_APERTURE },
1605 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
1606 };
1607
1608 static const struct memwin t5_memwin[] = {
1609 { MEMWIN0_BASE, MEMWIN0_APERTURE },
1610 { MEMWIN1_BASE, MEMWIN1_APERTURE },
1611 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1612 };
1613
1614 static void
setup_memwin(struct adapter * sc)1615 setup_memwin(struct adapter *sc)
1616 {
1617 const struct memwin *mw;
1618 int i, n;
1619 uint32_t bar0;
1620
1621 if (is_t4(sc)) {
1622 /*
1623 * Read low 32b of bar0 indirectly via the hardware backdoor
1624 * mechanism. Works from within PCI passthrough environments
1625 * too, where rman_get_start() can return a different value. We
1626 * need to program the T4 memory window decoders with the actual
1627 * addresses that will be coming across the PCIe link.
1628 */
1629 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1630 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1631
1632 mw = &t4_memwin[0];
1633 n = nitems(t4_memwin);
1634 } else {
1635 /* T5 uses the relative offset inside the PCIe BAR */
1636 bar0 = 0;
1637
1638 mw = &t5_memwin[0];
1639 n = nitems(t5_memwin);
1640 }
1641
1642 for (i = 0; i < n; i++, mw++) {
1643 t4_write_reg(sc,
1644 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
1645 (mw->base + bar0) | V_BIR(0) |
1646 V_WINDOW(ilog2(mw->aperture) - 10));
1647 }
1648
1649 /* flush */
1650 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1651 }
1652
1653 /*
1654 * Verify that the memory range specified by the addr/len pair is valid and lies
1655 * entirely within a single region (EDCx or MCx).
1656 */
1657 static int
validate_mem_range(struct adapter * sc,uint32_t addr,int len)1658 validate_mem_range(struct adapter *sc, uint32_t addr, int len)
1659 {
1660 uint32_t em, addr_len, maddr, mlen;
1661
1662 /* Memory can only be accessed in naturally aligned 4 byte units */
1663 if (addr & 3 || len & 3 || len == 0)
1664 return (EINVAL);
1665
1666 /* Enabled memories */
1667 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1668 if (em & F_EDRAM0_ENABLE) {
1669 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1670 maddr = G_EDRAM0_BASE(addr_len) << 20;
1671 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1672 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1673 addr + len <= maddr + mlen)
1674 return (0);
1675 }
1676 if (em & F_EDRAM1_ENABLE) {
1677 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1678 maddr = G_EDRAM1_BASE(addr_len) << 20;
1679 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1680 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1681 addr + len <= maddr + mlen)
1682 return (0);
1683 }
1684 if (em & F_EXT_MEM_ENABLE) {
1685 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1686 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1687 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1688 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1689 addr + len <= maddr + mlen)
1690 return (0);
1691 }
1692 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) {
1693 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1694 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1695 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1696 if (mlen > 0 && addr >= maddr && addr < maddr + mlen &&
1697 addr + len <= maddr + mlen)
1698 return (0);
1699 }
1700
1701 return (EFAULT);
1702 }
1703
1704 static int
fwmtype_to_hwmtype(int mtype)1705 fwmtype_to_hwmtype(int mtype)
1706 {
1707
1708 switch (mtype) {
1709 case FW_MEMTYPE_EDC0:
1710 return (MEM_EDC0);
1711 case FW_MEMTYPE_EDC1:
1712 return (MEM_EDC1);
1713 case FW_MEMTYPE_EXTMEM:
1714 return (MEM_MC0);
1715 case FW_MEMTYPE_EXTMEM1:
1716 return (MEM_MC1);
1717 default:
1718 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
1719 }
1720 }
1721
1722 /*
1723 * Verify that the memory range specified by the memtype/offset/len pair is
1724 * valid and lies entirely within the memtype specified. The global address of
1725 * the start of the range is returned in addr.
1726 */
1727 static int
validate_mt_off_len(struct adapter * sc,int mtype,uint32_t off,int len,uint32_t * addr)1728 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1729 uint32_t *addr)
1730 {
1731 uint32_t em, addr_len, maddr, mlen;
1732
1733 /* Memory can only be accessed in naturally aligned 4 byte units */
1734 if (off & 3 || len & 3 || len == 0)
1735 return (EINVAL);
1736
1737 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1738 switch (fwmtype_to_hwmtype(mtype)) {
1739 case MEM_EDC0:
1740 if (!(em & F_EDRAM0_ENABLE))
1741 return (EINVAL);
1742 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1743 maddr = G_EDRAM0_BASE(addr_len) << 20;
1744 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1745 break;
1746 case MEM_EDC1:
1747 if (!(em & F_EDRAM1_ENABLE))
1748 return (EINVAL);
1749 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1750 maddr = G_EDRAM1_BASE(addr_len) << 20;
1751 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1752 break;
1753 case MEM_MC:
1754 if (!(em & F_EXT_MEM_ENABLE))
1755 return (EINVAL);
1756 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1757 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1758 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1759 break;
1760 case MEM_MC1:
1761 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE))
1762 return (EINVAL);
1763 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1764 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1765 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1766 break;
1767 default:
1768 return (EINVAL);
1769 }
1770
1771 if (mlen > 0 && off < mlen && off + len <= mlen) {
1772 *addr = maddr + off; /* global address */
1773 return (0);
1774 }
1775
1776 return (EFAULT);
1777 }
1778
1779 static void
memwin_info(struct adapter * sc,int win,uint32_t * base,uint32_t * aperture)1780 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1781 {
1782 const struct memwin *mw;
1783
1784 if (is_t4(sc)) {
1785 KASSERT(win >= 0 && win < nitems(t4_memwin),
1786 ("%s: incorrect memwin# (%d)", __func__, win));
1787 mw = &t4_memwin[win];
1788 } else {
1789 KASSERT(win >= 0 && win < nitems(t5_memwin),
1790 ("%s: incorrect memwin# (%d)", __func__, win));
1791 mw = &t5_memwin[win];
1792 }
1793
1794 if (base != NULL)
1795 *base = mw->base;
1796 if (aperture != NULL)
1797 *aperture = mw->aperture;
1798 }
1799
1800 /*
1801 * Positions the memory window such that it can be used to access the specified
1802 * address in the chip's address space. The return value is the offset of addr
1803 * from the start of the window.
1804 */
1805 static uint32_t
position_memwin(struct adapter * sc,int n,uint32_t addr)1806 position_memwin(struct adapter *sc, int n, uint32_t addr)
1807 {
1808 uint32_t start, pf;
1809 uint32_t reg;
1810
1811 KASSERT(n >= 0 && n <= 3,
1812 ("%s: invalid window %d.", __func__, n));
1813 KASSERT((addr & 3) == 0,
1814 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr));
1815
1816 if (is_t4(sc)) {
1817 pf = 0;
1818 start = addr & ~0xf; /* start must be 16B aligned */
1819 } else {
1820 pf = V_PFNUM(sc->pf);
1821 start = addr & ~0x7f; /* start must be 128B aligned */
1822 }
1823 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1824
1825 t4_write_reg(sc, reg, start | pf);
1826 t4_read_reg(sc, reg);
1827
1828 return (addr - start);
1829 }
1830
1831 static int
cfg_itype_and_nqueues(struct adapter * sc,int n10g,int n1g,struct intrs_and_queues * iaq)1832 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1833 struct intrs_and_queues *iaq)
1834 {
1835 int rc, itype, navail, nrxq10g, nrxq1g, n;
1836 int nofldrxq10g = 0, nofldrxq1g = 0;
1837 int nnmrxq10g = 0, nnmrxq1g = 0;
1838
1839 bzero(iaq, sizeof(*iaq));
1840
1841 iaq->ntxq10g = t4_ntxq10g;
1842 iaq->ntxq1g = t4_ntxq1g;
1843 iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1844 iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1845 iaq->rsrv_noflowq = t4_rsrv_noflowq;
1846 #ifdef TCP_OFFLOAD
1847 if (is_offload(sc)) {
1848 iaq->nofldtxq10g = t4_nofldtxq10g;
1849 iaq->nofldtxq1g = t4_nofldtxq1g;
1850 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1851 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1852 }
1853 #endif
1854 #ifdef DEV_NETMAP
1855 iaq->nnmtxq10g = t4_nnmtxq10g;
1856 iaq->nnmtxq1g = t4_nnmtxq1g;
1857 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
1858 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
1859 #endif
1860
1861 for (itype = INTR_MSIX; itype; itype >>= 1) {
1862
1863 if ((itype & t4_intr_types) == 0)
1864 continue; /* not allowed */
1865
1866 if (itype == INTR_MSIX)
1867 navail = pci_msix_count(sc->dev);
1868 else if (itype == INTR_MSI)
1869 navail = pci_msi_count(sc->dev);
1870 else
1871 navail = 1;
1872 restart:
1873 if (navail == 0)
1874 continue;
1875
1876 iaq->intr_type = itype;
1877 iaq->intr_flags_10g = 0;
1878 iaq->intr_flags_1g = 0;
1879
1880 /*
1881 * Best option: an interrupt vector for errors, one for the
1882 * firmware event queue, and one for every rxq (NIC, TOE, and
1883 * netmap).
1884 */
1885 iaq->nirq = T4_EXTRA_INTR;
1886 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
1887 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
1888 if (iaq->nirq <= navail &&
1889 (itype != INTR_MSI || powerof2(iaq->nirq))) {
1890 iaq->intr_flags_10g = INTR_ALL;
1891 iaq->intr_flags_1g = INTR_ALL;
1892 goto allocate;
1893 }
1894
1895 /*
1896 * Second best option: a vector for errors, one for the firmware
1897 * event queue, and vectors for either all the NIC rx queues or
1898 * all the TOE rx queues. The queues that don't get vectors
1899 * will forward their interrupts to those that do.
1900 *
1901 * Note: netmap rx queues cannot be created early and so they
1902 * can't be setup to receive forwarded interrupts for others.
1903 */
1904 iaq->nirq = T4_EXTRA_INTR;
1905 if (nrxq10g >= nofldrxq10g) {
1906 iaq->intr_flags_10g = INTR_RXQ;
1907 iaq->nirq += n10g * nrxq10g;
1908 #ifdef DEV_NETMAP
1909 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
1910 #endif
1911 } else {
1912 iaq->intr_flags_10g = INTR_OFLD_RXQ;
1913 iaq->nirq += n10g * nofldrxq10g;
1914 #ifdef DEV_NETMAP
1915 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
1916 #endif
1917 }
1918 if (nrxq1g >= nofldrxq1g) {
1919 iaq->intr_flags_1g = INTR_RXQ;
1920 iaq->nirq += n1g * nrxq1g;
1921 #ifdef DEV_NETMAP
1922 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
1923 #endif
1924 } else {
1925 iaq->intr_flags_1g = INTR_OFLD_RXQ;
1926 iaq->nirq += n1g * nofldrxq1g;
1927 #ifdef DEV_NETMAP
1928 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
1929 #endif
1930 }
1931 if (iaq->nirq <= navail &&
1932 (itype != INTR_MSI || powerof2(iaq->nirq)))
1933 goto allocate;
1934
1935 /*
1936 * Next best option: an interrupt vector for errors, one for the
1937 * firmware event queue, and at least one per port. At this
1938 * point we know we'll have to downsize nrxq and/or nofldrxq
1939 * and/or nnmrxq to fit what's available to us.
1940 */
1941 iaq->nirq = T4_EXTRA_INTR;
1942 iaq->nirq += n10g + n1g;
1943 if (iaq->nirq <= navail) {
1944 int leftover = navail - iaq->nirq;
1945
1946 if (n10g > 0) {
1947 int target = max(nrxq10g, nofldrxq10g);
1948
1949 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
1950 INTR_RXQ : INTR_OFLD_RXQ;
1951
1952 n = 1;
1953 while (n < target && leftover >= n10g) {
1954 leftover -= n10g;
1955 iaq->nirq += n10g;
1956 n++;
1957 }
1958 iaq->nrxq10g = min(n, nrxq10g);
1959 #ifdef TCP_OFFLOAD
1960 iaq->nofldrxq10g = min(n, nofldrxq10g);
1961 #endif
1962 #ifdef DEV_NETMAP
1963 iaq->nnmrxq10g = min(n, nnmrxq10g);
1964 #endif
1965 }
1966
1967 if (n1g > 0) {
1968 int target = max(nrxq1g, nofldrxq1g);
1969
1970 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
1971 INTR_RXQ : INTR_OFLD_RXQ;
1972
1973 n = 1;
1974 while (n < target && leftover >= n1g) {
1975 leftover -= n1g;
1976 iaq->nirq += n1g;
1977 n++;
1978 }
1979 iaq->nrxq1g = min(n, nrxq1g);
1980 #ifdef TCP_OFFLOAD
1981 iaq->nofldrxq1g = min(n, nofldrxq1g);
1982 #endif
1983 #ifdef DEV_NETMAP
1984 iaq->nnmrxq1g = min(n, nnmrxq1g);
1985 #endif
1986 }
1987
1988 if (itype != INTR_MSI || powerof2(iaq->nirq))
1989 goto allocate;
1990 }
1991
1992 /*
1993 * Least desirable option: one interrupt vector for everything.
1994 */
1995 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
1996 iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
1997 #ifdef TCP_OFFLOAD
1998 if (is_offload(sc))
1999 iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2000 #endif
2001 #ifdef DEV_NETMAP
2002 iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
2003 #endif
2004
2005 allocate:
2006 navail = iaq->nirq;
2007 rc = 0;
2008 if (itype == INTR_MSIX)
2009 rc = pci_alloc_msix(sc->dev, &navail);
2010 else if (itype == INTR_MSI)
2011 rc = pci_alloc_msi(sc->dev, &navail);
2012
2013 if (rc == 0) {
2014 if (navail == iaq->nirq)
2015 return (0);
2016
2017 /*
2018 * Didn't get the number requested. Use whatever number
2019 * the kernel is willing to allocate (it's in navail).
2020 */
2021 device_printf(sc->dev, "fewer vectors than requested, "
2022 "type=%d, req=%d, rcvd=%d; will downshift req.\n",
2023 itype, iaq->nirq, navail);
2024 pci_release_msi(sc->dev);
2025 goto restart;
2026 }
2027
2028 device_printf(sc->dev,
2029 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
2030 itype, rc, iaq->nirq, navail);
2031 }
2032
2033 device_printf(sc->dev,
2034 "failed to find a usable interrupt type. "
2035 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
2036 pci_msix_count(sc->dev), pci_msi_count(sc->dev));
2037
2038 return (ENXIO);
2039 }
2040
2041 #define FW_VERSION(chip) ( \
2042 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
2043 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
2044 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
2045 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
2046 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
2047
2048 struct fw_info {
2049 uint8_t chip;
2050 char *kld_name;
2051 char *fw_mod_name;
2052 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */
2053 } fw_info[] = {
2054 {
2055 .chip = CHELSIO_T4,
2056 .kld_name = "t4fw_cfg",
2057 .fw_mod_name = "t4fw",
2058 .fw_hdr = {
2059 .chip = FW_HDR_CHIP_T4,
2060 .fw_ver = htobe32_const(FW_VERSION(T4)),
2061 .intfver_nic = FW_INTFVER(T4, NIC),
2062 .intfver_vnic = FW_INTFVER(T4, VNIC),
2063 .intfver_ofld = FW_INTFVER(T4, OFLD),
2064 .intfver_ri = FW_INTFVER(T4, RI),
2065 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
2066 .intfver_iscsi = FW_INTFVER(T4, ISCSI),
2067 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
2068 .intfver_fcoe = FW_INTFVER(T4, FCOE),
2069 },
2070 }, {
2071 .chip = CHELSIO_T5,
2072 .kld_name = "t5fw_cfg",
2073 .fw_mod_name = "t5fw",
2074 .fw_hdr = {
2075 .chip = FW_HDR_CHIP_T5,
2076 .fw_ver = htobe32_const(FW_VERSION(T5)),
2077 .intfver_nic = FW_INTFVER(T5, NIC),
2078 .intfver_vnic = FW_INTFVER(T5, VNIC),
2079 .intfver_ofld = FW_INTFVER(T5, OFLD),
2080 .intfver_ri = FW_INTFVER(T5, RI),
2081 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
2082 .intfver_iscsi = FW_INTFVER(T5, ISCSI),
2083 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
2084 .intfver_fcoe = FW_INTFVER(T5, FCOE),
2085 },
2086 }
2087 };
2088
2089 static struct fw_info *
find_fw_info(int chip)2090 find_fw_info(int chip)
2091 {
2092 int i;
2093
2094 for (i = 0; i < nitems(fw_info); i++) {
2095 if (fw_info[i].chip == chip)
2096 return (&fw_info[i]);
2097 }
2098 return (NULL);
2099 }
2100
2101 /*
2102 * Is the given firmware API compatible with the one the driver was compiled
2103 * with?
2104 */
2105 static int
fw_compatible(const struct fw_hdr * hdr1,const struct fw_hdr * hdr2)2106 fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
2107 {
2108
2109 /* short circuit if it's the exact same firmware version */
2110 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
2111 return (1);
2112
2113 /*
2114 * XXX: Is this too conservative? Perhaps I should limit this to the
2115 * features that are supported in the driver.
2116 */
2117 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
2118 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
2119 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
2120 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
2121 return (1);
2122 #undef SAME_INTF
2123
2124 return (0);
2125 }
2126
2127 /*
2128 * The firmware in the KLD is usable, but should it be installed? This routine
2129 * explains itself in detail if it indicates the KLD firmware should be
2130 * installed.
2131 */
2132 static int
should_install_kld_fw(struct adapter * sc,int card_fw_usable,int k,int c)2133 should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c)
2134 {
2135 const char *reason;
2136
2137 if (!card_fw_usable) {
2138 reason = "incompatible or unusable";
2139 goto install;
2140 }
2141
2142 if (k > c) {
2143 reason = "older than the version bundled with this driver";
2144 goto install;
2145 }
2146
2147 if (t4_fw_install == 2 && k != c) {
2148 reason = "different than the version bundled with this driver";
2149 goto install;
2150 }
2151
2152 return (0);
2153
2154 install:
2155 if (t4_fw_install == 0) {
2156 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2157 "but the driver is prohibited from installing a different "
2158 "firmware on the card.\n",
2159 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2160 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
2161
2162 return (0);
2163 }
2164
2165 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
2166 "installing firmware %u.%u.%u.%u on card.\n",
2167 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2168 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
2169 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2170 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2171
2172 return (1);
2173 }
2174 /*
2175 * Establish contact with the firmware and determine if we are the master driver
2176 * or not, and whether we are responsible for chip initialization.
2177 */
2178 static int
prep_firmware(struct adapter * sc)2179 prep_firmware(struct adapter *sc)
2180 {
2181 const struct firmware *fw = NULL, *default_cfg;
2182 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1;
2183 enum dev_state state;
2184 struct fw_info *fw_info;
2185 struct fw_hdr *card_fw; /* fw on the card */
2186 const struct fw_hdr *kld_fw; /* fw in the KLD */
2187 const struct fw_hdr *drv_fw; /* fw header the driver was compiled
2188 against */
2189
2190 /* Contact firmware. */
2191 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
2192 if (rc < 0 || state == DEV_STATE_ERR) {
2193 rc = -rc;
2194 device_printf(sc->dev,
2195 "failed to connect to the firmware: %d, %d.\n", rc, state);
2196 return (rc);
2197 }
2198 pf = rc;
2199 if (pf == sc->mbox)
2200 sc->flags |= MASTER_PF;
2201 else if (state == DEV_STATE_UNINIT) {
2202 /*
2203 * We didn't get to be the master so we definitely won't be
2204 * configuring the chip. It's a bug if someone else hasn't
2205 * configured it already.
2206 */
2207 device_printf(sc->dev, "couldn't be master(%d), "
2208 "device not already initialized either(%d).\n", rc, state);
2209 return (EDOOFUS);
2210 }
2211
2212 /* This is the firmware whose headers the driver was compiled against */
2213 fw_info = find_fw_info(chip_id(sc));
2214 if (fw_info == NULL) {
2215 device_printf(sc->dev,
2216 "unable to look up firmware information for chip %d.\n",
2217 chip_id(sc));
2218 return (EINVAL);
2219 }
2220 drv_fw = &fw_info->fw_hdr;
2221
2222 /*
2223 * The firmware KLD contains many modules. The KLD name is also the
2224 * name of the module that contains the default config file.
2225 */
2226 default_cfg = firmware_get(fw_info->kld_name);
2227
2228 /* Read the header of the firmware on the card */
2229 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
2230 rc = -t4_read_flash(sc, FLASH_FW_START,
2231 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1);
2232 if (rc == 0)
2233 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw);
2234 else {
2235 device_printf(sc->dev,
2236 "Unable to read card's firmware header: %d\n", rc);
2237 card_fw_usable = 0;
2238 }
2239
2240 /* This is the firmware in the KLD */
2241 fw = firmware_get(fw_info->fw_mod_name);
2242 if (fw != NULL) {
2243 kld_fw = (const void *)fw->data;
2244 kld_fw_usable = fw_compatible(drv_fw, kld_fw);
2245 } else {
2246 kld_fw = NULL;
2247 kld_fw_usable = 0;
2248 }
2249
2250 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
2251 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) {
2252 /*
2253 * Common case: the firmware on the card is an exact match and
2254 * the KLD is an exact match too, or the KLD is
2255 * absent/incompatible. Note that t4_fw_install = 2 is ignored
2256 * here -- use cxgbetool loadfw if you want to reinstall the
2257 * same firmware as the one on the card.
2258 */
2259 } else if (kld_fw_usable && state == DEV_STATE_UNINIT &&
2260 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver),
2261 be32toh(card_fw->fw_ver))) {
2262
2263 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
2264 if (rc != 0) {
2265 device_printf(sc->dev,
2266 "failed to install firmware: %d\n", rc);
2267 goto done;
2268 }
2269
2270 /* Installed successfully, update the cached header too. */
2271 memcpy(card_fw, kld_fw, sizeof(*card_fw));
2272 card_fw_usable = 1;
2273 need_fw_reset = 0; /* already reset as part of load_fw */
2274 }
2275
2276 if (!card_fw_usable) {
2277 uint32_t d, c, k;
2278
2279 d = ntohl(drv_fw->fw_ver);
2280 c = ntohl(card_fw->fw_ver);
2281 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0;
2282
2283 device_printf(sc->dev, "Cannot find a usable firmware: "
2284 "fw_install %d, chip state %d, "
2285 "driver compiled with %d.%d.%d.%d, "
2286 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n",
2287 t4_fw_install, state,
2288 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
2289 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
2290 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
2291 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
2292 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
2293 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
2294 rc = EINVAL;
2295 goto done;
2296 }
2297
2298 /* We're using whatever's on the card and it's known to be good. */
2299 sc->params.fw_vers = ntohl(card_fw->fw_ver);
2300 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
2301 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2302 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2303 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2304 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2305 t4_get_tp_version(sc, &sc->params.tp_vers);
2306
2307 /* Reset device */
2308 if (need_fw_reset &&
2309 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) {
2310 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
2311 if (rc != ETIMEDOUT && rc != EIO)
2312 t4_fw_bye(sc, sc->mbox);
2313 goto done;
2314 }
2315 sc->flags |= FW_OK;
2316
2317 rc = get_params__pre_init(sc);
2318 if (rc != 0)
2319 goto done; /* error message displayed already */
2320
2321 /* Partition adapter resources as specified in the config file. */
2322 if (state == DEV_STATE_UNINIT) {
2323
2324 KASSERT(sc->flags & MASTER_PF,
2325 ("%s: trying to change chip settings when not master.",
2326 __func__));
2327
2328 rc = partition_resources(sc, default_cfg, fw_info->kld_name);
2329 if (rc != 0)
2330 goto done; /* error message displayed already */
2331
2332 t4_tweak_chip_settings(sc);
2333
2334 /* get basic stuff going */
2335 rc = -t4_fw_initialize(sc, sc->mbox);
2336 if (rc != 0) {
2337 device_printf(sc->dev, "fw init failed: %d.\n", rc);
2338 goto done;
2339 }
2340 } else {
2341 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf);
2342 sc->cfcsum = 0;
2343 }
2344
2345 done:
2346 free(card_fw, M_CXGBE);
2347 if (fw != NULL)
2348 firmware_put(fw, FIRMWARE_UNLOAD);
2349 if (default_cfg != NULL)
2350 firmware_put(default_cfg, FIRMWARE_UNLOAD);
2351
2352 return (rc);
2353 }
2354
2355 #define FW_PARAM_DEV(param) \
2356 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2357 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2358 #define FW_PARAM_PFVF(param) \
2359 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
2360 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
2361
2362 /*
2363 * Partition chip resources for use between various PFs, VFs, etc.
2364 */
2365 static int
partition_resources(struct adapter * sc,const struct firmware * default_cfg,const char * name_prefix)2366 partition_resources(struct adapter *sc, const struct firmware *default_cfg,
2367 const char *name_prefix)
2368 {
2369 const struct firmware *cfg = NULL;
2370 int rc = 0;
2371 struct fw_caps_config_cmd caps;
2372 uint32_t mtype, moff, finicsum, cfcsum;
2373
2374 /*
2375 * Figure out what configuration file to use. Pick the default config
2376 * file for the card if the user hasn't specified one explicitly.
2377 */
2378 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file);
2379 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
2380 /* Card specific overrides go here. */
2381 if (pci_get_device(sc->dev) == 0x440a)
2382 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF);
2383 if (is_fpga(sc))
2384 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF);
2385 }
2386
2387 /*
2388 * We need to load another module if the profile is anything except
2389 * "default" or "flash".
2390 */
2391 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 &&
2392 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2393 char s[32];
2394
2395 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file);
2396 cfg = firmware_get(s);
2397 if (cfg == NULL) {
2398 if (default_cfg != NULL) {
2399 device_printf(sc->dev,
2400 "unable to load module \"%s\" for "
2401 "configuration profile \"%s\", will use "
2402 "the default config file instead.\n",
2403 s, sc->cfg_file);
2404 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2405 "%s", DEFAULT_CF);
2406 } else {
2407 device_printf(sc->dev,
2408 "unable to load module \"%s\" for "
2409 "configuration profile \"%s\", will use "
2410 "the config file on the card's flash "
2411 "instead.\n", s, sc->cfg_file);
2412 snprintf(sc->cfg_file, sizeof(sc->cfg_file),
2413 "%s", FLASH_CF);
2414 }
2415 }
2416 }
2417
2418 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 &&
2419 default_cfg == NULL) {
2420 device_printf(sc->dev,
2421 "default config file not available, will use the config "
2422 "file on the card's flash instead.\n");
2423 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF);
2424 }
2425
2426 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) {
2427 u_int cflen, i, n;
2428 const uint32_t *cfdata;
2429 uint32_t param, val, addr, off, mw_base, mw_aperture;
2430
2431 KASSERT(cfg != NULL || default_cfg != NULL,
2432 ("%s: no config to upload", __func__));
2433
2434 /*
2435 * Ask the firmware where it wants us to upload the config file.
2436 */
2437 param = FW_PARAM_DEV(CF);
2438 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
2439 if (rc != 0) {
2440 /* No support for config file? Shouldn't happen. */
2441 device_printf(sc->dev,
2442 "failed to query config file location: %d.\n", rc);
2443 goto done;
2444 }
2445 mtype = G_FW_PARAMS_PARAM_Y(val);
2446 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
2447
2448 /*
2449 * XXX: sheer laziness. We deliberately added 4 bytes of
2450 * useless stuffing/comments at the end of the config file so
2451 * it's ok to simply throw away the last remaining bytes when
2452 * the config file is not an exact multiple of 4. This also
2453 * helps with the validate_mt_off_len check.
2454 */
2455 if (cfg != NULL) {
2456 cflen = cfg->datasize & ~3;
2457 cfdata = cfg->data;
2458 } else {
2459 cflen = default_cfg->datasize & ~3;
2460 cfdata = default_cfg->data;
2461 }
2462
2463 if (cflen > FLASH_CFG_MAX_SIZE) {
2464 device_printf(sc->dev,
2465 "config file too long (%d, max allowed is %d). "
2466 "Will try to use the config on the card, if any.\n",
2467 cflen, FLASH_CFG_MAX_SIZE);
2468 goto use_config_on_flash;
2469 }
2470
2471 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
2472 if (rc != 0) {
2473 device_printf(sc->dev,
2474 "%s: addr (%d/0x%x) or len %d is not valid: %d. "
2475 "Will try to use the config on the card, if any.\n",
2476 __func__, mtype, moff, cflen, rc);
2477 goto use_config_on_flash;
2478 }
2479
2480 memwin_info(sc, 2, &mw_base, &mw_aperture);
2481 while (cflen) {
2482 off = position_memwin(sc, 2, addr);
2483 n = min(cflen, mw_aperture - off);
2484 for (i = 0; i < n; i += 4)
2485 t4_write_reg(sc, mw_base + off + i, *cfdata++);
2486 cflen -= n;
2487 addr += n;
2488 }
2489 } else {
2490 use_config_on_flash:
2491 mtype = FW_MEMTYPE_FLASH;
2492 moff = t4_flash_cfg_addr(sc);
2493 }
2494
2495 bzero(&caps, sizeof(caps));
2496 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2497 F_FW_CMD_REQUEST | F_FW_CMD_READ);
2498 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
2499 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
2500 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps));
2501 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2502 if (rc != 0) {
2503 device_printf(sc->dev,
2504 "failed to pre-process config file: %d "
2505 "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
2506 goto done;
2507 }
2508
2509 finicsum = be32toh(caps.finicsum);
2510 cfcsum = be32toh(caps.cfcsum);
2511 if (finicsum != cfcsum) {
2512 device_printf(sc->dev,
2513 "WARNING: config file checksum mismatch: %08x %08x\n",
2514 finicsum, cfcsum);
2515 }
2516 sc->cfcsum = cfcsum;
2517
2518 #define LIMIT_CAPS(x) do { \
2519 caps.x &= htobe16(t4_##x##_allowed); \
2520 } while (0)
2521
2522 /*
2523 * Let the firmware know what features will (not) be used so it can tune
2524 * things accordingly.
2525 */
2526 LIMIT_CAPS(linkcaps);
2527 LIMIT_CAPS(niccaps);
2528 LIMIT_CAPS(toecaps);
2529 LIMIT_CAPS(rdmacaps);
2530 LIMIT_CAPS(iscsicaps);
2531 LIMIT_CAPS(fcoecaps);
2532 #undef LIMIT_CAPS
2533
2534 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2535 F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
2536 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2537 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
2538 if (rc != 0) {
2539 device_printf(sc->dev,
2540 "failed to process config file: %d.\n", rc);
2541 }
2542 done:
2543 if (cfg != NULL)
2544 firmware_put(cfg, FIRMWARE_UNLOAD);
2545 return (rc);
2546 }
2547
2548 /*
2549 * Retrieve parameters that are needed (or nice to have) very early.
2550 */
2551 static int
get_params__pre_init(struct adapter * sc)2552 get_params__pre_init(struct adapter *sc)
2553 {
2554 int rc;
2555 uint32_t param[2], val[2];
2556 struct fw_devlog_cmd cmd;
2557 struct devlog_params *dlog = &sc->params.devlog;
2558
2559 param[0] = FW_PARAM_DEV(PORTVEC);
2560 param[1] = FW_PARAM_DEV(CCLK);
2561 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2562 if (rc != 0) {
2563 device_printf(sc->dev,
2564 "failed to query parameters (pre_init): %d.\n", rc);
2565 return (rc);
2566 }
2567
2568 sc->params.portvec = val[0];
2569 sc->params.nports = bitcount32(val[0]);
2570 sc->params.vpd.cclk = val[1];
2571
2572 /* Read device log parameters. */
2573 bzero(&cmd, sizeof(cmd));
2574 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
2575 F_FW_CMD_REQUEST | F_FW_CMD_READ);
2576 cmd.retval_len16 = htobe32(FW_LEN16(cmd));
2577 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
2578 if (rc != 0) {
2579 device_printf(sc->dev,
2580 "failed to get devlog parameters: %d.\n", rc);
2581 bzero(dlog, sizeof (*dlog));
2582 rc = 0; /* devlog isn't critical for device operation */
2583 } else {
2584 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
2585 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
2586 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
2587 dlog->size = be32toh(cmd.memsize_devlog);
2588 }
2589
2590 return (rc);
2591 }
2592
2593 /*
2594 * Retrieve various parameters that are of interest to the driver. The device
2595 * has been initialized by the firmware at this point.
2596 */
2597 static int
get_params__post_init(struct adapter * sc)2598 get_params__post_init(struct adapter *sc)
2599 {
2600 int rc;
2601 uint32_t param[7], val[7];
2602 struct fw_caps_config_cmd caps;
2603
2604 param[0] = FW_PARAM_PFVF(IQFLINT_START);
2605 param[1] = FW_PARAM_PFVF(EQ_START);
2606 param[2] = FW_PARAM_PFVF(FILTER_START);
2607 param[3] = FW_PARAM_PFVF(FILTER_END);
2608 param[4] = FW_PARAM_PFVF(L2T_START);
2609 param[5] = FW_PARAM_PFVF(L2T_END);
2610 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2611 if (rc != 0) {
2612 device_printf(sc->dev,
2613 "failed to query parameters (post_init): %d.\n", rc);
2614 return (rc);
2615 }
2616
2617 sc->sge.iq_start = val[0];
2618 sc->sge.eq_start = val[1];
2619 sc->tids.ftid_base = val[2];
2620 sc->tids.nftids = val[3] - val[2] + 1;
2621 sc->params.ftid_min = val[2];
2622 sc->params.ftid_max = val[3];
2623 sc->vres.l2t.start = val[4];
2624 sc->vres.l2t.size = val[5] - val[4] + 1;
2625 KASSERT(sc->vres.l2t.size <= L2T_SIZE,
2626 ("%s: L2 table size (%u) larger than expected (%u)",
2627 __func__, sc->vres.l2t.size, L2T_SIZE));
2628
2629 /* get capabilites */
2630 bzero(&caps, sizeof(caps));
2631 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2632 F_FW_CMD_REQUEST | F_FW_CMD_READ);
2633 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
2634 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
2635 if (rc != 0) {
2636 device_printf(sc->dev,
2637 "failed to get card capabilities: %d.\n", rc);
2638 return (rc);
2639 }
2640
2641 #define READ_CAPS(x) do { \
2642 sc->x = htobe16(caps.x); \
2643 } while (0)
2644 READ_CAPS(linkcaps);
2645 READ_CAPS(niccaps);
2646 READ_CAPS(toecaps);
2647 READ_CAPS(rdmacaps);
2648 READ_CAPS(iscsicaps);
2649 READ_CAPS(fcoecaps);
2650
2651 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
2652 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
2653 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
2654 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2655 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
2656 if (rc != 0) {
2657 device_printf(sc->dev,
2658 "failed to query NIC parameters: %d.\n", rc);
2659 return (rc);
2660 }
2661 sc->tids.etid_base = val[0];
2662 sc->params.etid_min = val[0];
2663 sc->tids.netids = val[1] - val[0] + 1;
2664 sc->params.netids = sc->tids.netids;
2665 sc->params.eo_wr_cred = val[2];
2666 sc->params.ethoffload = 1;
2667 }
2668
2669 if (sc->toecaps) {
2670 /* query offload-related parameters */
2671 param[0] = FW_PARAM_DEV(NTID);
2672 param[1] = FW_PARAM_PFVF(SERVER_START);
2673 param[2] = FW_PARAM_PFVF(SERVER_END);
2674 param[3] = FW_PARAM_PFVF(TDDP_START);
2675 param[4] = FW_PARAM_PFVF(TDDP_END);
2676 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
2677 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2678 if (rc != 0) {
2679 device_printf(sc->dev,
2680 "failed to query TOE parameters: %d.\n", rc);
2681 return (rc);
2682 }
2683 sc->tids.ntids = val[0];
2684 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
2685 sc->tids.stid_base = val[1];
2686 sc->tids.nstids = val[2] - val[1] + 1;
2687 sc->vres.ddp.start = val[3];
2688 sc->vres.ddp.size = val[4] - val[3] + 1;
2689 sc->params.ofldq_wr_cred = val[5];
2690 sc->params.offload = 1;
2691 }
2692 if (sc->rdmacaps) {
2693 param[0] = FW_PARAM_PFVF(STAG_START);
2694 param[1] = FW_PARAM_PFVF(STAG_END);
2695 param[2] = FW_PARAM_PFVF(RQ_START);
2696 param[3] = FW_PARAM_PFVF(RQ_END);
2697 param[4] = FW_PARAM_PFVF(PBL_START);
2698 param[5] = FW_PARAM_PFVF(PBL_END);
2699 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2700 if (rc != 0) {
2701 device_printf(sc->dev,
2702 "failed to query RDMA parameters(1): %d.\n", rc);
2703 return (rc);
2704 }
2705 sc->vres.stag.start = val[0];
2706 sc->vres.stag.size = val[1] - val[0] + 1;
2707 sc->vres.rq.start = val[2];
2708 sc->vres.rq.size = val[3] - val[2] + 1;
2709 sc->vres.pbl.start = val[4];
2710 sc->vres.pbl.size = val[5] - val[4] + 1;
2711
2712 param[0] = FW_PARAM_PFVF(SQRQ_START);
2713 param[1] = FW_PARAM_PFVF(SQRQ_END);
2714 param[2] = FW_PARAM_PFVF(CQ_START);
2715 param[3] = FW_PARAM_PFVF(CQ_END);
2716 param[4] = FW_PARAM_PFVF(OCQ_START);
2717 param[5] = FW_PARAM_PFVF(OCQ_END);
2718 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
2719 if (rc != 0) {
2720 device_printf(sc->dev,
2721 "failed to query RDMA parameters(2): %d.\n", rc);
2722 return (rc);
2723 }
2724 sc->vres.qp.start = val[0];
2725 sc->vres.qp.size = val[1] - val[0] + 1;
2726 sc->vres.cq.start = val[2];
2727 sc->vres.cq.size = val[3] - val[2] + 1;
2728 sc->vres.ocq.start = val[4];
2729 sc->vres.ocq.size = val[5] - val[4] + 1;
2730 }
2731 if (sc->iscsicaps) {
2732 param[0] = FW_PARAM_PFVF(ISCSI_START);
2733 param[1] = FW_PARAM_PFVF(ISCSI_END);
2734 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
2735 if (rc != 0) {
2736 device_printf(sc->dev,
2737 "failed to query iSCSI parameters: %d.\n", rc);
2738 return (rc);
2739 }
2740 sc->vres.iscsi.start = val[0];
2741 sc->vres.iscsi.size = val[1] - val[0] + 1;
2742 }
2743
2744 /*
2745 * We've got the params we wanted to query via the firmware. Now grab
2746 * some others directly from the chip.
2747 */
2748 rc = t4_read_chip_settings(sc);
2749
2750 return (rc);
2751 }
2752
2753 static int
set_params__post_init(struct adapter * sc)2754 set_params__post_init(struct adapter *sc)
2755 {
2756 uint32_t param, val;
2757
2758 /* ask for encapsulated CPLs */
2759 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
2760 val = 1;
2761 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
2762
2763 return (0);
2764 }
2765
2766 #undef FW_PARAM_PFVF
2767 #undef FW_PARAM_DEV
2768
2769 static void
t4_set_desc(struct adapter * sc)2770 t4_set_desc(struct adapter *sc)
2771 {
2772 char buf[128];
2773 struct adapter_params *p = &sc->params;
2774
2775 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, "
2776 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "",
2777 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec);
2778
2779 device_set_desc_copy(sc->dev, buf);
2780 }
2781
2782 static void
build_medialist(struct port_info * pi,struct ifmedia * media)2783 build_medialist(struct port_info *pi, struct ifmedia *media)
2784 {
2785 int data, m;
2786
2787 PORT_LOCK(pi);
2788
2789 ifmedia_removeall(media);
2790
2791 m = IFM_ETHER | IFM_FDX;
2792 data = (pi->port_type << 8) | pi->mod_type;
2793
2794 switch(pi->port_type) {
2795 case FW_PORT_TYPE_BT_XFI:
2796 case FW_PORT_TYPE_BT_XAUI:
2797 ifmedia_add(media, m | IFM_10G_T, data, NULL);
2798 /* fall through */
2799
2800 case FW_PORT_TYPE_BT_SGMII:
2801 ifmedia_add(media, m | IFM_1000_T, data, NULL);
2802 ifmedia_add(media, m | IFM_100_TX, data, NULL);
2803 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2804 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2805 break;
2806
2807 case FW_PORT_TYPE_CX4:
2808 ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2809 ifmedia_set(media, m | IFM_10G_CX4);
2810 break;
2811
2812 case FW_PORT_TYPE_QSFP_10G:
2813 case FW_PORT_TYPE_SFP:
2814 case FW_PORT_TYPE_FIBER_XFI:
2815 case FW_PORT_TYPE_FIBER_XAUI:
2816 switch (pi->mod_type) {
2817
2818 case FW_PORT_MOD_TYPE_LR:
2819 ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2820 ifmedia_set(media, m | IFM_10G_LR);
2821 break;
2822
2823 case FW_PORT_MOD_TYPE_SR:
2824 ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2825 ifmedia_set(media, m | IFM_10G_SR);
2826 break;
2827
2828 case FW_PORT_MOD_TYPE_LRM:
2829 ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2830 ifmedia_set(media, m | IFM_10G_LRM);
2831 break;
2832
2833 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2834 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2835 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2836 ifmedia_set(media, m | IFM_10G_TWINAX);
2837 break;
2838
2839 case FW_PORT_MOD_TYPE_NONE:
2840 m &= ~IFM_FDX;
2841 ifmedia_add(media, m | IFM_NONE, data, NULL);
2842 ifmedia_set(media, m | IFM_NONE);
2843 break;
2844
2845 case FW_PORT_MOD_TYPE_NA:
2846 case FW_PORT_MOD_TYPE_ER:
2847 default:
2848 device_printf(pi->dev,
2849 "unknown port_type (%d), mod_type (%d)\n",
2850 pi->port_type, pi->mod_type);
2851 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2852 ifmedia_set(media, m | IFM_UNKNOWN);
2853 break;
2854 }
2855 break;
2856
2857 case FW_PORT_TYPE_QSFP:
2858 switch (pi->mod_type) {
2859
2860 case FW_PORT_MOD_TYPE_LR:
2861 ifmedia_add(media, m | IFM_40G_LR4, data, NULL);
2862 ifmedia_set(media, m | IFM_40G_LR4);
2863 break;
2864
2865 case FW_PORT_MOD_TYPE_SR:
2866 ifmedia_add(media, m | IFM_40G_SR4, data, NULL);
2867 ifmedia_set(media, m | IFM_40G_SR4);
2868 break;
2869
2870 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2871 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2872 ifmedia_add(media, m | IFM_40G_CR4, data, NULL);
2873 ifmedia_set(media, m | IFM_40G_CR4);
2874 break;
2875
2876 case FW_PORT_MOD_TYPE_NONE:
2877 m &= ~IFM_FDX;
2878 ifmedia_add(media, m | IFM_NONE, data, NULL);
2879 ifmedia_set(media, m | IFM_NONE);
2880 break;
2881
2882 default:
2883 device_printf(pi->dev,
2884 "unknown port_type (%d), mod_type (%d)\n",
2885 pi->port_type, pi->mod_type);
2886 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2887 ifmedia_set(media, m | IFM_UNKNOWN);
2888 break;
2889 }
2890 break;
2891
2892 default:
2893 device_printf(pi->dev,
2894 "unknown port_type (%d), mod_type (%d)\n", pi->port_type,
2895 pi->mod_type);
2896 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2897 ifmedia_set(media, m | IFM_UNKNOWN);
2898 break;
2899 }
2900
2901 PORT_UNLOCK(pi);
2902 }
2903
2904 #define FW_MAC_EXACT_CHUNK 7
2905
2906 /*
2907 * Program the port's XGMAC based on parameters in ifnet. The caller also
2908 * indicates which parameters should be programmed (the rest are left alone).
2909 */
2910 int
update_mac_settings(struct ifnet * ifp,int flags)2911 update_mac_settings(struct ifnet *ifp, int flags)
2912 {
2913 int rc = 0;
2914 struct port_info *pi = ifp->if_softc;
2915 struct adapter *sc = pi->adapter;
2916 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2917 uint16_t viid = 0xffff;
2918 int16_t *xact_addr_filt = NULL;
2919
2920 ASSERT_SYNCHRONIZED_OP(sc);
2921 KASSERT(flags, ("%s: not told what to update.", __func__));
2922
2923 if (ifp == pi->ifp) {
2924 viid = pi->viid;
2925 xact_addr_filt = &pi->xact_addr_filt;
2926 }
2927 #ifdef DEV_NETMAP
2928 else if (ifp == pi->nm_ifp) {
2929 viid = pi->nm_viid;
2930 xact_addr_filt = &pi->nm_xact_addr_filt;
2931 }
2932 #endif
2933 if (flags & XGMAC_MTU)
2934 mtu = ifp->if_mtu;
2935
2936 if (flags & XGMAC_PROMISC)
2937 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2938
2939 if (flags & XGMAC_ALLMULTI)
2940 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2941
2942 if (flags & XGMAC_VLANEX)
2943 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2944
2945 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
2946 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
2947 1, vlanex, false);
2948 if (rc) {
2949 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
2950 rc);
2951 return (rc);
2952 }
2953 }
2954
2955 if (flags & XGMAC_UCADDR) {
2956 uint8_t ucaddr[ETHER_ADDR_LEN];
2957
2958 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2959 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
2960 true, true);
2961 if (rc < 0) {
2962 rc = -rc;
2963 if_printf(ifp, "change_mac failed: %d\n", rc);
2964 return (rc);
2965 } else {
2966 *xact_addr_filt = rc;
2967 rc = 0;
2968 }
2969 }
2970
2971 if (flags & XGMAC_MCADDRS) {
2972 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2973 int del = 1;
2974 uint64_t hash = 0;
2975 struct ifmultiaddr *ifma;
2976 int i = 0, j;
2977
2978 if_maddr_rlock(ifp);
2979 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2980 if (ifma->ifma_addr->sa_family != AF_LINK)
2981 continue;
2982 mcaddr[i] =
2983 LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2984 MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
2985 i++;
2986
2987 if (i == FW_MAC_EXACT_CHUNK) {
2988 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
2989 i, mcaddr, NULL, &hash, 0);
2990 if (rc < 0) {
2991 rc = -rc;
2992 for (j = 0; j < i; j++) {
2993 if_printf(ifp,
2994 "failed to add mc address"
2995 " %02x:%02x:%02x:"
2996 "%02x:%02x:%02x rc=%d\n",
2997 mcaddr[j][0], mcaddr[j][1],
2998 mcaddr[j][2], mcaddr[j][3],
2999 mcaddr[j][4], mcaddr[j][5],
3000 rc);
3001 }
3002 goto mcfail;
3003 }
3004 del = 0;
3005 i = 0;
3006 }
3007 }
3008 if (i > 0) {
3009 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
3010 mcaddr, NULL, &hash, 0);
3011 if (rc < 0) {
3012 rc = -rc;
3013 for (j = 0; j < i; j++) {
3014 if_printf(ifp,
3015 "failed to add mc address"
3016 " %02x:%02x:%02x:"
3017 "%02x:%02x:%02x rc=%d\n",
3018 mcaddr[j][0], mcaddr[j][1],
3019 mcaddr[j][2], mcaddr[j][3],
3020 mcaddr[j][4], mcaddr[j][5],
3021 rc);
3022 }
3023 goto mcfail;
3024 }
3025 }
3026
3027 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
3028 if (rc != 0)
3029 if_printf(ifp, "failed to set mc address hash: %d", rc);
3030 mcfail:
3031 if_maddr_runlock(ifp);
3032 }
3033
3034 return (rc);
3035 }
3036
3037 /*
3038 * {begin|end}_synchronized_op must be called from the same thread.
3039 */
3040 int
begin_synchronized_op(struct adapter * sc,struct port_info * pi,int flags,char * wmesg)3041 begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
3042 char *wmesg)
3043 {
3044 int rc, pri;
3045
3046 #ifdef WITNESS
3047 /* the caller thinks it's ok to sleep, but is it really? */
3048 if (flags & SLEEP_OK)
3049 pause("t4slptst", 1);
3050 #endif
3051
3052 if (INTR_OK)
3053 pri = PCATCH;
3054 else
3055 pri = 0;
3056
3057 ADAPTER_LOCK(sc);
3058 for (;;) {
3059
3060 if (pi && IS_DOOMED(pi)) {
3061 rc = ENXIO;
3062 goto done;
3063 }
3064
3065 if (!IS_BUSY(sc)) {
3066 rc = 0;
3067 break;
3068 }
3069
3070 if (!(flags & SLEEP_OK)) {
3071 rc = EBUSY;
3072 goto done;
3073 }
3074
3075 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
3076 rc = EINTR;
3077 goto done;
3078 }
3079 }
3080
3081 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
3082 SET_BUSY(sc);
3083 #ifdef INVARIANTS
3084 sc->last_op = wmesg;
3085 sc->last_op_thr = curthread;
3086 #endif
3087
3088 done:
3089 if (!(flags & HOLD_LOCK) || rc)
3090 ADAPTER_UNLOCK(sc);
3091
3092 return (rc);
3093 }
3094
3095 /*
3096 * {begin|end}_synchronized_op must be called from the same thread.
3097 */
3098 void
end_synchronized_op(struct adapter * sc,int flags)3099 end_synchronized_op(struct adapter *sc, int flags)
3100 {
3101
3102 if (flags & LOCK_HELD)
3103 ADAPTER_LOCK_ASSERT_OWNED(sc);
3104 else
3105 ADAPTER_LOCK(sc);
3106
3107 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
3108 CLR_BUSY(sc);
3109 wakeup(&sc->flags);
3110 ADAPTER_UNLOCK(sc);
3111 }
3112
3113 static int
cxgbe_init_synchronized(struct port_info * pi)3114 cxgbe_init_synchronized(struct port_info *pi)
3115 {
3116 struct adapter *sc = pi->adapter;
3117 struct ifnet *ifp = pi->ifp;
3118 int rc = 0, i;
3119 struct sge_txq *txq;
3120
3121 ASSERT_SYNCHRONIZED_OP(sc);
3122
3123 if (isset(&sc->open_device_map, pi->port_id)) {
3124 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
3125 ("mismatch between open_device_map and if_drv_flags"));
3126 return (0); /* already running */
3127 }
3128
3129 if (!(sc->flags & FULL_INIT_DONE) &&
3130 ((rc = adapter_full_init(sc)) != 0))
3131 return (rc); /* error message displayed already */
3132
3133 if (!(pi->flags & PORT_INIT_DONE) &&
3134 ((rc = port_full_init(pi)) != 0))
3135 return (rc); /* error message displayed already */
3136
3137 rc = update_mac_settings(ifp, XGMAC_ALL);
3138 if (rc)
3139 goto done; /* error message displayed already */
3140
3141 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
3142 if (rc != 0) {
3143 if_printf(ifp, "enable_vi failed: %d\n", rc);
3144 goto done;
3145 }
3146
3147 /*
3148 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized
3149 * if this changes.
3150 */
3151
3152 for_each_txq(pi, i, txq) {
3153 TXQ_LOCK(txq);
3154 txq->eq.flags |= EQ_ENABLED;
3155 TXQ_UNLOCK(txq);
3156 }
3157
3158 /*
3159 * The first iq of the first port to come up is used for tracing.
3160 */
3161 if (sc->traceq < 0) {
3162 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id;
3163 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL :
3164 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
3165 V_QUEUENUMBER(sc->traceq));
3166 pi->flags |= HAS_TRACEQ;
3167 }
3168
3169 /* all ok */
3170 setbit(&sc->open_device_map, pi->port_id);
3171 PORT_LOCK(pi);
3172 ifp->if_drv_flags |= IFF_DRV_RUNNING;
3173 PORT_UNLOCK(pi);
3174
3175 callout_reset(&pi->tick, hz, cxgbe_tick, pi);
3176 done:
3177 if (rc != 0)
3178 cxgbe_uninit_synchronized(pi);
3179
3180 return (rc);
3181 }
3182
3183 /*
3184 * Idempotent.
3185 */
3186 static int
cxgbe_uninit_synchronized(struct port_info * pi)3187 cxgbe_uninit_synchronized(struct port_info *pi)
3188 {
3189 struct adapter *sc = pi->adapter;
3190 struct ifnet *ifp = pi->ifp;
3191 int rc, i;
3192 struct sge_txq *txq;
3193
3194 ASSERT_SYNCHRONIZED_OP(sc);
3195
3196 /*
3197 * Disable the VI so that all its data in either direction is discarded
3198 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz
3199 * tick) intact as the TP can deliver negative advice or data that it's
3200 * holding in its RAM (for an offloaded connection) even after the VI is
3201 * disabled.
3202 */
3203 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
3204 if (rc) {
3205 if_printf(ifp, "disable_vi failed: %d\n", rc);
3206 return (rc);
3207 }
3208
3209 for_each_txq(pi, i, txq) {
3210 TXQ_LOCK(txq);
3211 txq->eq.flags &= ~EQ_ENABLED;
3212 TXQ_UNLOCK(txq);
3213 }
3214
3215 clrbit(&sc->open_device_map, pi->port_id);
3216 PORT_LOCK(pi);
3217 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3218 PORT_UNLOCK(pi);
3219
3220 pi->link_cfg.link_ok = 0;
3221 pi->link_cfg.speed = 0;
3222 pi->linkdnrc = -1;
3223 t4_os_link_changed(sc, pi->port_id, 0, -1);
3224
3225 return (0);
3226 }
3227
3228 /*
3229 * It is ok for this function to fail midway and return right away. t4_detach
3230 * will walk the entire sc->irq list and clean up whatever is valid.
3231 */
3232 static int
setup_intr_handlers(struct adapter * sc)3233 setup_intr_handlers(struct adapter *sc)
3234 {
3235 int rc, rid, p, q;
3236 char s[8];
3237 struct irq *irq;
3238 struct port_info *pi;
3239 struct sge_rxq *rxq;
3240 #ifdef TCP_OFFLOAD
3241 struct sge_ofld_rxq *ofld_rxq;
3242 #endif
3243 #ifdef DEV_NETMAP
3244 struct sge_nm_rxq *nm_rxq;
3245 #endif
3246
3247 /*
3248 * Setup interrupts.
3249 */
3250 irq = &sc->irq[0];
3251 rid = sc->intr_type == INTR_INTX ? 0 : 1;
3252 if (sc->intr_count == 1)
3253 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
3254
3255 /* Multiple interrupts. */
3256 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
3257 ("%s: too few intr.", __func__));
3258
3259 /* The first one is always error intr */
3260 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
3261 if (rc != 0)
3262 return (rc);
3263 irq++;
3264 rid++;
3265
3266 /* The second one is always the firmware event queue */
3267 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
3268 if (rc != 0)
3269 return (rc);
3270 irq++;
3271 rid++;
3272
3273 for_each_port(sc, p) {
3274 pi = sc->port[p];
3275
3276 if (pi->flags & INTR_RXQ) {
3277 for_each_rxq(pi, q, rxq) {
3278 snprintf(s, sizeof(s), "%d.%d", p, q);
3279 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
3280 s);
3281 if (rc != 0)
3282 return (rc);
3283 irq++;
3284 rid++;
3285 }
3286 }
3287 #ifdef TCP_OFFLOAD
3288 if (pi->flags & INTR_OFLD_RXQ) {
3289 for_each_ofld_rxq(pi, q, ofld_rxq) {
3290 snprintf(s, sizeof(s), "%d,%d", p, q);
3291 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
3292 ofld_rxq, s);
3293 if (rc != 0)
3294 return (rc);
3295 irq++;
3296 rid++;
3297 }
3298 }
3299 #endif
3300 #ifdef DEV_NETMAP
3301 if (pi->flags & INTR_NM_RXQ) {
3302 for_each_nm_rxq(pi, q, nm_rxq) {
3303 snprintf(s, sizeof(s), "%d-%d", p, q);
3304 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
3305 nm_rxq, s);
3306 if (rc != 0)
3307 return (rc);
3308 irq++;
3309 rid++;
3310 }
3311 }
3312 #endif
3313 }
3314 MPASS(irq == &sc->irq[sc->intr_count]);
3315
3316 return (0);
3317 }
3318
3319 int
adapter_full_init(struct adapter * sc)3320 adapter_full_init(struct adapter *sc)
3321 {
3322 int rc, i;
3323
3324 ASSERT_SYNCHRONIZED_OP(sc);
3325 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3326 KASSERT((sc->flags & FULL_INIT_DONE) == 0,
3327 ("%s: FULL_INIT_DONE already", __func__));
3328
3329 /*
3330 * queues that belong to the adapter (not any particular port).
3331 */
3332 rc = t4_setup_adapter_queues(sc);
3333 if (rc != 0)
3334 goto done;
3335
3336 for (i = 0; i < nitems(sc->tq); i++) {
3337 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
3338 taskqueue_thread_enqueue, &sc->tq[i]);
3339 if (sc->tq[i] == NULL) {
3340 device_printf(sc->dev,
3341 "failed to allocate task queue %d\n", i);
3342 rc = ENOMEM;
3343 goto done;
3344 }
3345 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
3346 device_get_nameunit(sc->dev), i);
3347 }
3348
3349 t4_intr_enable(sc);
3350 sc->flags |= FULL_INIT_DONE;
3351 done:
3352 if (rc != 0)
3353 adapter_full_uninit(sc);
3354
3355 return (rc);
3356 }
3357
3358 int
adapter_full_uninit(struct adapter * sc)3359 adapter_full_uninit(struct adapter *sc)
3360 {
3361 int i;
3362
3363 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
3364
3365 t4_teardown_adapter_queues(sc);
3366
3367 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
3368 taskqueue_free(sc->tq[i]);
3369 sc->tq[i] = NULL;
3370 }
3371
3372 sc->flags &= ~FULL_INIT_DONE;
3373
3374 return (0);
3375 }
3376
3377 int
port_full_init(struct port_info * pi)3378 port_full_init(struct port_info *pi)
3379 {
3380 struct adapter *sc = pi->adapter;
3381 struct ifnet *ifp = pi->ifp;
3382 uint16_t *rss;
3383 struct sge_rxq *rxq;
3384 int rc, i, j;
3385
3386 ASSERT_SYNCHRONIZED_OP(sc);
3387 KASSERT((pi->flags & PORT_INIT_DONE) == 0,
3388 ("%s: PORT_INIT_DONE already", __func__));
3389
3390 sysctl_ctx_init(&pi->ctx);
3391 pi->flags |= PORT_SYSCTL_CTX;
3392
3393 /*
3394 * Allocate tx/rx/fl queues for this port.
3395 */
3396 rc = t4_setup_port_queues(pi);
3397 if (rc != 0)
3398 goto done; /* error message displayed already */
3399
3400 /*
3401 * Setup RSS for this port. Save a copy of the RSS table for later use.
3402 */
3403 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
3404 for (i = 0; i < pi->rss_size;) {
3405 for_each_rxq(pi, j, rxq) {
3406 rss[i++] = rxq->iq.abs_id;
3407 if (i == pi->rss_size)
3408 break;
3409 }
3410 }
3411
3412 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
3413 pi->rss_size);
3414 if (rc != 0) {
3415 if_printf(ifp, "rss_config failed: %d\n", rc);
3416 goto done;
3417 }
3418
3419 pi->rss = rss;
3420 pi->flags |= PORT_INIT_DONE;
3421 done:
3422 if (rc != 0)
3423 port_full_uninit(pi);
3424
3425 return (rc);
3426 }
3427
3428 /*
3429 * Idempotent.
3430 */
3431 int
port_full_uninit(struct port_info * pi)3432 port_full_uninit(struct port_info *pi)
3433 {
3434 struct adapter *sc = pi->adapter;
3435 int i;
3436 struct sge_rxq *rxq;
3437 struct sge_txq *txq;
3438 #ifdef TCP_OFFLOAD
3439 struct sge_ofld_rxq *ofld_rxq;
3440 struct sge_wrq *ofld_txq;
3441 #endif
3442
3443 if (pi->flags & PORT_INIT_DONE) {
3444
3445 /* Need to quiesce queues. */
3446
3447 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
3448
3449 for_each_txq(pi, i, txq) {
3450 quiesce_txq(sc, txq);
3451 }
3452
3453 #ifdef TCP_OFFLOAD
3454 for_each_ofld_txq(pi, i, ofld_txq) {
3455 quiesce_wrq(sc, ofld_txq);
3456 }
3457 #endif
3458
3459 for_each_rxq(pi, i, rxq) {
3460 quiesce_iq(sc, &rxq->iq);
3461 quiesce_fl(sc, &rxq->fl);
3462 }
3463
3464 #ifdef TCP_OFFLOAD
3465 for_each_ofld_rxq(pi, i, ofld_rxq) {
3466 quiesce_iq(sc, &ofld_rxq->iq);
3467 quiesce_fl(sc, &ofld_rxq->fl);
3468 }
3469 #endif
3470 free(pi->rss, M_CXGBE);
3471 }
3472
3473 t4_teardown_port_queues(pi);
3474 pi->flags &= ~PORT_INIT_DONE;
3475
3476 return (0);
3477 }
3478
3479 static void
quiesce_txq(struct adapter * sc,struct sge_txq * txq)3480 quiesce_txq(struct adapter *sc, struct sge_txq *txq)
3481 {
3482 struct sge_eq *eq = &txq->eq;
3483 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
3484
3485 (void) sc; /* unused */
3486
3487 #ifdef INVARIANTS
3488 TXQ_LOCK(txq);
3489 MPASS((eq->flags & EQ_ENABLED) == 0);
3490 TXQ_UNLOCK(txq);
3491 #endif
3492
3493 /* Wait for the mp_ring to empty. */
3494 while (!mp_ring_is_idle(txq->r)) {
3495 mp_ring_check_drainage(txq->r, 0);
3496 pause("rquiesce", 1);
3497 }
3498
3499 /* Then wait for the hardware to finish. */
3500 while (spg->cidx != htobe16(eq->pidx))
3501 pause("equiesce", 1);
3502
3503 /* Finally, wait for the driver to reclaim all descriptors. */
3504 while (eq->cidx != eq->pidx)
3505 pause("dquiesce", 1);
3506 }
3507
3508 static void
quiesce_wrq(struct adapter * sc,struct sge_wrq * wrq)3509 quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
3510 {
3511
3512 /* XXXTX */
3513 }
3514
3515 static void
quiesce_iq(struct adapter * sc,struct sge_iq * iq)3516 quiesce_iq(struct adapter *sc, struct sge_iq *iq)
3517 {
3518 (void) sc; /* unused */
3519
3520 /* Synchronize with the interrupt handler */
3521 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
3522 pause("iqfree", 1);
3523 }
3524
3525 static void
quiesce_fl(struct adapter * sc,struct sge_fl * fl)3526 quiesce_fl(struct adapter *sc, struct sge_fl *fl)
3527 {
3528 mtx_lock(&sc->sfl_lock);
3529 FL_LOCK(fl);
3530 fl->flags |= FL_DOOMED;
3531 FL_UNLOCK(fl);
3532 mtx_unlock(&sc->sfl_lock);
3533
3534 callout_drain(&sc->sfl_callout);
3535 KASSERT((fl->flags & FL_STARVING) == 0,
3536 ("%s: still starving", __func__));
3537 }
3538
3539 static int
t4_alloc_irq(struct adapter * sc,struct irq * irq,int rid,driver_intr_t * handler,void * arg,char * name)3540 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
3541 driver_intr_t *handler, void *arg, char *name)
3542 {
3543 int rc;
3544
3545 irq->rid = rid;
3546 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
3547 RF_SHAREABLE | RF_ACTIVE);
3548 if (irq->res == NULL) {
3549 device_printf(sc->dev,
3550 "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
3551 return (ENOMEM);
3552 }
3553
3554 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
3555 NULL, handler, arg, &irq->tag);
3556 if (rc != 0) {
3557 device_printf(sc->dev,
3558 "failed to setup interrupt for rid %d, name %s: %d\n",
3559 rid, name, rc);
3560 } else if (name)
3561 bus_describe_intr(sc->dev, irq->res, irq->tag, name);
3562
3563 return (rc);
3564 }
3565
3566 static int
t4_free_irq(struct adapter * sc,struct irq * irq)3567 t4_free_irq(struct adapter *sc, struct irq *irq)
3568 {
3569 if (irq->tag)
3570 bus_teardown_intr(sc->dev, irq->res, irq->tag);
3571 if (irq->res)
3572 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
3573
3574 bzero(irq, sizeof(*irq));
3575
3576 return (0);
3577 }
3578
3579 static void
reg_block_dump(struct adapter * sc,uint8_t * buf,unsigned int start,unsigned int end)3580 reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
3581 unsigned int end)
3582 {
3583 uint32_t *p = (uint32_t *)(buf + start);
3584
3585 for ( ; start <= end; start += sizeof(uint32_t))
3586 *p++ = t4_read_reg(sc, start);
3587 }
3588
3589 static void
t4_get_regs(struct adapter * sc,struct t4_regdump * regs,uint8_t * buf)3590 t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
3591 {
3592 int i, n;
3593 const unsigned int *reg_ranges;
3594 static const unsigned int t4_reg_ranges[] = {
3595 0x1008, 0x1108,
3596 0x1180, 0x11b4,
3597 0x11fc, 0x123c,
3598 0x1300, 0x173c,
3599 0x1800, 0x18fc,
3600 0x3000, 0x30d8,
3601 0x30e0, 0x5924,
3602 0x5960, 0x59d4,
3603 0x5a00, 0x5af8,
3604 0x6000, 0x6098,
3605 0x6100, 0x6150,
3606 0x6200, 0x6208,
3607 0x6240, 0x6248,
3608 0x6280, 0x6338,
3609 0x6370, 0x638c,
3610 0x6400, 0x643c,
3611 0x6500, 0x6524,
3612 0x6a00, 0x6a38,
3613 0x6a60, 0x6a78,
3614 0x6b00, 0x6b84,
3615 0x6bf0, 0x6c84,
3616 0x6cf0, 0x6d84,
3617 0x6df0, 0x6e84,
3618 0x6ef0, 0x6f84,
3619 0x6ff0, 0x7084,
3620 0x70f0, 0x7184,
3621 0x71f0, 0x7284,
3622 0x72f0, 0x7384,
3623 0x73f0, 0x7450,
3624 0x7500, 0x7530,
3625 0x7600, 0x761c,
3626 0x7680, 0x76cc,
3627 0x7700, 0x7798,
3628 0x77c0, 0x77fc,
3629 0x7900, 0x79fc,
3630 0x7b00, 0x7c38,
3631 0x7d00, 0x7efc,
3632 0x8dc0, 0x8e1c,
3633 0x8e30, 0x8e78,
3634 0x8ea0, 0x8f6c,
3635 0x8fc0, 0x9074,
3636 0x90fc, 0x90fc,
3637 0x9400, 0x9458,
3638 0x9600, 0x96bc,
3639 0x9800, 0x9808,
3640 0x9820, 0x983c,
3641 0x9850, 0x9864,
3642 0x9c00, 0x9c6c,
3643 0x9c80, 0x9cec,
3644 0x9d00, 0x9d6c,
3645 0x9d80, 0x9dec,
3646 0x9e00, 0x9e6c,
3647 0x9e80, 0x9eec,
3648 0x9f00, 0x9f6c,
3649 0x9f80, 0x9fec,
3650 0xd004, 0xd03c,
3651 0xdfc0, 0xdfe0,
3652 0xe000, 0xea7c,
3653 0xf000, 0x11110,
3654 0x11118, 0x11190,
3655 0x19040, 0x1906c,
3656 0x19078, 0x19080,
3657 0x1908c, 0x19124,
3658 0x19150, 0x191b0,
3659 0x191d0, 0x191e8,
3660 0x19238, 0x1924c,
3661 0x193f8, 0x19474,
3662 0x19490, 0x194f8,
3663 0x19800, 0x19f30,
3664 0x1a000, 0x1a06c,
3665 0x1a0b0, 0x1a120,
3666 0x1a128, 0x1a138,
3667 0x1a190, 0x1a1c4,
3668 0x1a1fc, 0x1a1fc,
3669 0x1e040, 0x1e04c,
3670 0x1e284, 0x1e28c,
3671 0x1e2c0, 0x1e2c0,
3672 0x1e2e0, 0x1e2e0,
3673 0x1e300, 0x1e384,
3674 0x1e3c0, 0x1e3c8,
3675 0x1e440, 0x1e44c,
3676 0x1e684, 0x1e68c,
3677 0x1e6c0, 0x1e6c0,
3678 0x1e6e0, 0x1e6e0,
3679 0x1e700, 0x1e784,
3680 0x1e7c0, 0x1e7c8,
3681 0x1e840, 0x1e84c,
3682 0x1ea84, 0x1ea8c,
3683 0x1eac0, 0x1eac0,
3684 0x1eae0, 0x1eae0,
3685 0x1eb00, 0x1eb84,
3686 0x1ebc0, 0x1ebc8,
3687 0x1ec40, 0x1ec4c,
3688 0x1ee84, 0x1ee8c,
3689 0x1eec0, 0x1eec0,
3690 0x1eee0, 0x1eee0,
3691 0x1ef00, 0x1ef84,
3692 0x1efc0, 0x1efc8,
3693 0x1f040, 0x1f04c,
3694 0x1f284, 0x1f28c,
3695 0x1f2c0, 0x1f2c0,
3696 0x1f2e0, 0x1f2e0,
3697 0x1f300, 0x1f384,
3698 0x1f3c0, 0x1f3c8,
3699 0x1f440, 0x1f44c,
3700 0x1f684, 0x1f68c,
3701 0x1f6c0, 0x1f6c0,
3702 0x1f6e0, 0x1f6e0,
3703 0x1f700, 0x1f784,
3704 0x1f7c0, 0x1f7c8,
3705 0x1f840, 0x1f84c,
3706 0x1fa84, 0x1fa8c,
3707 0x1fac0, 0x1fac0,
3708 0x1fae0, 0x1fae0,
3709 0x1fb00, 0x1fb84,
3710 0x1fbc0, 0x1fbc8,
3711 0x1fc40, 0x1fc4c,
3712 0x1fe84, 0x1fe8c,
3713 0x1fec0, 0x1fec0,
3714 0x1fee0, 0x1fee0,
3715 0x1ff00, 0x1ff84,
3716 0x1ffc0, 0x1ffc8,
3717 0x20000, 0x2002c,
3718 0x20100, 0x2013c,
3719 0x20190, 0x201c8,
3720 0x20200, 0x20318,
3721 0x20400, 0x20528,
3722 0x20540, 0x20614,
3723 0x21000, 0x21040,
3724 0x2104c, 0x21060,
3725 0x210c0, 0x210ec,
3726 0x21200, 0x21268,
3727 0x21270, 0x21284,
3728 0x212fc, 0x21388,
3729 0x21400, 0x21404,
3730 0x21500, 0x21518,
3731 0x2152c, 0x2153c,
3732 0x21550, 0x21554,
3733 0x21600, 0x21600,
3734 0x21608, 0x21628,
3735 0x21630, 0x2163c,
3736 0x21700, 0x2171c,
3737 0x21780, 0x2178c,
3738 0x21800, 0x21c38,
3739 0x21c80, 0x21d7c,
3740 0x21e00, 0x21e04,
3741 0x22000, 0x2202c,
3742 0x22100, 0x2213c,
3743 0x22190, 0x221c8,
3744 0x22200, 0x22318,
3745 0x22400, 0x22528,
3746 0x22540, 0x22614,
3747 0x23000, 0x23040,
3748 0x2304c, 0x23060,
3749 0x230c0, 0x230ec,
3750 0x23200, 0x23268,
3751 0x23270, 0x23284,
3752 0x232fc, 0x23388,
3753 0x23400, 0x23404,
3754 0x23500, 0x23518,
3755 0x2352c, 0x2353c,
3756 0x23550, 0x23554,
3757 0x23600, 0x23600,
3758 0x23608, 0x23628,
3759 0x23630, 0x2363c,
3760 0x23700, 0x2371c,
3761 0x23780, 0x2378c,
3762 0x23800, 0x23c38,
3763 0x23c80, 0x23d7c,
3764 0x23e00, 0x23e04,
3765 0x24000, 0x2402c,
3766 0x24100, 0x2413c,
3767 0x24190, 0x241c8,
3768 0x24200, 0x24318,
3769 0x24400, 0x24528,
3770 0x24540, 0x24614,
3771 0x25000, 0x25040,
3772 0x2504c, 0x25060,
3773 0x250c0, 0x250ec,
3774 0x25200, 0x25268,
3775 0x25270, 0x25284,
3776 0x252fc, 0x25388,
3777 0x25400, 0x25404,
3778 0x25500, 0x25518,
3779 0x2552c, 0x2553c,
3780 0x25550, 0x25554,
3781 0x25600, 0x25600,
3782 0x25608, 0x25628,
3783 0x25630, 0x2563c,
3784 0x25700, 0x2571c,
3785 0x25780, 0x2578c,
3786 0x25800, 0x25c38,
3787 0x25c80, 0x25d7c,
3788 0x25e00, 0x25e04,
3789 0x26000, 0x2602c,
3790 0x26100, 0x2613c,
3791 0x26190, 0x261c8,
3792 0x26200, 0x26318,
3793 0x26400, 0x26528,
3794 0x26540, 0x26614,
3795 0x27000, 0x27040,
3796 0x2704c, 0x27060,
3797 0x270c0, 0x270ec,
3798 0x27200, 0x27268,
3799 0x27270, 0x27284,
3800 0x272fc, 0x27388,
3801 0x27400, 0x27404,
3802 0x27500, 0x27518,
3803 0x2752c, 0x2753c,
3804 0x27550, 0x27554,
3805 0x27600, 0x27600,
3806 0x27608, 0x27628,
3807 0x27630, 0x2763c,
3808 0x27700, 0x2771c,
3809 0x27780, 0x2778c,
3810 0x27800, 0x27c38,
3811 0x27c80, 0x27d7c,
3812 0x27e00, 0x27e04
3813 };
3814 static const unsigned int t5_reg_ranges[] = {
3815 0x1008, 0x1148,
3816 0x1180, 0x11b4,
3817 0x11fc, 0x123c,
3818 0x1280, 0x173c,
3819 0x1800, 0x18fc,
3820 0x3000, 0x3028,
3821 0x3060, 0x30d8,
3822 0x30e0, 0x30fc,
3823 0x3140, 0x357c,
3824 0x35a8, 0x35cc,
3825 0x35ec, 0x35ec,
3826 0x3600, 0x5624,
3827 0x56cc, 0x575c,
3828 0x580c, 0x5814,
3829 0x5890, 0x58bc,
3830 0x5940, 0x59dc,
3831 0x59fc, 0x5a18,
3832 0x5a60, 0x5a9c,
3833 0x5b94, 0x5bfc,
3834 0x6000, 0x6040,
3835 0x6058, 0x614c,
3836 0x7700, 0x7798,
3837 0x77c0, 0x78fc,
3838 0x7b00, 0x7c54,
3839 0x7d00, 0x7efc,
3840 0x8dc0, 0x8de0,
3841 0x8df8, 0x8e84,
3842 0x8ea0, 0x8f84,
3843 0x8fc0, 0x90f8,
3844 0x9400, 0x9470,
3845 0x9600, 0x96f4,
3846 0x9800, 0x9808,
3847 0x9820, 0x983c,
3848 0x9850, 0x9864,
3849 0x9c00, 0x9c6c,
3850 0x9c80, 0x9cec,
3851 0x9d00, 0x9d6c,
3852 0x9d80, 0x9dec,
3853 0x9e00, 0x9e6c,
3854 0x9e80, 0x9eec,
3855 0x9f00, 0x9f6c,
3856 0x9f80, 0xa020,
3857 0xd004, 0xd03c,
3858 0xdfc0, 0xdfe0,
3859 0xe000, 0x11088,
3860 0x1109c, 0x11110,
3861 0x11118, 0x1117c,
3862 0x11190, 0x11204,
3863 0x19040, 0x1906c,
3864 0x19078, 0x19080,
3865 0x1908c, 0x19124,
3866 0x19150, 0x191b0,
3867 0x191d0, 0x191e8,
3868 0x19238, 0x19290,
3869 0x193f8, 0x19474,
3870 0x19490, 0x194cc,
3871 0x194f0, 0x194f8,
3872 0x19c00, 0x19c60,
3873 0x19c94, 0x19e10,
3874 0x19e50, 0x19f34,
3875 0x19f40, 0x19f50,
3876 0x19f90, 0x19fe4,
3877 0x1a000, 0x1a06c,
3878 0x1a0b0, 0x1a120,
3879 0x1a128, 0x1a138,
3880 0x1a190, 0x1a1c4,
3881 0x1a1fc, 0x1a1fc,
3882 0x1e008, 0x1e00c,
3883 0x1e040, 0x1e04c,
3884 0x1e284, 0x1e290,
3885 0x1e2c0, 0x1e2c0,
3886 0x1e2e0, 0x1e2e0,
3887 0x1e300, 0x1e384,
3888 0x1e3c0, 0x1e3c8,
3889 0x1e408, 0x1e40c,
3890 0x1e440, 0x1e44c,
3891 0x1e684, 0x1e690,
3892 0x1e6c0, 0x1e6c0,
3893 0x1e6e0, 0x1e6e0,
3894 0x1e700, 0x1e784,
3895 0x1e7c0, 0x1e7c8,
3896 0x1e808, 0x1e80c,
3897 0x1e840, 0x1e84c,
3898 0x1ea84, 0x1ea90,
3899 0x1eac0, 0x1eac0,
3900 0x1eae0, 0x1eae0,
3901 0x1eb00, 0x1eb84,
3902 0x1ebc0, 0x1ebc8,
3903 0x1ec08, 0x1ec0c,
3904 0x1ec40, 0x1ec4c,
3905 0x1ee84, 0x1ee90,
3906 0x1eec0, 0x1eec0,
3907 0x1eee0, 0x1eee0,
3908 0x1ef00, 0x1ef84,
3909 0x1efc0, 0x1efc8,
3910 0x1f008, 0x1f00c,
3911 0x1f040, 0x1f04c,
3912 0x1f284, 0x1f290,
3913 0x1f2c0, 0x1f2c0,
3914 0x1f2e0, 0x1f2e0,
3915 0x1f300, 0x1f384,
3916 0x1f3c0, 0x1f3c8,
3917 0x1f408, 0x1f40c,
3918 0x1f440, 0x1f44c,
3919 0x1f684, 0x1f690,
3920 0x1f6c0, 0x1f6c0,
3921 0x1f6e0, 0x1f6e0,
3922 0x1f700, 0x1f784,
3923 0x1f7c0, 0x1f7c8,
3924 0x1f808, 0x1f80c,
3925 0x1f840, 0x1f84c,
3926 0x1fa84, 0x1fa90,
3927 0x1fac0, 0x1fac0,
3928 0x1fae0, 0x1fae0,
3929 0x1fb00, 0x1fb84,
3930 0x1fbc0, 0x1fbc8,
3931 0x1fc08, 0x1fc0c,
3932 0x1fc40, 0x1fc4c,
3933 0x1fe84, 0x1fe90,
3934 0x1fec0, 0x1fec0,
3935 0x1fee0, 0x1fee0,
3936 0x1ff00, 0x1ff84,
3937 0x1ffc0, 0x1ffc8,
3938 0x30000, 0x30030,
3939 0x30100, 0x30144,
3940 0x30190, 0x301d0,
3941 0x30200, 0x30318,
3942 0x30400, 0x3052c,
3943 0x30540, 0x3061c,
3944 0x30800, 0x30834,
3945 0x308c0, 0x30908,
3946 0x30910, 0x309ac,
3947 0x30a00, 0x30a2c,
3948 0x30a44, 0x30a50,
3949 0x30a74, 0x30c24,
3950 0x30d00, 0x30d00,
3951 0x30d08, 0x30d14,
3952 0x30d1c, 0x30d20,
3953 0x30d3c, 0x30d50,
3954 0x31200, 0x3120c,
3955 0x31220, 0x31220,
3956 0x31240, 0x31240,
3957 0x31600, 0x3160c,
3958 0x31a00, 0x31a1c,
3959 0x31e00, 0x31e20,
3960 0x31e38, 0x31e3c,
3961 0x31e80, 0x31e80,
3962 0x31e88, 0x31ea8,
3963 0x31eb0, 0x31eb4,
3964 0x31ec8, 0x31ed4,
3965 0x31fb8, 0x32004,
3966 0x32200, 0x32200,
3967 0x32208, 0x32240,
3968 0x32248, 0x32280,
3969 0x32288, 0x322c0,
3970 0x322c8, 0x322fc,
3971 0x32600, 0x32630,
3972 0x32a00, 0x32abc,
3973 0x32b00, 0x32b70,
3974 0x33000, 0x33048,
3975 0x33060, 0x3309c,
3976 0x330f0, 0x33148,
3977 0x33160, 0x3319c,
3978 0x331f0, 0x332e4,
3979 0x332f8, 0x333e4,
3980 0x333f8, 0x33448,
3981 0x33460, 0x3349c,
3982 0x334f0, 0x33548,
3983 0x33560, 0x3359c,
3984 0x335f0, 0x336e4,
3985 0x336f8, 0x337e4,
3986 0x337f8, 0x337fc,
3987 0x33814, 0x33814,
3988 0x3382c, 0x3382c,
3989 0x33880, 0x3388c,
3990 0x338e8, 0x338ec,
3991 0x33900, 0x33948,
3992 0x33960, 0x3399c,
3993 0x339f0, 0x33ae4,
3994 0x33af8, 0x33b10,
3995 0x33b28, 0x33b28,
3996 0x33b3c, 0x33b50,
3997 0x33bf0, 0x33c10,
3998 0x33c28, 0x33c28,
3999 0x33c3c, 0x33c50,
4000 0x33cf0, 0x33cfc,
4001 0x34000, 0x34030,
4002 0x34100, 0x34144,
4003 0x34190, 0x341d0,
4004 0x34200, 0x34318,
4005 0x34400, 0x3452c,
4006 0x34540, 0x3461c,
4007 0x34800, 0x34834,
4008 0x348c0, 0x34908,
4009 0x34910, 0x349ac,
4010 0x34a00, 0x34a2c,
4011 0x34a44, 0x34a50,
4012 0x34a74, 0x34c24,
4013 0x34d00, 0x34d00,
4014 0x34d08, 0x34d14,
4015 0x34d1c, 0x34d20,
4016 0x34d3c, 0x34d50,
4017 0x35200, 0x3520c,
4018 0x35220, 0x35220,
4019 0x35240, 0x35240,
4020 0x35600, 0x3560c,
4021 0x35a00, 0x35a1c,
4022 0x35e00, 0x35e20,
4023 0x35e38, 0x35e3c,
4024 0x35e80, 0x35e80,
4025 0x35e88, 0x35ea8,
4026 0x35eb0, 0x35eb4,
4027 0x35ec8, 0x35ed4,
4028 0x35fb8, 0x36004,
4029 0x36200, 0x36200,
4030 0x36208, 0x36240,
4031 0x36248, 0x36280,
4032 0x36288, 0x362c0,
4033 0x362c8, 0x362fc,
4034 0x36600, 0x36630,
4035 0x36a00, 0x36abc,
4036 0x36b00, 0x36b70,
4037 0x37000, 0x37048,
4038 0x37060, 0x3709c,
4039 0x370f0, 0x37148,
4040 0x37160, 0x3719c,
4041 0x371f0, 0x372e4,
4042 0x372f8, 0x373e4,
4043 0x373f8, 0x37448,
4044 0x37460, 0x3749c,
4045 0x374f0, 0x37548,
4046 0x37560, 0x3759c,
4047 0x375f0, 0x376e4,
4048 0x376f8, 0x377e4,
4049 0x377f8, 0x377fc,
4050 0x37814, 0x37814,
4051 0x3782c, 0x3782c,
4052 0x37880, 0x3788c,
4053 0x378e8, 0x378ec,
4054 0x37900, 0x37948,
4055 0x37960, 0x3799c,
4056 0x379f0, 0x37ae4,
4057 0x37af8, 0x37b10,
4058 0x37b28, 0x37b28,
4059 0x37b3c, 0x37b50,
4060 0x37bf0, 0x37c10,
4061 0x37c28, 0x37c28,
4062 0x37c3c, 0x37c50,
4063 0x37cf0, 0x37cfc,
4064 0x38000, 0x38030,
4065 0x38100, 0x38144,
4066 0x38190, 0x381d0,
4067 0x38200, 0x38318,
4068 0x38400, 0x3852c,
4069 0x38540, 0x3861c,
4070 0x38800, 0x38834,
4071 0x388c0, 0x38908,
4072 0x38910, 0x389ac,
4073 0x38a00, 0x38a2c,
4074 0x38a44, 0x38a50,
4075 0x38a74, 0x38c24,
4076 0x38d00, 0x38d00,
4077 0x38d08, 0x38d14,
4078 0x38d1c, 0x38d20,
4079 0x38d3c, 0x38d50,
4080 0x39200, 0x3920c,
4081 0x39220, 0x39220,
4082 0x39240, 0x39240,
4083 0x39600, 0x3960c,
4084 0x39a00, 0x39a1c,
4085 0x39e00, 0x39e20,
4086 0x39e38, 0x39e3c,
4087 0x39e80, 0x39e80,
4088 0x39e88, 0x39ea8,
4089 0x39eb0, 0x39eb4,
4090 0x39ec8, 0x39ed4,
4091 0x39fb8, 0x3a004,
4092 0x3a200, 0x3a200,
4093 0x3a208, 0x3a240,
4094 0x3a248, 0x3a280,
4095 0x3a288, 0x3a2c0,
4096 0x3a2c8, 0x3a2fc,
4097 0x3a600, 0x3a630,
4098 0x3aa00, 0x3aabc,
4099 0x3ab00, 0x3ab70,
4100 0x3b000, 0x3b048,
4101 0x3b060, 0x3b09c,
4102 0x3b0f0, 0x3b148,
4103 0x3b160, 0x3b19c,
4104 0x3b1f0, 0x3b2e4,
4105 0x3b2f8, 0x3b3e4,
4106 0x3b3f8, 0x3b448,
4107 0x3b460, 0x3b49c,
4108 0x3b4f0, 0x3b548,
4109 0x3b560, 0x3b59c,
4110 0x3b5f0, 0x3b6e4,
4111 0x3b6f8, 0x3b7e4,
4112 0x3b7f8, 0x3b7fc,
4113 0x3b814, 0x3b814,
4114 0x3b82c, 0x3b82c,
4115 0x3b880, 0x3b88c,
4116 0x3b8e8, 0x3b8ec,
4117 0x3b900, 0x3b948,
4118 0x3b960, 0x3b99c,
4119 0x3b9f0, 0x3bae4,
4120 0x3baf8, 0x3bb10,
4121 0x3bb28, 0x3bb28,
4122 0x3bb3c, 0x3bb50,
4123 0x3bbf0, 0x3bc10,
4124 0x3bc28, 0x3bc28,
4125 0x3bc3c, 0x3bc50,
4126 0x3bcf0, 0x3bcfc,
4127 0x3c000, 0x3c030,
4128 0x3c100, 0x3c144,
4129 0x3c190, 0x3c1d0,
4130 0x3c200, 0x3c318,
4131 0x3c400, 0x3c52c,
4132 0x3c540, 0x3c61c,
4133 0x3c800, 0x3c834,
4134 0x3c8c0, 0x3c908,
4135 0x3c910, 0x3c9ac,
4136 0x3ca00, 0x3ca2c,
4137 0x3ca44, 0x3ca50,
4138 0x3ca74, 0x3cc24,
4139 0x3cd00, 0x3cd00,
4140 0x3cd08, 0x3cd14,
4141 0x3cd1c, 0x3cd20,
4142 0x3cd3c, 0x3cd50,
4143 0x3d200, 0x3d20c,
4144 0x3d220, 0x3d220,
4145 0x3d240, 0x3d240,
4146 0x3d600, 0x3d60c,
4147 0x3da00, 0x3da1c,
4148 0x3de00, 0x3de20,
4149 0x3de38, 0x3de3c,
4150 0x3de80, 0x3de80,
4151 0x3de88, 0x3dea8,
4152 0x3deb0, 0x3deb4,
4153 0x3dec8, 0x3ded4,
4154 0x3dfb8, 0x3e004,
4155 0x3e200, 0x3e200,
4156 0x3e208, 0x3e240,
4157 0x3e248, 0x3e280,
4158 0x3e288, 0x3e2c0,
4159 0x3e2c8, 0x3e2fc,
4160 0x3e600, 0x3e630,
4161 0x3ea00, 0x3eabc,
4162 0x3eb00, 0x3eb70,
4163 0x3f000, 0x3f048,
4164 0x3f060, 0x3f09c,
4165 0x3f0f0, 0x3f148,
4166 0x3f160, 0x3f19c,
4167 0x3f1f0, 0x3f2e4,
4168 0x3f2f8, 0x3f3e4,
4169 0x3f3f8, 0x3f448,
4170 0x3f460, 0x3f49c,
4171 0x3f4f0, 0x3f548,
4172 0x3f560, 0x3f59c,
4173 0x3f5f0, 0x3f6e4,
4174 0x3f6f8, 0x3f7e4,
4175 0x3f7f8, 0x3f7fc,
4176 0x3f814, 0x3f814,
4177 0x3f82c, 0x3f82c,
4178 0x3f880, 0x3f88c,
4179 0x3f8e8, 0x3f8ec,
4180 0x3f900, 0x3f948,
4181 0x3f960, 0x3f99c,
4182 0x3f9f0, 0x3fae4,
4183 0x3faf8, 0x3fb10,
4184 0x3fb28, 0x3fb28,
4185 0x3fb3c, 0x3fb50,
4186 0x3fbf0, 0x3fc10,
4187 0x3fc28, 0x3fc28,
4188 0x3fc3c, 0x3fc50,
4189 0x3fcf0, 0x3fcfc,
4190 0x40000, 0x4000c,
4191 0x40040, 0x40068,
4192 0x4007c, 0x40144,
4193 0x40180, 0x4018c,
4194 0x40200, 0x40298,
4195 0x402ac, 0x4033c,
4196 0x403f8, 0x403fc,
4197 0x41304, 0x413c4,
4198 0x41400, 0x4141c,
4199 0x41480, 0x414d0,
4200 0x44000, 0x44078,
4201 0x440c0, 0x44278,
4202 0x442c0, 0x44478,
4203 0x444c0, 0x44678,
4204 0x446c0, 0x44878,
4205 0x448c0, 0x449fc,
4206 0x45000, 0x45068,
4207 0x45080, 0x45084,
4208 0x450a0, 0x450b0,
4209 0x45200, 0x45268,
4210 0x45280, 0x45284,
4211 0x452a0, 0x452b0,
4212 0x460c0, 0x460e4,
4213 0x47000, 0x4708c,
4214 0x47200, 0x47250,
4215 0x47400, 0x47420,
4216 0x47600, 0x47618,
4217 0x47800, 0x47814,
4218 0x48000, 0x4800c,
4219 0x48040, 0x48068,
4220 0x4807c, 0x48144,
4221 0x48180, 0x4818c,
4222 0x48200, 0x48298,
4223 0x482ac, 0x4833c,
4224 0x483f8, 0x483fc,
4225 0x49304, 0x493c4,
4226 0x49400, 0x4941c,
4227 0x49480, 0x494d0,
4228 0x4c000, 0x4c078,
4229 0x4c0c0, 0x4c278,
4230 0x4c2c0, 0x4c478,
4231 0x4c4c0, 0x4c678,
4232 0x4c6c0, 0x4c878,
4233 0x4c8c0, 0x4c9fc,
4234 0x4d000, 0x4d068,
4235 0x4d080, 0x4d084,
4236 0x4d0a0, 0x4d0b0,
4237 0x4d200, 0x4d268,
4238 0x4d280, 0x4d284,
4239 0x4d2a0, 0x4d2b0,
4240 0x4e0c0, 0x4e0e4,
4241 0x4f000, 0x4f08c,
4242 0x4f200, 0x4f250,
4243 0x4f400, 0x4f420,
4244 0x4f600, 0x4f618,
4245 0x4f800, 0x4f814,
4246 0x50000, 0x500cc,
4247 0x50400, 0x50400,
4248 0x50800, 0x508cc,
4249 0x50c00, 0x50c00,
4250 0x51000, 0x5101c,
4251 0x51300, 0x51308,
4252 };
4253
4254 if (is_t4(sc)) {
4255 reg_ranges = &t4_reg_ranges[0];
4256 n = nitems(t4_reg_ranges);
4257 } else {
4258 reg_ranges = &t5_reg_ranges[0];
4259 n = nitems(t5_reg_ranges);
4260 }
4261
4262 regs->version = chip_id(sc) | chip_rev(sc) << 10;
4263 for (i = 0; i < n; i += 2)
4264 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
4265 }
4266
4267 static void
cxgbe_refresh_stats(struct adapter * sc,struct port_info * pi)4268 cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
4269 {
4270 struct ifnet *ifp = pi->ifp;
4271 struct sge_txq *txq;
4272 int i, drops;
4273 struct port_stats *s = &pi->stats;
4274 struct timeval tv;
4275 const struct timeval interval = {0, 250000}; /* 250ms */
4276
4277 getmicrotime(&tv);
4278 timevalsub(&tv, &interval);
4279 if (timevalcmp(&tv, &pi->last_refreshed, <))
4280 return;
4281
4282 t4_get_port_stats(sc, pi->tx_chan, s);
4283
4284 ifp->if_opackets = s->tx_frames - s->tx_pause;
4285 ifp->if_ipackets = s->rx_frames - s->rx_pause;
4286 ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
4287 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
4288 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
4289 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
4290 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
4291 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
4292 s->rx_trunc3;
4293 for (i = 0; i < NCHAN; i++) {
4294 if (pi->rx_chan_map & (1 << i)) {
4295 uint32_t v;
4296
4297 mtx_lock(&sc->regwin_lock);
4298 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
4299 1, A_TP_MIB_TNL_CNG_DROP_0 + i);
4300 mtx_unlock(&sc->regwin_lock);
4301 ifp->if_iqdrops += v;
4302 }
4303 }
4304
4305 drops = s->tx_drop;
4306 for_each_txq(pi, i, txq)
4307 drops += counter_u64_fetch(txq->r->drops);
4308 ifp->if_snd.ifq_drops = drops;
4309
4310 ifp->if_oerrors = s->tx_error_frames;
4311 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
4312 s->rx_fcs_err + s->rx_len_err;
4313
4314 getmicrotime(&pi->last_refreshed);
4315 }
4316
4317 static void
cxgbe_tick(void * arg)4318 cxgbe_tick(void *arg)
4319 {
4320 struct port_info *pi = arg;
4321 struct adapter *sc = pi->adapter;
4322 struct ifnet *ifp = pi->ifp;
4323
4324 PORT_LOCK(pi);
4325 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4326 PORT_UNLOCK(pi);
4327 return; /* without scheduling another callout */
4328 }
4329
4330 cxgbe_refresh_stats(sc, pi);
4331
4332 callout_schedule(&pi->tick, hz);
4333 PORT_UNLOCK(pi);
4334 }
4335
4336 static void
cxgbe_vlan_config(void * arg,struct ifnet * ifp,uint16_t vid)4337 cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
4338 {
4339 struct ifnet *vlan;
4340
4341 if (arg != ifp || ifp->if_type != IFT_ETHER)
4342 return;
4343
4344 vlan = VLAN_DEVAT(ifp, vid);
4345 VLAN_SETCOOKIE(vlan, ifp);
4346 }
4347
4348 static int
cpl_not_handled(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)4349 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4350 {
4351
4352 #ifdef INVARIANTS
4353 panic("%s: opcode 0x%02x on iq %p with payload %p",
4354 __func__, rss->opcode, iq, m);
4355 #else
4356 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
4357 __func__, rss->opcode, iq, m);
4358 m_freem(m);
4359 #endif
4360 return (EDOOFUS);
4361 }
4362
4363 int
t4_register_cpl_handler(struct adapter * sc,int opcode,cpl_handler_t h)4364 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
4365 {
4366 uintptr_t *loc, new;
4367
4368 if (opcode >= nitems(sc->cpl_handler))
4369 return (EINVAL);
4370
4371 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
4372 loc = (uintptr_t *) &sc->cpl_handler[opcode];
4373 atomic_store_rel_ptr(loc, new);
4374
4375 return (0);
4376 }
4377
4378 static int
an_not_handled(struct sge_iq * iq,const struct rsp_ctrl * ctrl)4379 an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
4380 {
4381
4382 #ifdef INVARIANTS
4383 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
4384 #else
4385 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
4386 __func__, iq, ctrl);
4387 #endif
4388 return (EDOOFUS);
4389 }
4390
4391 int
t4_register_an_handler(struct adapter * sc,an_handler_t h)4392 t4_register_an_handler(struct adapter *sc, an_handler_t h)
4393 {
4394 uintptr_t *loc, new;
4395
4396 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
4397 loc = (uintptr_t *) &sc->an_handler;
4398 atomic_store_rel_ptr(loc, new);
4399
4400 return (0);
4401 }
4402
4403 static int
fw_msg_not_handled(struct adapter * sc,const __be64 * rpl)4404 fw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
4405 {
4406 const struct cpl_fw6_msg *cpl =
4407 __containerof(rpl, struct cpl_fw6_msg, data[0]);
4408
4409 #ifdef INVARIANTS
4410 panic("%s: fw_msg type %d", __func__, cpl->type);
4411 #else
4412 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
4413 #endif
4414 return (EDOOFUS);
4415 }
4416
4417 int
t4_register_fw_msg_handler(struct adapter * sc,int type,fw_msg_handler_t h)4418 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
4419 {
4420 uintptr_t *loc, new;
4421
4422 if (type >= nitems(sc->fw_msg_handler))
4423 return (EINVAL);
4424
4425 /*
4426 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
4427 * handler dispatch table. Reject any attempt to install a handler for
4428 * this subtype.
4429 */
4430 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
4431 return (EINVAL);
4432
4433 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
4434 loc = (uintptr_t *) &sc->fw_msg_handler[type];
4435 atomic_store_rel_ptr(loc, new);
4436
4437 return (0);
4438 }
4439
4440 static int
t4_sysctls(struct adapter * sc)4441 t4_sysctls(struct adapter *sc)
4442 {
4443 struct sysctl_ctx_list *ctx;
4444 struct sysctl_oid *oid;
4445 struct sysctl_oid_list *children, *c0;
4446 static char *caps[] = {
4447 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */
4448 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */
4449 "\6HASHFILTER\7ETHOFLD",
4450 "\20\1TOE", /* caps[2] toecaps */
4451 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */
4452 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */
4453 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
4454 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
4455 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */
4456 "\4PO_INITIAOR\5PO_TARGET"
4457 };
4458 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
4459
4460 ctx = device_get_sysctl_ctx(sc->dev);
4461
4462 /*
4463 * dev.t4nex.X.
4464 */
4465 oid = device_get_sysctl_tree(sc->dev);
4466 c0 = children = SYSCTL_CHILDREN(oid);
4467
4468 sc->sc_do_rxcopy = 1;
4469 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
4470 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
4471
4472 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
4473 sc->params.nports, "# of ports");
4474
4475 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
4476 NULL, chip_rev(sc), "chip hardware revision");
4477
4478 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
4479 CTLFLAG_RD, sc->fw_version, 0, "firmware version");
4480
4481 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
4482 CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
4483
4484 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
4485 sc->cfcsum, "config file checksum");
4486
4487 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
4488 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells,
4489 sysctl_bitfield, "A", "available doorbells");
4490
4491 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
4492 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
4493 sysctl_bitfield, "A", "available link capabilities");
4494
4495 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
4496 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
4497 sysctl_bitfield, "A", "available NIC capabilities");
4498
4499 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
4500 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
4501 sysctl_bitfield, "A", "available TCP offload capabilities");
4502
4503 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
4504 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
4505 sysctl_bitfield, "A", "available RDMA capabilities");
4506
4507 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
4508 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
4509 sysctl_bitfield, "A", "available iSCSI capabilities");
4510
4511 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
4512 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
4513 sysctl_bitfield, "A", "available FCoE capabilities");
4514
4515 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
4516 sc->params.vpd.cclk, "core clock frequency (in KHz)");
4517
4518 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
4519 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
4520 sizeof(sc->sge.timer_val), sysctl_int_array, "A",
4521 "interrupt holdoff timer values (us)");
4522
4523 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
4524 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
4525 sizeof(sc->sge.counter_val), sysctl_int_array, "A",
4526 "interrupt holdoff packet counter values");
4527
4528 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
4529 NULL, sc->tids.nftids, "number of filters");
4530
4531 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
4532 CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
4533 "chip temperature (in Celsius)");
4534
4535 t4_sge_sysctls(sc, ctx, children);
4536
4537 sc->lro_timeout = 100;
4538 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
4539 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
4540
4541 #ifdef SBUF_DRAIN
4542 /*
4543 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload.
4544 */
4545 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
4546 CTLFLAG_RD | CTLFLAG_SKIP, NULL,
4547 "logs and miscellaneous information");
4548 children = SYSCTL_CHILDREN(oid);
4549
4550 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
4551 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4552 sysctl_cctrl, "A", "congestion control");
4553
4554 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
4555 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4556 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
4557
4558 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
4559 CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
4560 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
4561
4562 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
4563 CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
4564 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
4565
4566 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
4567 CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
4568 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
4569
4570 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
4571 CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
4572 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
4573
4574 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
4575 CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
4576 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
4577
4578 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
4579 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4580 sysctl_cim_la, "A", "CIM logic analyzer");
4581
4582 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
4583 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4584 sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
4585
4586 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
4587 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
4588 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
4589
4590 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
4591 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
4592 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
4593
4594 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
4595 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
4596 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
4597
4598 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
4599 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
4600 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
4601
4602 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
4603 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
4604 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
4605
4606 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
4607 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
4608 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
4609
4610 if (is_t5(sc)) {
4611 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
4612 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
4613 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
4614
4615 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
4616 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
4617 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
4618 }
4619
4620 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
4621 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4622 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
4623
4624 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
4625 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4626 sysctl_cim_qcfg, "A", "CIM queue configuration");
4627
4628 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
4629 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4630 sysctl_cpl_stats, "A", "CPL statistics");
4631
4632 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
4633 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4634 sysctl_ddp_stats, "A", "non-TCP DDP statistics");
4635
4636 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
4637 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4638 sysctl_devlog, "A", "firmware's device log");
4639
4640 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
4641 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4642 sysctl_fcoe_stats, "A", "FCoE statistics");
4643
4644 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
4645 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4646 sysctl_hw_sched, "A", "hardware scheduler ");
4647
4648 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
4649 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4650 sysctl_l2t, "A", "hardware L2 table");
4651
4652 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
4653 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4654 sysctl_lb_stats, "A", "loopback statistics");
4655
4656 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
4657 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4658 sysctl_meminfo, "A", "memory regions");
4659
4660 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
4661 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4662 sysctl_mps_tcam, "A", "MPS TCAM entries");
4663
4664 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
4665 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4666 sysctl_path_mtus, "A", "path MTUs");
4667
4668 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
4669 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4670 sysctl_pm_stats, "A", "PM statistics");
4671
4672 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
4673 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4674 sysctl_rdma_stats, "A", "RDMA statistics");
4675
4676 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
4677 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4678 sysctl_tcp_stats, "A", "TCP statistics");
4679
4680 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
4681 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4682 sysctl_tids, "A", "TID information");
4683
4684 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
4685 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4686 sysctl_tp_err_stats, "A", "TP error statistics");
4687
4688 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
4689 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4690 sysctl_tp_la, "A", "TP logic analyzer");
4691
4692 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
4693 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4694 sysctl_tx_rate, "A", "Tx rate");
4695
4696 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
4697 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4698 sysctl_ulprx_la, "A", "ULPRX logic analyzer");
4699
4700 if (is_t5(sc)) {
4701 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
4702 CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
4703 sysctl_wcwr_stats, "A", "write combined work requests");
4704 }
4705 #endif
4706
4707 #ifdef TCP_OFFLOAD
4708 if (is_offload(sc)) {
4709 /*
4710 * dev.t4nex.X.toe.
4711 */
4712 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
4713 NULL, "TOE parameters");
4714 children = SYSCTL_CHILDREN(oid);
4715
4716 sc->tt.sndbuf = 256 * 1024;
4717 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
4718 &sc->tt.sndbuf, 0, "max hardware send buffer size");
4719
4720 sc->tt.ddp = 0;
4721 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
4722 &sc->tt.ddp, 0, "DDP allowed");
4723
4724 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
4725 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
4726 &sc->tt.indsz, 0, "DDP max indicate size allowed");
4727
4728 sc->tt.ddp_thres =
4729 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
4730 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
4731 &sc->tt.ddp_thres, 0, "DDP threshold");
4732
4733 sc->tt.rx_coalesce = 1;
4734 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
4735 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
4736
4737 sc->tt.tx_align = 1;
4738 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
4739 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
4740 }
4741 #endif
4742
4743
4744 return (0);
4745 }
4746
4747 static int
cxgbe_sysctls(struct port_info * pi)4748 cxgbe_sysctls(struct port_info *pi)
4749 {
4750 struct sysctl_ctx_list *ctx;
4751 struct sysctl_oid *oid;
4752 struct sysctl_oid_list *children;
4753 struct adapter *sc = pi->adapter;
4754
4755 ctx = device_get_sysctl_ctx(pi->dev);
4756
4757 /*
4758 * dev.cxgbe.X.
4759 */
4760 oid = device_get_sysctl_tree(pi->dev);
4761 children = SYSCTL_CHILDREN(oid);
4762
4763 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
4764 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
4765 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
4766 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
4767 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
4768 "PHY temperature (in Celsius)");
4769 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
4770 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
4771 "PHY firmware version");
4772 }
4773 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
4774 &pi->nrxq, 0, "# of rx queues");
4775 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
4776 &pi->ntxq, 0, "# of tx queues");
4777 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
4778 &pi->first_rxq, 0, "index of first rx queue");
4779 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
4780 &pi->first_txq, 0, "index of first tx queue");
4781 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT |
4782 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU",
4783 "Reserve queue 0 for non-flowid packets");
4784
4785 #ifdef TCP_OFFLOAD
4786 if (is_offload(sc)) {
4787 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
4788 &pi->nofldrxq, 0,
4789 "# of rx queues for offloaded TCP connections");
4790 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
4791 &pi->nofldtxq, 0,
4792 "# of tx queues for offloaded TCP connections");
4793 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
4794 CTLFLAG_RD, &pi->first_ofld_rxq, 0,
4795 "index of first TOE rx queue");
4796 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
4797 CTLFLAG_RD, &pi->first_ofld_txq, 0,
4798 "index of first TOE tx queue");
4799 }
4800 #endif
4801 #ifdef DEV_NETMAP
4802 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
4803 &pi->nnmrxq, 0, "# of rx queues for netmap");
4804 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
4805 &pi->nnmtxq, 0, "# of tx queues for netmap");
4806 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
4807 CTLFLAG_RD, &pi->first_nm_rxq, 0,
4808 "index of first netmap rx queue");
4809 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
4810 CTLFLAG_RD, &pi->first_nm_txq, 0,
4811 "index of first netmap tx queue");
4812 #endif
4813
4814 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
4815 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
4816 "holdoff timer index");
4817 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
4818 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
4819 "holdoff packet counter index");
4820
4821 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
4822 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
4823 "rx queue size");
4824 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
4825 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
4826 "tx queue size");
4827
4828 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
4829 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings,
4830 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)");
4831
4832 /*
4833 * dev.cxgbe.X.stats.
4834 */
4835 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
4836 NULL, "port statistics");
4837 children = SYSCTL_CHILDREN(oid);
4838 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
4839 &pi->tx_parse_error, 0,
4840 "# of tx packets with invalid length or # of segments");
4841
4842 #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
4843 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
4844 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
4845 sysctl_handle_t4_reg64, "QU", desc)
4846
4847 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
4848 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
4849 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
4850 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
4851 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
4852 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
4853 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
4854 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
4855 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
4856 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
4857 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
4858 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
4859 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
4860 "# of tx frames in this range",
4861 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
4862 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
4863 "# of tx frames in this range",
4864 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
4865 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
4866 "# of tx frames in this range",
4867 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
4868 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
4869 "# of tx frames in this range",
4870 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
4871 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
4872 "# of tx frames in this range",
4873 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
4874 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
4875 "# of tx frames in this range",
4876 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
4877 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
4878 "# of tx frames in this range",
4879 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
4880 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
4881 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
4882 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
4883 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
4884 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
4885 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
4886 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
4887 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
4888 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
4889 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
4890 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
4891 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
4892 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
4893 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
4894 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
4895 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
4896 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
4897 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
4898 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
4899 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
4900
4901 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
4902 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
4903 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
4904 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
4905 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
4906 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
4907 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
4908 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
4909 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
4910 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
4911 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
4912 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
4913 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
4914 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
4915 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
4916 "# of frames received with bad FCS",
4917 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
4918 SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
4919 "# of frames received with length error",
4920 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
4921 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
4922 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
4923 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
4924 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
4925 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
4926 "# of rx frames in this range",
4927 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
4928 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
4929 "# of rx frames in this range",
4930 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
4931 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
4932 "# of rx frames in this range",
4933 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
4934 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
4935 "# of rx frames in this range",
4936 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
4937 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
4938 "# of rx frames in this range",
4939 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
4940 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
4941 "# of rx frames in this range",
4942 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
4943 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
4944 "# of rx frames in this range",
4945 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
4946 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
4947 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
4948 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
4949 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
4950 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
4951 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
4952 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
4953 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
4954 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
4955 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
4956 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
4957 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
4958 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
4959 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
4960 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
4961 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
4962 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
4963 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
4964
4965 #undef SYSCTL_ADD_T4_REG64
4966
4967 #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
4968 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
4969 &pi->stats.name, desc)
4970
4971 /* We get these from port_stats and they may be stale by upto 1s */
4972 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
4973 "# drops due to buffer-group 0 overflows");
4974 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
4975 "# drops due to buffer-group 1 overflows");
4976 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
4977 "# drops due to buffer-group 2 overflows");
4978 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
4979 "# drops due to buffer-group 3 overflows");
4980 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
4981 "# of buffer-group 0 truncated packets");
4982 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
4983 "# of buffer-group 1 truncated packets");
4984 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
4985 "# of buffer-group 2 truncated packets");
4986 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
4987 "# of buffer-group 3 truncated packets");
4988
4989 #undef SYSCTL_ADD_T4_PORTSTAT
4990
4991 return (0);
4992 }
4993
4994 static int
sysctl_int_array(SYSCTL_HANDLER_ARGS)4995 sysctl_int_array(SYSCTL_HANDLER_ARGS)
4996 {
4997 int rc, *i, space = 0;
4998 struct sbuf sb;
4999
5000 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
5001 for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
5002 if (space)
5003 sbuf_printf(&sb, " ");
5004 sbuf_printf(&sb, "%d", *i);
5005 space = 1;
5006 }
5007 sbuf_finish(&sb);
5008 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
5009 sbuf_delete(&sb);
5010 return (rc);
5011 }
5012
5013 static int
sysctl_bitfield(SYSCTL_HANDLER_ARGS)5014 sysctl_bitfield(SYSCTL_HANDLER_ARGS)
5015 {
5016 int rc;
5017 struct sbuf *sb;
5018
5019 rc = sysctl_wire_old_buffer(req, 0);
5020 if (rc != 0)
5021 return(rc);
5022
5023 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5024 if (sb == NULL)
5025 return (ENOMEM);
5026
5027 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
5028 rc = sbuf_finish(sb);
5029 sbuf_delete(sb);
5030
5031 return (rc);
5032 }
5033
5034 static int
sysctl_btphy(SYSCTL_HANDLER_ARGS)5035 sysctl_btphy(SYSCTL_HANDLER_ARGS)
5036 {
5037 struct port_info *pi = arg1;
5038 int op = arg2;
5039 struct adapter *sc = pi->adapter;
5040 u_int v;
5041 int rc;
5042
5043 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt");
5044 if (rc)
5045 return (rc);
5046 /* XXX: magic numbers */
5047 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
5048 &v);
5049 end_synchronized_op(sc, 0);
5050 if (rc)
5051 return (rc);
5052 if (op == 0)
5053 v /= 256;
5054
5055 rc = sysctl_handle_int(oidp, &v, 0, req);
5056 return (rc);
5057 }
5058
5059 static int
sysctl_noflowq(SYSCTL_HANDLER_ARGS)5060 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
5061 {
5062 struct port_info *pi = arg1;
5063 int rc, val;
5064
5065 val = pi->rsrv_noflowq;
5066 rc = sysctl_handle_int(oidp, &val, 0, req);
5067 if (rc != 0 || req->newptr == NULL)
5068 return (rc);
5069
5070 if ((val >= 1) && (pi->ntxq > 1))
5071 pi->rsrv_noflowq = 1;
5072 else
5073 pi->rsrv_noflowq = 0;
5074
5075 return (rc);
5076 }
5077
5078 static int
sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)5079 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
5080 {
5081 struct port_info *pi = arg1;
5082 struct adapter *sc = pi->adapter;
5083 int idx, rc, i;
5084 struct sge_rxq *rxq;
5085 #ifdef TCP_OFFLOAD
5086 struct sge_ofld_rxq *ofld_rxq;
5087 #endif
5088 uint8_t v;
5089
5090 idx = pi->tmr_idx;
5091
5092 rc = sysctl_handle_int(oidp, &idx, 0, req);
5093 if (rc != 0 || req->newptr == NULL)
5094 return (rc);
5095
5096 if (idx < 0 || idx >= SGE_NTIMERS)
5097 return (EINVAL);
5098
5099 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5100 "t4tmr");
5101 if (rc)
5102 return (rc);
5103
5104 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
5105 for_each_rxq(pi, i, rxq) {
5106 #ifdef atomic_store_rel_8
5107 atomic_store_rel_8(&rxq->iq.intr_params, v);
5108 #else
5109 rxq->iq.intr_params = v;
5110 #endif
5111 }
5112 #ifdef TCP_OFFLOAD
5113 for_each_ofld_rxq(pi, i, ofld_rxq) {
5114 #ifdef atomic_store_rel_8
5115 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
5116 #else
5117 ofld_rxq->iq.intr_params = v;
5118 #endif
5119 }
5120 #endif
5121 pi->tmr_idx = idx;
5122
5123 end_synchronized_op(sc, LOCK_HELD);
5124 return (0);
5125 }
5126
5127 static int
sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)5128 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
5129 {
5130 struct port_info *pi = arg1;
5131 struct adapter *sc = pi->adapter;
5132 int idx, rc;
5133
5134 idx = pi->pktc_idx;
5135
5136 rc = sysctl_handle_int(oidp, &idx, 0, req);
5137 if (rc != 0 || req->newptr == NULL)
5138 return (rc);
5139
5140 if (idx < -1 || idx >= SGE_NCOUNTERS)
5141 return (EINVAL);
5142
5143 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5144 "t4pktc");
5145 if (rc)
5146 return (rc);
5147
5148 if (pi->flags & PORT_INIT_DONE)
5149 rc = EBUSY; /* cannot be changed once the queues are created */
5150 else
5151 pi->pktc_idx = idx;
5152
5153 end_synchronized_op(sc, LOCK_HELD);
5154 return (rc);
5155 }
5156
5157 static int
sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)5158 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
5159 {
5160 struct port_info *pi = arg1;
5161 struct adapter *sc = pi->adapter;
5162 int qsize, rc;
5163
5164 qsize = pi->qsize_rxq;
5165
5166 rc = sysctl_handle_int(oidp, &qsize, 0, req);
5167 if (rc != 0 || req->newptr == NULL)
5168 return (rc);
5169
5170 if (qsize < 128 || (qsize & 7))
5171 return (EINVAL);
5172
5173 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5174 "t4rxqs");
5175 if (rc)
5176 return (rc);
5177
5178 if (pi->flags & PORT_INIT_DONE)
5179 rc = EBUSY; /* cannot be changed once the queues are created */
5180 else
5181 pi->qsize_rxq = qsize;
5182
5183 end_synchronized_op(sc, LOCK_HELD);
5184 return (rc);
5185 }
5186
5187 static int
sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)5188 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
5189 {
5190 struct port_info *pi = arg1;
5191 struct adapter *sc = pi->adapter;
5192 int qsize, rc;
5193
5194 qsize = pi->qsize_txq;
5195
5196 rc = sysctl_handle_int(oidp, &qsize, 0, req);
5197 if (rc != 0 || req->newptr == NULL)
5198 return (rc);
5199
5200 if (qsize < 128 || qsize > 65536)
5201 return (EINVAL);
5202
5203 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
5204 "t4txqs");
5205 if (rc)
5206 return (rc);
5207
5208 if (pi->flags & PORT_INIT_DONE)
5209 rc = EBUSY; /* cannot be changed once the queues are created */
5210 else
5211 pi->qsize_txq = qsize;
5212
5213 end_synchronized_op(sc, LOCK_HELD);
5214 return (rc);
5215 }
5216
5217 static int
sysctl_pause_settings(SYSCTL_HANDLER_ARGS)5218 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
5219 {
5220 struct port_info *pi = arg1;
5221 struct adapter *sc = pi->adapter;
5222 struct link_config *lc = &pi->link_cfg;
5223 int rc;
5224
5225 if (req->newptr == NULL) {
5226 struct sbuf *sb;
5227 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX";
5228
5229 rc = sysctl_wire_old_buffer(req, 0);
5230 if (rc != 0)
5231 return(rc);
5232
5233 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
5234 if (sb == NULL)
5235 return (ENOMEM);
5236
5237 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits);
5238 rc = sbuf_finish(sb);
5239 sbuf_delete(sb);
5240 } else {
5241 char s[2];
5242 int n;
5243
5244 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX));
5245 s[1] = 0;
5246
5247 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
5248 if (rc != 0)
5249 return(rc);
5250
5251 if (s[1] != 0)
5252 return (EINVAL);
5253 if (s[0] < '0' || s[0] > '9')
5254 return (EINVAL); /* not a number */
5255 n = s[0] - '0';
5256 if (n & ~(PAUSE_TX | PAUSE_RX))
5257 return (EINVAL); /* some other bit is set too */
5258
5259 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE");
5260 if (rc)
5261 return (rc);
5262 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) {
5263 int link_ok = lc->link_ok;
5264
5265 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX);
5266 lc->requested_fc |= n;
5267 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc);
5268 lc->link_ok = link_ok; /* restore */
5269 }
5270 end_synchronized_op(sc, 0);
5271 }
5272
5273 return (rc);
5274 }
5275
5276 static int
sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)5277 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
5278 {
5279 struct adapter *sc = arg1;
5280 int reg = arg2;
5281 uint64_t val;
5282
5283 val = t4_read_reg64(sc, reg);
5284
5285 return (sysctl_handle_64(oidp, &val, 0, req));
5286 }
5287
5288 static int
sysctl_temperature(SYSCTL_HANDLER_ARGS)5289 sysctl_temperature(SYSCTL_HANDLER_ARGS)
5290 {
5291 struct adapter *sc = arg1;
5292 int rc, t;
5293 uint32_t param, val;
5294
5295 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
5296 if (rc)
5297 return (rc);
5298 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5299 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5300 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
5301 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
5302 end_synchronized_op(sc, 0);
5303 if (rc)
5304 return (rc);
5305
5306 /* unknown is returned as 0 but we display -1 in that case */
5307 t = val == 0 ? -1 : val;
5308
5309 rc = sysctl_handle_int(oidp, &t, 0, req);
5310 return (rc);
5311 }
5312
5313 #ifdef SBUF_DRAIN
5314 static int
sysctl_cctrl(SYSCTL_HANDLER_ARGS)5315 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
5316 {
5317 struct adapter *sc = arg1;
5318 struct sbuf *sb;
5319 int rc, i;
5320 uint16_t incr[NMTUS][NCCTRL_WIN];
5321 static const char *dec_fac[] = {
5322 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
5323 "0.9375"
5324 };
5325
5326 rc = sysctl_wire_old_buffer(req, 0);
5327 if (rc != 0)
5328 return (rc);
5329
5330 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5331 if (sb == NULL)
5332 return (ENOMEM);
5333
5334 t4_read_cong_tbl(sc, incr);
5335
5336 for (i = 0; i < NCCTRL_WIN; ++i) {
5337 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
5338 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
5339 incr[5][i], incr[6][i], incr[7][i]);
5340 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
5341 incr[8][i], incr[9][i], incr[10][i], incr[11][i],
5342 incr[12][i], incr[13][i], incr[14][i], incr[15][i],
5343 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
5344 }
5345
5346 rc = sbuf_finish(sb);
5347 sbuf_delete(sb);
5348
5349 return (rc);
5350 }
5351
5352 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
5353 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */
5354 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
5355 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */
5356 };
5357
5358 static int
sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)5359 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
5360 {
5361 struct adapter *sc = arg1;
5362 struct sbuf *sb;
5363 int rc, i, n, qid = arg2;
5364 uint32_t *buf, *p;
5365 char *qtype;
5366 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5;
5367
5368 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
5369 ("%s: bad qid %d\n", __func__, qid));
5370
5371 if (qid < CIM_NUM_IBQ) {
5372 /* inbound queue */
5373 qtype = "IBQ";
5374 n = 4 * CIM_IBQ_SIZE;
5375 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5376 rc = t4_read_cim_ibq(sc, qid, buf, n);
5377 } else {
5378 /* outbound queue */
5379 qtype = "OBQ";
5380 qid -= CIM_NUM_IBQ;
5381 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
5382 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
5383 rc = t4_read_cim_obq(sc, qid, buf, n);
5384 }
5385
5386 if (rc < 0) {
5387 rc = -rc;
5388 goto done;
5389 }
5390 n = rc * sizeof(uint32_t); /* rc has # of words actually read */
5391
5392 rc = sysctl_wire_old_buffer(req, 0);
5393 if (rc != 0)
5394 goto done;
5395
5396 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5397 if (sb == NULL) {
5398 rc = ENOMEM;
5399 goto done;
5400 }
5401
5402 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
5403 for (i = 0, p = buf; i < n; i += 16, p += 4)
5404 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
5405 p[2], p[3]);
5406
5407 rc = sbuf_finish(sb);
5408 sbuf_delete(sb);
5409 done:
5410 free(buf, M_CXGBE);
5411 return (rc);
5412 }
5413
5414 static int
sysctl_cim_la(SYSCTL_HANDLER_ARGS)5415 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
5416 {
5417 struct adapter *sc = arg1;
5418 u_int cfg;
5419 struct sbuf *sb;
5420 uint32_t *buf, *p;
5421 int rc;
5422
5423 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
5424 if (rc != 0)
5425 return (rc);
5426
5427 rc = sysctl_wire_old_buffer(req, 0);
5428 if (rc != 0)
5429 return (rc);
5430
5431 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5432 if (sb == NULL)
5433 return (ENOMEM);
5434
5435 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
5436 M_ZERO | M_WAITOK);
5437
5438 rc = -t4_cim_read_la(sc, buf, NULL);
5439 if (rc != 0)
5440 goto done;
5441
5442 sbuf_printf(sb, "Status Data PC%s",
5443 cfg & F_UPDBGLACAPTPCONLY ? "" :
5444 " LS0Stat LS0Addr LS0Data");
5445
5446 KASSERT((sc->params.cim_la_size & 7) == 0,
5447 ("%s: p will walk off the end of buf", __func__));
5448
5449 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) {
5450 if (cfg & F_UPDBGLACAPTPCONLY) {
5451 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff,
5452 p[6], p[7]);
5453 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x",
5454 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
5455 p[4] & 0xff, p[5] >> 8);
5456 sbuf_printf(sb, "\n %02x %x%07x %x%07x",
5457 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5458 p[1] & 0xf, p[2] >> 4);
5459 } else {
5460 sbuf_printf(sb,
5461 "\n %02x %x%07x %x%07x %08x %08x "
5462 "%08x%08x%08x%08x",
5463 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
5464 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
5465 p[6], p[7]);
5466 }
5467 }
5468
5469 rc = sbuf_finish(sb);
5470 sbuf_delete(sb);
5471 done:
5472 free(buf, M_CXGBE);
5473 return (rc);
5474 }
5475
5476 static int
sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)5477 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
5478 {
5479 struct adapter *sc = arg1;
5480 u_int i;
5481 struct sbuf *sb;
5482 uint32_t *buf, *p;
5483 int rc;
5484
5485 rc = sysctl_wire_old_buffer(req, 0);
5486 if (rc != 0)
5487 return (rc);
5488
5489 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5490 if (sb == NULL)
5491 return (ENOMEM);
5492
5493 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
5494 M_ZERO | M_WAITOK);
5495
5496 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
5497 p = buf;
5498
5499 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5500 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
5501 p[1], p[0]);
5502 }
5503
5504 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD");
5505 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
5506 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u",
5507 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
5508 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
5509 (p[1] >> 2) | ((p[2] & 3) << 30),
5510 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
5511 p[0] & 1);
5512 }
5513
5514 rc = sbuf_finish(sb);
5515 sbuf_delete(sb);
5516 free(buf, M_CXGBE);
5517 return (rc);
5518 }
5519
5520 static int
sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)5521 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
5522 {
5523 struct adapter *sc = arg1;
5524 u_int i;
5525 struct sbuf *sb;
5526 uint32_t *buf, *p;
5527 int rc;
5528
5529 rc = sysctl_wire_old_buffer(req, 0);
5530 if (rc != 0)
5531 return (rc);
5532
5533 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5534 if (sb == NULL)
5535 return (ENOMEM);
5536
5537 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
5538 M_ZERO | M_WAITOK);
5539
5540 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
5541 p = buf;
5542
5543 sbuf_printf(sb, "Cntl ID DataBE Addr Data");
5544 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5545 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x",
5546 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
5547 p[4], p[3], p[2], p[1], p[0]);
5548 }
5549
5550 sbuf_printf(sb, "\n\nCntl ID Data");
5551 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) {
5552 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x",
5553 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
5554 }
5555
5556 rc = sbuf_finish(sb);
5557 sbuf_delete(sb);
5558 free(buf, M_CXGBE);
5559 return (rc);
5560 }
5561
5562 static int
sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)5563 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
5564 {
5565 struct adapter *sc = arg1;
5566 struct sbuf *sb;
5567 int rc, i;
5568 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5569 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
5570 uint16_t thres[CIM_NUM_IBQ];
5571 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
5572 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
5573 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
5574
5575 if (is_t4(sc)) {
5576 cim_num_obq = CIM_NUM_OBQ;
5577 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
5578 obq_rdaddr = A_UP_OBQ_0_REALADDR;
5579 } else {
5580 cim_num_obq = CIM_NUM_OBQ_T5;
5581 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
5582 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
5583 }
5584 nq = CIM_NUM_IBQ + cim_num_obq;
5585
5586 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
5587 if (rc == 0)
5588 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr);
5589 if (rc != 0)
5590 return (rc);
5591
5592 t4_read_cimq_cfg(sc, base, size, thres);
5593
5594 rc = sysctl_wire_old_buffer(req, 0);
5595 if (rc != 0)
5596 return (rc);
5597
5598 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
5599 if (sb == NULL)
5600 return (ENOMEM);
5601
5602 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail");
5603
5604 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
5605 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u",
5606 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
5607 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5608 G_QUEREMFLITS(p[2]) * 16);
5609 for ( ; i < nq; i++, p += 4, wr += 2)
5610 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i],
5611 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
5612 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
5613 G_QUEREMFLITS(p[2]) * 16);
5614
5615 rc = sbuf_finish(sb);
5616 sbuf_delete(sb);
5617
5618 return (rc);
5619 }
5620
5621 static int
sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)5622 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
5623 {
5624 struct adapter *sc = arg1;
5625 struct sbuf *sb;
5626 int rc;
5627 struct tp_cpl_stats stats;
5628
5629 rc = sysctl_wire_old_buffer(req, 0);
5630 if (rc != 0)
5631 return (rc);
5632
5633 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5634 if (sb == NULL)
5635 return (ENOMEM);
5636
5637 t4_tp_get_cpl_stats(sc, &stats);
5638
5639 sbuf_printf(sb, " channel 0 channel 1 channel 2 "
5640 "channel 3\n");
5641 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n",
5642 stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
5643 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u",
5644 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
5645
5646 rc = sbuf_finish(sb);
5647 sbuf_delete(sb);
5648
5649 return (rc);
5650 }
5651
5652 static int
sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)5653 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
5654 {
5655 struct adapter *sc = arg1;
5656 struct sbuf *sb;
5657 int rc;
5658 struct tp_usm_stats stats;
5659
5660 rc = sysctl_wire_old_buffer(req, 0);
5661 if (rc != 0)
5662 return(rc);
5663
5664 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5665 if (sb == NULL)
5666 return (ENOMEM);
5667
5668 t4_get_usm_stats(sc, &stats);
5669
5670 sbuf_printf(sb, "Frames: %u\n", stats.frames);
5671 sbuf_printf(sb, "Octets: %ju\n", stats.octets);
5672 sbuf_printf(sb, "Drops: %u", stats.drops);
5673
5674 rc = sbuf_finish(sb);
5675 sbuf_delete(sb);
5676
5677 return (rc);
5678 }
5679
5680 const char *devlog_level_strings[] = {
5681 [FW_DEVLOG_LEVEL_EMERG] = "EMERG",
5682 [FW_DEVLOG_LEVEL_CRIT] = "CRIT",
5683 [FW_DEVLOG_LEVEL_ERR] = "ERR",
5684 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE",
5685 [FW_DEVLOG_LEVEL_INFO] = "INFO",
5686 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG"
5687 };
5688
5689 const char *devlog_facility_strings[] = {
5690 [FW_DEVLOG_FACILITY_CORE] = "CORE",
5691 [FW_DEVLOG_FACILITY_CF] = "CF",
5692 [FW_DEVLOG_FACILITY_SCHED] = "SCHED",
5693 [FW_DEVLOG_FACILITY_TIMER] = "TIMER",
5694 [FW_DEVLOG_FACILITY_RES] = "RES",
5695 [FW_DEVLOG_FACILITY_HW] = "HW",
5696 [FW_DEVLOG_FACILITY_FLR] = "FLR",
5697 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ",
5698 [FW_DEVLOG_FACILITY_PHY] = "PHY",
5699 [FW_DEVLOG_FACILITY_MAC] = "MAC",
5700 [FW_DEVLOG_FACILITY_PORT] = "PORT",
5701 [FW_DEVLOG_FACILITY_VI] = "VI",
5702 [FW_DEVLOG_FACILITY_FILTER] = "FILTER",
5703 [FW_DEVLOG_FACILITY_ACL] = "ACL",
5704 [FW_DEVLOG_FACILITY_TM] = "TM",
5705 [FW_DEVLOG_FACILITY_QFC] = "QFC",
5706 [FW_DEVLOG_FACILITY_DCB] = "DCB",
5707 [FW_DEVLOG_FACILITY_ETH] = "ETH",
5708 [FW_DEVLOG_FACILITY_OFLD] = "OFLD",
5709 [FW_DEVLOG_FACILITY_RI] = "RI",
5710 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI",
5711 [FW_DEVLOG_FACILITY_FCOE] = "FCOE",
5712 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI",
5713 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE"
5714 };
5715
5716 static int
sysctl_devlog(SYSCTL_HANDLER_ARGS)5717 sysctl_devlog(SYSCTL_HANDLER_ARGS)
5718 {
5719 struct adapter *sc = arg1;
5720 struct devlog_params *dparams = &sc->params.devlog;
5721 struct fw_devlog_e *buf, *e;
5722 int i, j, rc, nentries, first = 0, m;
5723 struct sbuf *sb;
5724 uint64_t ftstamp = UINT64_MAX;
5725
5726 if (dparams->start == 0) {
5727 dparams->memtype = FW_MEMTYPE_EDC0;
5728 dparams->start = 0x84000;
5729 dparams->size = 32768;
5730 }
5731
5732 nentries = dparams->size / sizeof(struct fw_devlog_e);
5733
5734 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
5735 if (buf == NULL)
5736 return (ENOMEM);
5737
5738 m = fwmtype_to_hwmtype(dparams->memtype);
5739 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf);
5740 if (rc != 0)
5741 goto done;
5742
5743 for (i = 0; i < nentries; i++) {
5744 e = &buf[i];
5745
5746 if (e->timestamp == 0)
5747 break; /* end */
5748
5749 e->timestamp = be64toh(e->timestamp);
5750 e->seqno = be32toh(e->seqno);
5751 for (j = 0; j < 8; j++)
5752 e->params[j] = be32toh(e->params[j]);
5753
5754 if (e->timestamp < ftstamp) {
5755 ftstamp = e->timestamp;
5756 first = i;
5757 }
5758 }
5759
5760 if (buf[first].timestamp == 0)
5761 goto done; /* nothing in the log */
5762
5763 rc = sysctl_wire_old_buffer(req, 0);
5764 if (rc != 0)
5765 goto done;
5766
5767 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5768 if (sb == NULL) {
5769 rc = ENOMEM;
5770 goto done;
5771 }
5772 sbuf_printf(sb, "%10s %15s %8s %8s %s\n",
5773 "Seq#", "Tstamp", "Level", "Facility", "Message");
5774
5775 i = first;
5776 do {
5777 e = &buf[i];
5778 if (e->timestamp == 0)
5779 break; /* end */
5780
5781 sbuf_printf(sb, "%10d %15ju %8s %8s ",
5782 e->seqno, e->timestamp,
5783 (e->level < nitems(devlog_level_strings) ?
5784 devlog_level_strings[e->level] : "UNKNOWN"),
5785 (e->facility < nitems(devlog_facility_strings) ?
5786 devlog_facility_strings[e->facility] : "UNKNOWN"));
5787 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
5788 e->params[2], e->params[3], e->params[4],
5789 e->params[5], e->params[6], e->params[7]);
5790
5791 if (++i == nentries)
5792 i = 0;
5793 } while (i != first);
5794
5795 rc = sbuf_finish(sb);
5796 sbuf_delete(sb);
5797 done:
5798 free(buf, M_CXGBE);
5799 return (rc);
5800 }
5801
5802 static int
sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)5803 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
5804 {
5805 struct adapter *sc = arg1;
5806 struct sbuf *sb;
5807 int rc;
5808 struct tp_fcoe_stats stats[4];
5809
5810 rc = sysctl_wire_old_buffer(req, 0);
5811 if (rc != 0)
5812 return (rc);
5813
5814 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5815 if (sb == NULL)
5816 return (ENOMEM);
5817
5818 t4_get_fcoe_stats(sc, 0, &stats[0]);
5819 t4_get_fcoe_stats(sc, 1, &stats[1]);
5820 t4_get_fcoe_stats(sc, 2, &stats[2]);
5821 t4_get_fcoe_stats(sc, 3, &stats[3]);
5822
5823 sbuf_printf(sb, " channel 0 channel 1 "
5824 "channel 2 channel 3\n");
5825 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n",
5826 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
5827 stats[3].octetsDDP);
5828 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP,
5829 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
5830 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
5831 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
5832 stats[3].framesDrop);
5833
5834 rc = sbuf_finish(sb);
5835 sbuf_delete(sb);
5836
5837 return (rc);
5838 }
5839
5840 static int
sysctl_hw_sched(SYSCTL_HANDLER_ARGS)5841 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
5842 {
5843 struct adapter *sc = arg1;
5844 struct sbuf *sb;
5845 int rc, i;
5846 unsigned int map, kbps, ipg, mode;
5847 unsigned int pace_tab[NTX_SCHED];
5848
5849 rc = sysctl_wire_old_buffer(req, 0);
5850 if (rc != 0)
5851 return (rc);
5852
5853 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
5854 if (sb == NULL)
5855 return (ENOMEM);
5856
5857 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
5858 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
5859 t4_read_pace_tbl(sc, pace_tab);
5860
5861 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) "
5862 "Class IPG (0.1 ns) Flow IPG (us)");
5863
5864 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
5865 t4_get_tx_sched(sc, i, &kbps, &ipg);
5866 sbuf_printf(sb, "\n %u %-5s %u ", i,
5867 (mode & (1 << i)) ? "flow" : "class", map & 3);
5868 if (kbps)
5869 sbuf_printf(sb, "%9u ", kbps);
5870 else
5871 sbuf_printf(sb, " disabled ");
5872
5873 if (ipg)
5874 sbuf_printf(sb, "%13u ", ipg);
5875 else
5876 sbuf_printf(sb, " disabled ");
5877
5878 if (pace_tab[i])
5879 sbuf_printf(sb, "%10u", pace_tab[i]);
5880 else
5881 sbuf_printf(sb, " disabled");
5882 }
5883
5884 rc = sbuf_finish(sb);
5885 sbuf_delete(sb);
5886
5887 return (rc);
5888 }
5889
5890 static int
sysctl_lb_stats(SYSCTL_HANDLER_ARGS)5891 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
5892 {
5893 struct adapter *sc = arg1;
5894 struct sbuf *sb;
5895 int rc, i, j;
5896 uint64_t *p0, *p1;
5897 struct lb_port_stats s[2];
5898 static const char *stat_name[] = {
5899 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
5900 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
5901 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
5902 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
5903 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
5904 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
5905 "BG2FramesTrunc:", "BG3FramesTrunc:"
5906 };
5907
5908 rc = sysctl_wire_old_buffer(req, 0);
5909 if (rc != 0)
5910 return (rc);
5911
5912 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
5913 if (sb == NULL)
5914 return (ENOMEM);
5915
5916 memset(s, 0, sizeof(s));
5917
5918 for (i = 0; i < 4; i += 2) {
5919 t4_get_lb_stats(sc, i, &s[0]);
5920 t4_get_lb_stats(sc, i + 1, &s[1]);
5921
5922 p0 = &s[0].octets;
5923 p1 = &s[1].octets;
5924 sbuf_printf(sb, "%s Loopback %u"
5925 " Loopback %u", i == 0 ? "" : "\n", i, i + 1);
5926
5927 for (j = 0; j < nitems(stat_name); j++)
5928 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
5929 *p0++, *p1++);
5930 }
5931
5932 rc = sbuf_finish(sb);
5933 sbuf_delete(sb);
5934
5935 return (rc);
5936 }
5937
5938 static int
sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)5939 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
5940 {
5941 int rc = 0;
5942 struct port_info *pi = arg1;
5943 struct sbuf *sb;
5944 static const char *linkdnreasons[] = {
5945 "non-specific", "remote fault", "autoneg failed", "reserved3",
5946 "PHY overheated", "unknown", "rx los", "reserved7"
5947 };
5948
5949 rc = sysctl_wire_old_buffer(req, 0);
5950 if (rc != 0)
5951 return(rc);
5952 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
5953 if (sb == NULL)
5954 return (ENOMEM);
5955
5956 if (pi->linkdnrc < 0)
5957 sbuf_printf(sb, "n/a");
5958 else if (pi->linkdnrc < nitems(linkdnreasons))
5959 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]);
5960 else
5961 sbuf_printf(sb, "%d", pi->linkdnrc);
5962
5963 rc = sbuf_finish(sb);
5964 sbuf_delete(sb);
5965
5966 return (rc);
5967 }
5968
5969 struct mem_desc {
5970 unsigned int base;
5971 unsigned int limit;
5972 unsigned int idx;
5973 };
5974
5975 static int
mem_desc_cmp(const void * a,const void * b)5976 mem_desc_cmp(const void *a, const void *b)
5977 {
5978 return ((const struct mem_desc *)a)->base -
5979 ((const struct mem_desc *)b)->base;
5980 }
5981
5982 static void
mem_region_show(struct sbuf * sb,const char * name,unsigned int from,unsigned int to)5983 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
5984 unsigned int to)
5985 {
5986 unsigned int size;
5987
5988 size = to - from + 1;
5989 if (size == 0)
5990 return;
5991
5992 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
5993 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
5994 }
5995
5996 static int
sysctl_meminfo(SYSCTL_HANDLER_ARGS)5997 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
5998 {
5999 struct adapter *sc = arg1;
6000 struct sbuf *sb;
6001 int rc, i, n;
6002 uint32_t lo, hi, used, alloc;
6003 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"};
6004 static const char *region[] = {
6005 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
6006 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
6007 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
6008 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
6009 "RQUDP region:", "PBL region:", "TXPBL region:",
6010 "DBVFIFO region:", "ULPRX state:", "ULPTX state:",
6011 "On-chip queues:"
6012 };
6013 struct mem_desc avail[4];
6014 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */
6015 struct mem_desc *md = mem;
6016
6017 rc = sysctl_wire_old_buffer(req, 0);
6018 if (rc != 0)
6019 return (rc);
6020
6021 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6022 if (sb == NULL)
6023 return (ENOMEM);
6024
6025 for (i = 0; i < nitems(mem); i++) {
6026 mem[i].limit = 0;
6027 mem[i].idx = i;
6028 }
6029
6030 /* Find and sort the populated memory ranges */
6031 i = 0;
6032 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
6033 if (lo & F_EDRAM0_ENABLE) {
6034 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
6035 avail[i].base = G_EDRAM0_BASE(hi) << 20;
6036 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
6037 avail[i].idx = 0;
6038 i++;
6039 }
6040 if (lo & F_EDRAM1_ENABLE) {
6041 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
6042 avail[i].base = G_EDRAM1_BASE(hi) << 20;
6043 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
6044 avail[i].idx = 1;
6045 i++;
6046 }
6047 if (lo & F_EXT_MEM_ENABLE) {
6048 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
6049 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
6050 avail[i].limit = avail[i].base +
6051 (G_EXT_MEM_SIZE(hi) << 20);
6052 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */
6053 i++;
6054 }
6055 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) {
6056 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
6057 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
6058 avail[i].limit = avail[i].base +
6059 (G_EXT_MEM1_SIZE(hi) << 20);
6060 avail[i].idx = 4;
6061 i++;
6062 }
6063 if (!i) /* no memory available */
6064 return 0;
6065 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
6066
6067 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
6068 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
6069 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
6070 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
6071 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
6072 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
6073 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
6074 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
6075 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
6076
6077 /* the next few have explicit upper bounds */
6078 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
6079 md->limit = md->base - 1 +
6080 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
6081 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
6082 md++;
6083
6084 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
6085 md->limit = md->base - 1 +
6086 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
6087 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
6088 md++;
6089
6090 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6091 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
6092 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
6093 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
6094 } else {
6095 md->base = 0;
6096 md->idx = nitems(region); /* hide it */
6097 }
6098 md++;
6099
6100 #define ulp_region(reg) \
6101 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
6102 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
6103
6104 ulp_region(RX_ISCSI);
6105 ulp_region(RX_TDDP);
6106 ulp_region(TX_TPT);
6107 ulp_region(RX_STAG);
6108 ulp_region(RX_RQ);
6109 ulp_region(RX_RQUDP);
6110 ulp_region(RX_PBL);
6111 ulp_region(TX_PBL);
6112 #undef ulp_region
6113
6114 md->base = 0;
6115 md->idx = nitems(region);
6116 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) {
6117 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR));
6118 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc,
6119 A_SGE_DBVFIFO_SIZE))) << 2) - 1;
6120 }
6121 md++;
6122
6123 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
6124 md->limit = md->base + sc->tids.ntids - 1;
6125 md++;
6126 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
6127 md->limit = md->base + sc->tids.ntids - 1;
6128 md++;
6129
6130 md->base = sc->vres.ocq.start;
6131 if (sc->vres.ocq.size)
6132 md->limit = md->base + sc->vres.ocq.size - 1;
6133 else
6134 md->idx = nitems(region); /* hide it */
6135 md++;
6136
6137 /* add any address-space holes, there can be up to 3 */
6138 for (n = 0; n < i - 1; n++)
6139 if (avail[n].limit < avail[n + 1].base)
6140 (md++)->base = avail[n].limit;
6141 if (avail[n].limit)
6142 (md++)->base = avail[n].limit;
6143
6144 n = md - mem;
6145 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
6146
6147 for (lo = 0; lo < i; lo++)
6148 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
6149 avail[lo].limit - 1);
6150
6151 sbuf_printf(sb, "\n");
6152 for (i = 0; i < n; i++) {
6153 if (mem[i].idx >= nitems(region))
6154 continue; /* skip holes */
6155 if (!mem[i].limit)
6156 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
6157 mem_region_show(sb, region[mem[i].idx], mem[i].base,
6158 mem[i].limit);
6159 }
6160
6161 sbuf_printf(sb, "\n");
6162 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
6163 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
6164 mem_region_show(sb, "uP RAM:", lo, hi);
6165
6166 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
6167 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
6168 mem_region_show(sb, "uP Extmem2:", lo, hi);
6169
6170 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
6171 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
6172 G_PMRXMAXPAGE(lo),
6173 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
6174 (lo & F_PMRXNUMCHN) ? 2 : 1);
6175
6176 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
6177 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
6178 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
6179 G_PMTXMAXPAGE(lo),
6180 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
6181 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
6182 sbuf_printf(sb, "%u p-structs\n",
6183 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
6184
6185 for (i = 0; i < 4; i++) {
6186 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
6187 if (is_t4(sc)) {
6188 used = G_USED(lo);
6189 alloc = G_ALLOC(lo);
6190 } else {
6191 used = G_T5_USED(lo);
6192 alloc = G_T5_ALLOC(lo);
6193 }
6194 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
6195 i, used, alloc);
6196 }
6197 for (i = 0; i < 4; i++) {
6198 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
6199 if (is_t4(sc)) {
6200 used = G_USED(lo);
6201 alloc = G_ALLOC(lo);
6202 } else {
6203 used = G_T5_USED(lo);
6204 alloc = G_T5_ALLOC(lo);
6205 }
6206 sbuf_printf(sb,
6207 "\nLoopback %d using %u pages out of %u allocated",
6208 i, used, alloc);
6209 }
6210
6211 rc = sbuf_finish(sb);
6212 sbuf_delete(sb);
6213
6214 return (rc);
6215 }
6216
6217 static inline void
tcamxy2valmask(uint64_t x,uint64_t y,uint8_t * addr,uint64_t * mask)6218 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
6219 {
6220 *mask = x | y;
6221 y = htobe64(y);
6222 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
6223 }
6224
6225 static int
sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)6226 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
6227 {
6228 struct adapter *sc = arg1;
6229 struct sbuf *sb;
6230 int rc, i, n;
6231
6232 rc = sysctl_wire_old_buffer(req, 0);
6233 if (rc != 0)
6234 return (rc);
6235
6236 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6237 if (sb == NULL)
6238 return (ENOMEM);
6239
6240 sbuf_printf(sb,
6241 "Idx Ethernet address Mask Vld Ports PF"
6242 " VF Replication P0 P1 P2 P3 ML");
6243 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES :
6244 NUM_MPS_T5_CLS_SRAM_L_INSTANCES;
6245 for (i = 0; i < n; i++) {
6246 uint64_t tcamx, tcamy, mask;
6247 uint32_t cls_lo, cls_hi;
6248 uint8_t addr[ETHER_ADDR_LEN];
6249
6250 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
6251 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
6252 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
6253 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
6254
6255 if (tcamx & tcamy)
6256 continue;
6257
6258 tcamxy2valmask(tcamx, tcamy, addr, &mask);
6259 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
6260 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2],
6261 addr[3], addr[4], addr[5], (uintmax_t)mask,
6262 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
6263 G_PORTMAP(cls_hi), G_PF(cls_lo),
6264 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
6265
6266 if (cls_lo & F_REPLICATE) {
6267 struct fw_ldst_cmd ldst_cmd;
6268
6269 memset(&ldst_cmd, 0, sizeof(ldst_cmd));
6270 ldst_cmd.op_to_addrspace =
6271 htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
6272 F_FW_CMD_REQUEST | F_FW_CMD_READ |
6273 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
6274 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
6275 ldst_cmd.u.mps.fid_ctl =
6276 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
6277 V_FW_LDST_CMD_CTL(i));
6278
6279 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
6280 "t4mps");
6281 if (rc)
6282 break;
6283 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
6284 sizeof(ldst_cmd), &ldst_cmd);
6285 end_synchronized_op(sc, 0);
6286
6287 if (rc != 0) {
6288 sbuf_printf(sb,
6289 " ------------ error %3u ------------", rc);
6290 rc = 0;
6291 } else {
6292 sbuf_printf(sb, " %08x %08x %08x %08x",
6293 be32toh(ldst_cmd.u.mps.rplc127_96),
6294 be32toh(ldst_cmd.u.mps.rplc95_64),
6295 be32toh(ldst_cmd.u.mps.rplc63_32),
6296 be32toh(ldst_cmd.u.mps.rplc31_0));
6297 }
6298 } else
6299 sbuf_printf(sb, "%36s", "");
6300
6301 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
6302 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
6303 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
6304 }
6305
6306 if (rc)
6307 (void) sbuf_finish(sb);
6308 else
6309 rc = sbuf_finish(sb);
6310 sbuf_delete(sb);
6311
6312 return (rc);
6313 }
6314
6315 static int
sysctl_path_mtus(SYSCTL_HANDLER_ARGS)6316 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
6317 {
6318 struct adapter *sc = arg1;
6319 struct sbuf *sb;
6320 int rc;
6321 uint16_t mtus[NMTUS];
6322
6323 rc = sysctl_wire_old_buffer(req, 0);
6324 if (rc != 0)
6325 return (rc);
6326
6327 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6328 if (sb == NULL)
6329 return (ENOMEM);
6330
6331 t4_read_mtu_tbl(sc, mtus, NULL);
6332
6333 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
6334 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
6335 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
6336 mtus[14], mtus[15]);
6337
6338 rc = sbuf_finish(sb);
6339 sbuf_delete(sb);
6340
6341 return (rc);
6342 }
6343
6344 static int
sysctl_pm_stats(SYSCTL_HANDLER_ARGS)6345 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
6346 {
6347 struct adapter *sc = arg1;
6348 struct sbuf *sb;
6349 int rc, i;
6350 uint32_t cnt[PM_NSTATS];
6351 uint64_t cyc[PM_NSTATS];
6352 static const char *rx_stats[] = {
6353 "Read:", "Write bypass:", "Write mem:", "Flush:"
6354 };
6355 static const char *tx_stats[] = {
6356 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:"
6357 };
6358
6359 rc = sysctl_wire_old_buffer(req, 0);
6360 if (rc != 0)
6361 return (rc);
6362
6363 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6364 if (sb == NULL)
6365 return (ENOMEM);
6366
6367 t4_pmtx_get_stats(sc, cnt, cyc);
6368 sbuf_printf(sb, " Tx pcmds Tx bytes");
6369 for (i = 0; i < ARRAY_SIZE(tx_stats); i++)
6370 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i],
6371 cyc[i]);
6372
6373 t4_pmrx_get_stats(sc, cnt, cyc);
6374 sbuf_printf(sb, "\n Rx pcmds Rx bytes");
6375 for (i = 0; i < ARRAY_SIZE(rx_stats); i++)
6376 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i],
6377 cyc[i]);
6378
6379 rc = sbuf_finish(sb);
6380 sbuf_delete(sb);
6381
6382 return (rc);
6383 }
6384
6385 static int
sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)6386 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
6387 {
6388 struct adapter *sc = arg1;
6389 struct sbuf *sb;
6390 int rc;
6391 struct tp_rdma_stats stats;
6392
6393 rc = sysctl_wire_old_buffer(req, 0);
6394 if (rc != 0)
6395 return (rc);
6396
6397 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6398 if (sb == NULL)
6399 return (ENOMEM);
6400
6401 t4_tp_get_rdma_stats(sc, &stats);
6402 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
6403 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
6404
6405 rc = sbuf_finish(sb);
6406 sbuf_delete(sb);
6407
6408 return (rc);
6409 }
6410
6411 static int
sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)6412 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
6413 {
6414 struct adapter *sc = arg1;
6415 struct sbuf *sb;
6416 int rc;
6417 struct tp_tcp_stats v4, v6;
6418
6419 rc = sysctl_wire_old_buffer(req, 0);
6420 if (rc != 0)
6421 return (rc);
6422
6423 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6424 if (sb == NULL)
6425 return (ENOMEM);
6426
6427 t4_tp_get_tcp_stats(sc, &v4, &v6);
6428 sbuf_printf(sb,
6429 " IP IPv6\n");
6430 sbuf_printf(sb, "OutRsts: %20u %20u\n",
6431 v4.tcpOutRsts, v6.tcpOutRsts);
6432 sbuf_printf(sb, "InSegs: %20ju %20ju\n",
6433 v4.tcpInSegs, v6.tcpInSegs);
6434 sbuf_printf(sb, "OutSegs: %20ju %20ju\n",
6435 v4.tcpOutSegs, v6.tcpOutSegs);
6436 sbuf_printf(sb, "RetransSegs: %20ju %20ju",
6437 v4.tcpRetransSegs, v6.tcpRetransSegs);
6438
6439 rc = sbuf_finish(sb);
6440 sbuf_delete(sb);
6441
6442 return (rc);
6443 }
6444
6445 static int
sysctl_tids(SYSCTL_HANDLER_ARGS)6446 sysctl_tids(SYSCTL_HANDLER_ARGS)
6447 {
6448 struct adapter *sc = arg1;
6449 struct sbuf *sb;
6450 int rc;
6451 struct tid_info *t = &sc->tids;
6452
6453 rc = sysctl_wire_old_buffer(req, 0);
6454 if (rc != 0)
6455 return (rc);
6456
6457 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6458 if (sb == NULL)
6459 return (ENOMEM);
6460
6461 if (t->natids) {
6462 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
6463 t->atids_in_use);
6464 }
6465
6466 if (t->ntids) {
6467 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
6468 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
6469
6470 if (b) {
6471 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
6472 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6473 t->ntids - 1);
6474 } else {
6475 sbuf_printf(sb, "TID range: %u-%u",
6476 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
6477 t->ntids - 1);
6478 }
6479 } else
6480 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
6481 sbuf_printf(sb, ", in use: %u\n",
6482 atomic_load_acq_int(&t->tids_in_use));
6483 }
6484
6485 if (t->nstids) {
6486 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
6487 t->stid_base + t->nstids - 1, t->stids_in_use);
6488 }
6489
6490 if (t->nftids) {
6491 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
6492 t->ftid_base + t->nftids - 1);
6493 }
6494
6495 if (t->netids) {
6496 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base,
6497 t->etid_base + t->netids - 1);
6498 }
6499
6500 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
6501 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
6502 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
6503
6504 rc = sbuf_finish(sb);
6505 sbuf_delete(sb);
6506
6507 return (rc);
6508 }
6509
6510 static int
sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)6511 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
6512 {
6513 struct adapter *sc = arg1;
6514 struct sbuf *sb;
6515 int rc;
6516 struct tp_err_stats stats;
6517
6518 rc = sysctl_wire_old_buffer(req, 0);
6519 if (rc != 0)
6520 return (rc);
6521
6522 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6523 if (sb == NULL)
6524 return (ENOMEM);
6525
6526 t4_tp_get_err_stats(sc, &stats);
6527
6528 sbuf_printf(sb, " channel 0 channel 1 channel 2 "
6529 "channel 3\n");
6530 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n",
6531 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
6532 stats.macInErrs[3]);
6533 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n",
6534 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
6535 stats.hdrInErrs[3]);
6536 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n",
6537 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
6538 stats.tcpInErrs[3]);
6539 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n",
6540 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
6541 stats.tcp6InErrs[3]);
6542 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n",
6543 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
6544 stats.tnlCongDrops[3]);
6545 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n",
6546 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
6547 stats.tnlTxDrops[3]);
6548 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n",
6549 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
6550 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
6551 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n",
6552 stats.ofldChanDrops[0], stats.ofldChanDrops[1],
6553 stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
6554 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u",
6555 stats.ofldNoNeigh, stats.ofldCongDefer);
6556
6557 rc = sbuf_finish(sb);
6558 sbuf_delete(sb);
6559
6560 return (rc);
6561 }
6562
6563 struct field_desc {
6564 const char *name;
6565 u_int start;
6566 u_int width;
6567 };
6568
6569 static void
field_desc_show(struct sbuf * sb,uint64_t v,const struct field_desc * f)6570 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
6571 {
6572 char buf[32];
6573 int line_size = 0;
6574
6575 while (f->name) {
6576 uint64_t mask = (1ULL << f->width) - 1;
6577 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
6578 ((uintmax_t)v >> f->start) & mask);
6579
6580 if (line_size + len >= 79) {
6581 line_size = 8;
6582 sbuf_printf(sb, "\n ");
6583 }
6584 sbuf_printf(sb, "%s ", buf);
6585 line_size += len + 1;
6586 f++;
6587 }
6588 sbuf_printf(sb, "\n");
6589 }
6590
6591 static struct field_desc tp_la0[] = {
6592 { "RcfOpCodeOut", 60, 4 },
6593 { "State", 56, 4 },
6594 { "WcfState", 52, 4 },
6595 { "RcfOpcSrcOut", 50, 2 },
6596 { "CRxError", 49, 1 },
6597 { "ERxError", 48, 1 },
6598 { "SanityFailed", 47, 1 },
6599 { "SpuriousMsg", 46, 1 },
6600 { "FlushInputMsg", 45, 1 },
6601 { "FlushInputCpl", 44, 1 },
6602 { "RssUpBit", 43, 1 },
6603 { "RssFilterHit", 42, 1 },
6604 { "Tid", 32, 10 },
6605 { "InitTcb", 31, 1 },
6606 { "LineNumber", 24, 7 },
6607 { "Emsg", 23, 1 },
6608 { "EdataOut", 22, 1 },
6609 { "Cmsg", 21, 1 },
6610 { "CdataOut", 20, 1 },
6611 { "EreadPdu", 19, 1 },
6612 { "CreadPdu", 18, 1 },
6613 { "TunnelPkt", 17, 1 },
6614 { "RcfPeerFin", 16, 1 },
6615 { "RcfReasonOut", 12, 4 },
6616 { "TxCchannel", 10, 2 },
6617 { "RcfTxChannel", 8, 2 },
6618 { "RxEchannel", 6, 2 },
6619 { "RcfRxChannel", 5, 1 },
6620 { "RcfDataOutSrdy", 4, 1 },
6621 { "RxDvld", 3, 1 },
6622 { "RxOoDvld", 2, 1 },
6623 { "RxCongestion", 1, 1 },
6624 { "TxCongestion", 0, 1 },
6625 { NULL }
6626 };
6627
6628 static struct field_desc tp_la1[] = {
6629 { "CplCmdIn", 56, 8 },
6630 { "CplCmdOut", 48, 8 },
6631 { "ESynOut", 47, 1 },
6632 { "EAckOut", 46, 1 },
6633 { "EFinOut", 45, 1 },
6634 { "ERstOut", 44, 1 },
6635 { "SynIn", 43, 1 },
6636 { "AckIn", 42, 1 },
6637 { "FinIn", 41, 1 },
6638 { "RstIn", 40, 1 },
6639 { "DataIn", 39, 1 },
6640 { "DataInVld", 38, 1 },
6641 { "PadIn", 37, 1 },
6642 { "RxBufEmpty", 36, 1 },
6643 { "RxDdp", 35, 1 },
6644 { "RxFbCongestion", 34, 1 },
6645 { "TxFbCongestion", 33, 1 },
6646 { "TxPktSumSrdy", 32, 1 },
6647 { "RcfUlpType", 28, 4 },
6648 { "Eread", 27, 1 },
6649 { "Ebypass", 26, 1 },
6650 { "Esave", 25, 1 },
6651 { "Static0", 24, 1 },
6652 { "Cread", 23, 1 },
6653 { "Cbypass", 22, 1 },
6654 { "Csave", 21, 1 },
6655 { "CPktOut", 20, 1 },
6656 { "RxPagePoolFull", 18, 2 },
6657 { "RxLpbkPkt", 17, 1 },
6658 { "TxLpbkPkt", 16, 1 },
6659 { "RxVfValid", 15, 1 },
6660 { "SynLearned", 14, 1 },
6661 { "SetDelEntry", 13, 1 },
6662 { "SetInvEntry", 12, 1 },
6663 { "CpcmdDvld", 11, 1 },
6664 { "CpcmdSave", 10, 1 },
6665 { "RxPstructsFull", 8, 2 },
6666 { "EpcmdDvld", 7, 1 },
6667 { "EpcmdFlush", 6, 1 },
6668 { "EpcmdTrimPrefix", 5, 1 },
6669 { "EpcmdTrimPostfix", 4, 1 },
6670 { "ERssIp4Pkt", 3, 1 },
6671 { "ERssIp6Pkt", 2, 1 },
6672 { "ERssTcpUdpPkt", 1, 1 },
6673 { "ERssFceFipPkt", 0, 1 },
6674 { NULL }
6675 };
6676
6677 static struct field_desc tp_la2[] = {
6678 { "CplCmdIn", 56, 8 },
6679 { "MpsVfVld", 55, 1 },
6680 { "MpsPf", 52, 3 },
6681 { "MpsVf", 44, 8 },
6682 { "SynIn", 43, 1 },
6683 { "AckIn", 42, 1 },
6684 { "FinIn", 41, 1 },
6685 { "RstIn", 40, 1 },
6686 { "DataIn", 39, 1 },
6687 { "DataInVld", 38, 1 },
6688 { "PadIn", 37, 1 },
6689 { "RxBufEmpty", 36, 1 },
6690 { "RxDdp", 35, 1 },
6691 { "RxFbCongestion", 34, 1 },
6692 { "TxFbCongestion", 33, 1 },
6693 { "TxPktSumSrdy", 32, 1 },
6694 { "RcfUlpType", 28, 4 },
6695 { "Eread", 27, 1 },
6696 { "Ebypass", 26, 1 },
6697 { "Esave", 25, 1 },
6698 { "Static0", 24, 1 },
6699 { "Cread", 23, 1 },
6700 { "Cbypass", 22, 1 },
6701 { "Csave", 21, 1 },
6702 { "CPktOut", 20, 1 },
6703 { "RxPagePoolFull", 18, 2 },
6704 { "RxLpbkPkt", 17, 1 },
6705 { "TxLpbkPkt", 16, 1 },
6706 { "RxVfValid", 15, 1 },
6707 { "SynLearned", 14, 1 },
6708 { "SetDelEntry", 13, 1 },
6709 { "SetInvEntry", 12, 1 },
6710 { "CpcmdDvld", 11, 1 },
6711 { "CpcmdSave", 10, 1 },
6712 { "RxPstructsFull", 8, 2 },
6713 { "EpcmdDvld", 7, 1 },
6714 { "EpcmdFlush", 6, 1 },
6715 { "EpcmdTrimPrefix", 5, 1 },
6716 { "EpcmdTrimPostfix", 4, 1 },
6717 { "ERssIp4Pkt", 3, 1 },
6718 { "ERssIp6Pkt", 2, 1 },
6719 { "ERssTcpUdpPkt", 1, 1 },
6720 { "ERssFceFipPkt", 0, 1 },
6721 { NULL }
6722 };
6723
6724 static void
tp_la_show(struct sbuf * sb,uint64_t * p,int idx)6725 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
6726 {
6727
6728 field_desc_show(sb, *p, tp_la0);
6729 }
6730
6731 static void
tp_la_show2(struct sbuf * sb,uint64_t * p,int idx)6732 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
6733 {
6734
6735 if (idx)
6736 sbuf_printf(sb, "\n");
6737 field_desc_show(sb, p[0], tp_la0);
6738 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6739 field_desc_show(sb, p[1], tp_la0);
6740 }
6741
6742 static void
tp_la_show3(struct sbuf * sb,uint64_t * p,int idx)6743 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
6744 {
6745
6746 if (idx)
6747 sbuf_printf(sb, "\n");
6748 field_desc_show(sb, p[0], tp_la0);
6749 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
6750 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
6751 }
6752
6753 static int
sysctl_tp_la(SYSCTL_HANDLER_ARGS)6754 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
6755 {
6756 struct adapter *sc = arg1;
6757 struct sbuf *sb;
6758 uint64_t *buf, *p;
6759 int rc;
6760 u_int i, inc;
6761 void (*show_func)(struct sbuf *, uint64_t *, int);
6762
6763 rc = sysctl_wire_old_buffer(req, 0);
6764 if (rc != 0)
6765 return (rc);
6766
6767 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6768 if (sb == NULL)
6769 return (ENOMEM);
6770
6771 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
6772
6773 t4_tp_read_la(sc, buf, NULL);
6774 p = buf;
6775
6776 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
6777 case 2:
6778 inc = 2;
6779 show_func = tp_la_show2;
6780 break;
6781 case 3:
6782 inc = 2;
6783 show_func = tp_la_show3;
6784 break;
6785 default:
6786 inc = 1;
6787 show_func = tp_la_show;
6788 }
6789
6790 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
6791 (*show_func)(sb, p, i);
6792
6793 rc = sbuf_finish(sb);
6794 sbuf_delete(sb);
6795 free(buf, M_CXGBE);
6796 return (rc);
6797 }
6798
6799 static int
sysctl_tx_rate(SYSCTL_HANDLER_ARGS)6800 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
6801 {
6802 struct adapter *sc = arg1;
6803 struct sbuf *sb;
6804 int rc;
6805 u64 nrate[NCHAN], orate[NCHAN];
6806
6807 rc = sysctl_wire_old_buffer(req, 0);
6808 if (rc != 0)
6809 return (rc);
6810
6811 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
6812 if (sb == NULL)
6813 return (ENOMEM);
6814
6815 t4_get_chan_txrate(sc, nrate, orate);
6816 sbuf_printf(sb, " channel 0 channel 1 channel 2 "
6817 "channel 3\n");
6818 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n",
6819 nrate[0], nrate[1], nrate[2], nrate[3]);
6820 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju",
6821 orate[0], orate[1], orate[2], orate[3]);
6822
6823 rc = sbuf_finish(sb);
6824 sbuf_delete(sb);
6825
6826 return (rc);
6827 }
6828
6829 static int
sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)6830 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
6831 {
6832 struct adapter *sc = arg1;
6833 struct sbuf *sb;
6834 uint32_t *buf, *p;
6835 int rc, i;
6836
6837 rc = sysctl_wire_old_buffer(req, 0);
6838 if (rc != 0)
6839 return (rc);
6840
6841 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6842 if (sb == NULL)
6843 return (ENOMEM);
6844
6845 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
6846 M_ZERO | M_WAITOK);
6847
6848 t4_ulprx_read_la(sc, buf);
6849 p = buf;
6850
6851 sbuf_printf(sb, " Pcmd Type Message"
6852 " Data");
6853 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
6854 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x",
6855 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
6856 }
6857
6858 rc = sbuf_finish(sb);
6859 sbuf_delete(sb);
6860 free(buf, M_CXGBE);
6861 return (rc);
6862 }
6863
6864 static int
sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)6865 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
6866 {
6867 struct adapter *sc = arg1;
6868 struct sbuf *sb;
6869 int rc, v;
6870
6871 rc = sysctl_wire_old_buffer(req, 0);
6872 if (rc != 0)
6873 return (rc);
6874
6875 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
6876 if (sb == NULL)
6877 return (ENOMEM);
6878
6879 v = t4_read_reg(sc, A_SGE_STAT_CFG);
6880 if (G_STATSOURCE_T5(v) == 7) {
6881 if (G_STATMODE(v) == 0) {
6882 sbuf_printf(sb, "total %d, incomplete %d",
6883 t4_read_reg(sc, A_SGE_STAT_TOTAL),
6884 t4_read_reg(sc, A_SGE_STAT_MATCH));
6885 } else if (G_STATMODE(v) == 1) {
6886 sbuf_printf(sb, "total %d, data overflow %d",
6887 t4_read_reg(sc, A_SGE_STAT_TOTAL),
6888 t4_read_reg(sc, A_SGE_STAT_MATCH));
6889 }
6890 }
6891 rc = sbuf_finish(sb);
6892 sbuf_delete(sb);
6893
6894 return (rc);
6895 }
6896 #endif
6897
6898 static uint32_t
fconf_to_mode(uint32_t fconf)6899 fconf_to_mode(uint32_t fconf)
6900 {
6901 uint32_t mode;
6902
6903 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
6904 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
6905
6906 if (fconf & F_FRAGMENTATION)
6907 mode |= T4_FILTER_IP_FRAGMENT;
6908
6909 if (fconf & F_MPSHITTYPE)
6910 mode |= T4_FILTER_MPS_HIT_TYPE;
6911
6912 if (fconf & F_MACMATCH)
6913 mode |= T4_FILTER_MAC_IDX;
6914
6915 if (fconf & F_ETHERTYPE)
6916 mode |= T4_FILTER_ETH_TYPE;
6917
6918 if (fconf & F_PROTOCOL)
6919 mode |= T4_FILTER_IP_PROTO;
6920
6921 if (fconf & F_TOS)
6922 mode |= T4_FILTER_IP_TOS;
6923
6924 if (fconf & F_VLAN)
6925 mode |= T4_FILTER_VLAN;
6926
6927 if (fconf & F_VNIC_ID)
6928 mode |= T4_FILTER_VNIC;
6929
6930 if (fconf & F_PORT)
6931 mode |= T4_FILTER_PORT;
6932
6933 if (fconf & F_FCOE)
6934 mode |= T4_FILTER_FCoE;
6935
6936 return (mode);
6937 }
6938
6939 static uint32_t
mode_to_fconf(uint32_t mode)6940 mode_to_fconf(uint32_t mode)
6941 {
6942 uint32_t fconf = 0;
6943
6944 if (mode & T4_FILTER_IP_FRAGMENT)
6945 fconf |= F_FRAGMENTATION;
6946
6947 if (mode & T4_FILTER_MPS_HIT_TYPE)
6948 fconf |= F_MPSHITTYPE;
6949
6950 if (mode & T4_FILTER_MAC_IDX)
6951 fconf |= F_MACMATCH;
6952
6953 if (mode & T4_FILTER_ETH_TYPE)
6954 fconf |= F_ETHERTYPE;
6955
6956 if (mode & T4_FILTER_IP_PROTO)
6957 fconf |= F_PROTOCOL;
6958
6959 if (mode & T4_FILTER_IP_TOS)
6960 fconf |= F_TOS;
6961
6962 if (mode & T4_FILTER_VLAN)
6963 fconf |= F_VLAN;
6964
6965 if (mode & T4_FILTER_VNIC)
6966 fconf |= F_VNIC_ID;
6967
6968 if (mode & T4_FILTER_PORT)
6969 fconf |= F_PORT;
6970
6971 if (mode & T4_FILTER_FCoE)
6972 fconf |= F_FCOE;
6973
6974 return (fconf);
6975 }
6976
6977 static uint32_t
fspec_to_fconf(struct t4_filter_specification * fs)6978 fspec_to_fconf(struct t4_filter_specification *fs)
6979 {
6980 uint32_t fconf = 0;
6981
6982 if (fs->val.frag || fs->mask.frag)
6983 fconf |= F_FRAGMENTATION;
6984
6985 if (fs->val.matchtype || fs->mask.matchtype)
6986 fconf |= F_MPSHITTYPE;
6987
6988 if (fs->val.macidx || fs->mask.macidx)
6989 fconf |= F_MACMATCH;
6990
6991 if (fs->val.ethtype || fs->mask.ethtype)
6992 fconf |= F_ETHERTYPE;
6993
6994 if (fs->val.proto || fs->mask.proto)
6995 fconf |= F_PROTOCOL;
6996
6997 if (fs->val.tos || fs->mask.tos)
6998 fconf |= F_TOS;
6999
7000 if (fs->val.vlan_vld || fs->mask.vlan_vld)
7001 fconf |= F_VLAN;
7002
7003 if (fs->val.vnic_vld || fs->mask.vnic_vld)
7004 fconf |= F_VNIC_ID;
7005
7006 if (fs->val.iport || fs->mask.iport)
7007 fconf |= F_PORT;
7008
7009 if (fs->val.fcoe || fs->mask.fcoe)
7010 fconf |= F_FCOE;
7011
7012 return (fconf);
7013 }
7014
7015 static int
get_filter_mode(struct adapter * sc,uint32_t * mode)7016 get_filter_mode(struct adapter *sc, uint32_t *mode)
7017 {
7018 int rc;
7019 uint32_t fconf;
7020
7021 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7022 "t4getfm");
7023 if (rc)
7024 return (rc);
7025
7026 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
7027 A_TP_VLAN_PRI_MAP);
7028
7029 if (sc->params.tp.vlan_pri_map != fconf) {
7030 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
7031 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
7032 fconf);
7033 }
7034
7035 *mode = fconf_to_mode(fconf);
7036
7037 end_synchronized_op(sc, LOCK_HELD);
7038 return (0);
7039 }
7040
7041 static int
set_filter_mode(struct adapter * sc,uint32_t mode)7042 set_filter_mode(struct adapter *sc, uint32_t mode)
7043 {
7044 uint32_t fconf;
7045 int rc;
7046
7047 fconf = mode_to_fconf(mode);
7048
7049 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7050 "t4setfm");
7051 if (rc)
7052 return (rc);
7053
7054 if (sc->tids.ftids_in_use > 0) {
7055 rc = EBUSY;
7056 goto done;
7057 }
7058
7059 #ifdef TCP_OFFLOAD
7060 if (uld_active(sc, ULD_TOM)) {
7061 rc = EBUSY;
7062 goto done;
7063 }
7064 #endif
7065
7066 rc = -t4_set_filter_mode(sc, fconf);
7067 done:
7068 end_synchronized_op(sc, LOCK_HELD);
7069 return (rc);
7070 }
7071
7072 static inline uint64_t
get_filter_hits(struct adapter * sc,uint32_t fid)7073 get_filter_hits(struct adapter *sc, uint32_t fid)
7074 {
7075 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
7076 uint64_t hits;
7077
7078 memwin_info(sc, 0, &mw_base, NULL);
7079 off = position_memwin(sc, 0,
7080 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
7081 if (is_t4(sc)) {
7082 hits = t4_read_reg64(sc, mw_base + off + 16);
7083 hits = be64toh(hits);
7084 } else {
7085 hits = t4_read_reg(sc, mw_base + off + 24);
7086 hits = be32toh(hits);
7087 }
7088
7089 return (hits);
7090 }
7091
7092 static int
get_filter(struct adapter * sc,struct t4_filter * t)7093 get_filter(struct adapter *sc, struct t4_filter *t)
7094 {
7095 int i, rc, nfilters = sc->tids.nftids;
7096 struct filter_entry *f;
7097
7098 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
7099 "t4getf");
7100 if (rc)
7101 return (rc);
7102
7103 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
7104 t->idx >= nfilters) {
7105 t->idx = 0xffffffff;
7106 goto done;
7107 }
7108
7109 f = &sc->tids.ftid_tab[t->idx];
7110 for (i = t->idx; i < nfilters; i++, f++) {
7111 if (f->valid) {
7112 t->idx = i;
7113 t->l2tidx = f->l2t ? f->l2t->idx : 0;
7114 t->smtidx = f->smtidx;
7115 if (f->fs.hitcnts)
7116 t->hits = get_filter_hits(sc, t->idx);
7117 else
7118 t->hits = UINT64_MAX;
7119 t->fs = f->fs;
7120
7121 goto done;
7122 }
7123 }
7124
7125 t->idx = 0xffffffff;
7126 done:
7127 end_synchronized_op(sc, LOCK_HELD);
7128 return (0);
7129 }
7130
7131 static int
set_filter(struct adapter * sc,struct t4_filter * t)7132 set_filter(struct adapter *sc, struct t4_filter *t)
7133 {
7134 unsigned int nfilters, nports;
7135 struct filter_entry *f;
7136 int i, rc;
7137
7138 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
7139 if (rc)
7140 return (rc);
7141
7142 nfilters = sc->tids.nftids;
7143 nports = sc->params.nports;
7144
7145 if (nfilters == 0) {
7146 rc = ENOTSUP;
7147 goto done;
7148 }
7149
7150 if (!(sc->flags & FULL_INIT_DONE)) {
7151 rc = EAGAIN;
7152 goto done;
7153 }
7154
7155 if (t->idx >= nfilters) {
7156 rc = EINVAL;
7157 goto done;
7158 }
7159
7160 /* Validate against the global filter mode */
7161 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
7162 sc->params.tp.vlan_pri_map) {
7163 rc = E2BIG;
7164 goto done;
7165 }
7166
7167 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
7168 rc = EINVAL;
7169 goto done;
7170 }
7171
7172 if (t->fs.val.iport >= nports) {
7173 rc = EINVAL;
7174 goto done;
7175 }
7176
7177 /* Can't specify an iq if not steering to it */
7178 if (!t->fs.dirsteer && t->fs.iq) {
7179 rc = EINVAL;
7180 goto done;
7181 }
7182
7183 /* IPv6 filter idx must be 4 aligned */
7184 if (t->fs.type == 1 &&
7185 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
7186 rc = EINVAL;
7187 goto done;
7188 }
7189
7190 if (sc->tids.ftid_tab == NULL) {
7191 KASSERT(sc->tids.ftids_in_use == 0,
7192 ("%s: no memory allocated but filters_in_use > 0",
7193 __func__));
7194
7195 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
7196 nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
7197 if (sc->tids.ftid_tab == NULL) {
7198 rc = ENOMEM;
7199 goto done;
7200 }
7201 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
7202 }
7203
7204 for (i = 0; i < 4; i++) {
7205 f = &sc->tids.ftid_tab[t->idx + i];
7206
7207 if (f->pending || f->valid) {
7208 rc = EBUSY;
7209 goto done;
7210 }
7211 if (f->locked) {
7212 rc = EPERM;
7213 goto done;
7214 }
7215
7216 if (t->fs.type == 0)
7217 break;
7218 }
7219
7220 f = &sc->tids.ftid_tab[t->idx];
7221 f->fs = t->fs;
7222
7223 rc = set_filter_wr(sc, t->idx);
7224 done:
7225 end_synchronized_op(sc, 0);
7226
7227 if (rc == 0) {
7228 mtx_lock(&sc->tids.ftid_lock);
7229 for (;;) {
7230 if (f->pending == 0) {
7231 rc = f->valid ? 0 : EIO;
7232 break;
7233 }
7234
7235 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7236 PCATCH, "t4setfw", 0)) {
7237 rc = EINPROGRESS;
7238 break;
7239 }
7240 }
7241 mtx_unlock(&sc->tids.ftid_lock);
7242 }
7243 return (rc);
7244 }
7245
7246 static int
del_filter(struct adapter * sc,struct t4_filter * t)7247 del_filter(struct adapter *sc, struct t4_filter *t)
7248 {
7249 unsigned int nfilters;
7250 struct filter_entry *f;
7251 int rc;
7252
7253 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
7254 if (rc)
7255 return (rc);
7256
7257 nfilters = sc->tids.nftids;
7258
7259 if (nfilters == 0) {
7260 rc = ENOTSUP;
7261 goto done;
7262 }
7263
7264 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
7265 t->idx >= nfilters) {
7266 rc = EINVAL;
7267 goto done;
7268 }
7269
7270 if (!(sc->flags & FULL_INIT_DONE)) {
7271 rc = EAGAIN;
7272 goto done;
7273 }
7274
7275 f = &sc->tids.ftid_tab[t->idx];
7276
7277 if (f->pending) {
7278 rc = EBUSY;
7279 goto done;
7280 }
7281 if (f->locked) {
7282 rc = EPERM;
7283 goto done;
7284 }
7285
7286 if (f->valid) {
7287 t->fs = f->fs; /* extra info for the caller */
7288 rc = del_filter_wr(sc, t->idx);
7289 }
7290
7291 done:
7292 end_synchronized_op(sc, 0);
7293
7294 if (rc == 0) {
7295 mtx_lock(&sc->tids.ftid_lock);
7296 for (;;) {
7297 if (f->pending == 0) {
7298 rc = f->valid ? EIO : 0;
7299 break;
7300 }
7301
7302 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
7303 PCATCH, "t4delfw", 0)) {
7304 rc = EINPROGRESS;
7305 break;
7306 }
7307 }
7308 mtx_unlock(&sc->tids.ftid_lock);
7309 }
7310
7311 return (rc);
7312 }
7313
7314 static void
clear_filter(struct filter_entry * f)7315 clear_filter(struct filter_entry *f)
7316 {
7317 if (f->l2t)
7318 t4_l2t_release(f->l2t);
7319
7320 bzero(f, sizeof (*f));
7321 }
7322
7323 static int
set_filter_wr(struct adapter * sc,int fidx)7324 set_filter_wr(struct adapter *sc, int fidx)
7325 {
7326 struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7327 struct fw_filter_wr *fwr;
7328 unsigned int ftid;
7329 struct wrq_cookie cookie;
7330
7331 ASSERT_SYNCHRONIZED_OP(sc);
7332
7333 if (f->fs.newdmac || f->fs.newvlan) {
7334 /* This filter needs an L2T entry; allocate one. */
7335 f->l2t = t4_l2t_alloc_switching(sc->l2t);
7336 if (f->l2t == NULL)
7337 return (EAGAIN);
7338 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
7339 f->fs.dmac)) {
7340 t4_l2t_release(f->l2t);
7341 f->l2t = NULL;
7342 return (ENOMEM);
7343 }
7344 }
7345
7346 ftid = sc->tids.ftid_base + fidx;
7347
7348 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7349 if (fwr == NULL)
7350 return (ENOMEM);
7351 bzero(fwr, sizeof(*fwr));
7352
7353 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
7354 fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
7355 fwr->tid_to_iq =
7356 htobe32(V_FW_FILTER_WR_TID(ftid) |
7357 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
7358 V_FW_FILTER_WR_NOREPLY(0) |
7359 V_FW_FILTER_WR_IQ(f->fs.iq));
7360 fwr->del_filter_to_l2tix =
7361 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
7362 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
7363 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
7364 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
7365 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
7366 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
7367 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
7368 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
7369 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
7370 f->fs.newvlan == VLAN_REWRITE) |
7371 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
7372 f->fs.newvlan == VLAN_REWRITE) |
7373 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
7374 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
7375 V_FW_FILTER_WR_PRIO(f->fs.prio) |
7376 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
7377 fwr->ethtype = htobe16(f->fs.val.ethtype);
7378 fwr->ethtypem = htobe16(f->fs.mask.ethtype);
7379 fwr->frag_to_ovlan_vldm =
7380 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
7381 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
7382 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
7383 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
7384 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
7385 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
7386 fwr->smac_sel = 0;
7387 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
7388 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
7389 fwr->maci_to_matchtypem =
7390 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
7391 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
7392 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
7393 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
7394 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
7395 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
7396 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
7397 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
7398 fwr->ptcl = f->fs.val.proto;
7399 fwr->ptclm = f->fs.mask.proto;
7400 fwr->ttyp = f->fs.val.tos;
7401 fwr->ttypm = f->fs.mask.tos;
7402 fwr->ivlan = htobe16(f->fs.val.vlan);
7403 fwr->ivlanm = htobe16(f->fs.mask.vlan);
7404 fwr->ovlan = htobe16(f->fs.val.vnic);
7405 fwr->ovlanm = htobe16(f->fs.mask.vnic);
7406 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
7407 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
7408 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
7409 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
7410 fwr->lp = htobe16(f->fs.val.dport);
7411 fwr->lpm = htobe16(f->fs.mask.dport);
7412 fwr->fp = htobe16(f->fs.val.sport);
7413 fwr->fpm = htobe16(f->fs.mask.sport);
7414 if (f->fs.newsmac)
7415 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
7416
7417 f->pending = 1;
7418 sc->tids.ftids_in_use++;
7419
7420 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7421 return (0);
7422 }
7423
7424 static int
del_filter_wr(struct adapter * sc,int fidx)7425 del_filter_wr(struct adapter *sc, int fidx)
7426 {
7427 struct filter_entry *f = &sc->tids.ftid_tab[fidx];
7428 struct fw_filter_wr *fwr;
7429 unsigned int ftid;
7430 struct wrq_cookie cookie;
7431
7432 ftid = sc->tids.ftid_base + fidx;
7433
7434 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
7435 if (fwr == NULL)
7436 return (ENOMEM);
7437 bzero(fwr, sizeof (*fwr));
7438
7439 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
7440
7441 f->pending = 1;
7442 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie);
7443 return (0);
7444 }
7445
7446 int
t4_filter_rpl(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)7447 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
7448 {
7449 struct adapter *sc = iq->adapter;
7450 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
7451 unsigned int idx = GET_TID(rpl);
7452 unsigned int rc;
7453 struct filter_entry *f;
7454
7455 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
7456 rss->opcode));
7457
7458 if (is_ftid(sc, idx)) {
7459
7460 idx -= sc->tids.ftid_base;
7461 f = &sc->tids.ftid_tab[idx];
7462 rc = G_COOKIE(rpl->cookie);
7463
7464 mtx_lock(&sc->tids.ftid_lock);
7465 if (rc == FW_FILTER_WR_FLT_ADDED) {
7466 KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
7467 __func__, idx));
7468 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
7469 f->pending = 0; /* asynchronous setup completed */
7470 f->valid = 1;
7471 } else {
7472 if (rc != FW_FILTER_WR_FLT_DELETED) {
7473 /* Add or delete failed, display an error */
7474 log(LOG_ERR,
7475 "filter %u setup failed with error %u\n",
7476 idx, rc);
7477 }
7478
7479 clear_filter(f);
7480 sc->tids.ftids_in_use--;
7481 }
7482 wakeup(&sc->tids.ftid_tab);
7483 mtx_unlock(&sc->tids.ftid_lock);
7484 }
7485
7486 return (0);
7487 }
7488
7489 static int
get_sge_context(struct adapter * sc,struct t4_sge_context * cntxt)7490 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
7491 {
7492 int rc;
7493
7494 if (cntxt->cid > M_CTXTQID)
7495 return (EINVAL);
7496
7497 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
7498 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
7499 return (EINVAL);
7500
7501 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
7502 if (rc)
7503 return (rc);
7504
7505 if (sc->flags & FW_OK) {
7506 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
7507 &cntxt->data[0]);
7508 if (rc == 0)
7509 goto done;
7510 }
7511
7512 /*
7513 * Read via firmware failed or wasn't even attempted. Read directly via
7514 * the backdoor.
7515 */
7516 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
7517 done:
7518 end_synchronized_op(sc, 0);
7519 return (rc);
7520 }
7521
7522 static int
load_fw(struct adapter * sc,struct t4_data * fw)7523 load_fw(struct adapter *sc, struct t4_data *fw)
7524 {
7525 int rc;
7526 uint8_t *fw_data;
7527
7528 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
7529 if (rc)
7530 return (rc);
7531
7532 if (sc->flags & FULL_INIT_DONE) {
7533 rc = EBUSY;
7534 goto done;
7535 }
7536
7537 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
7538 if (fw_data == NULL) {
7539 rc = ENOMEM;
7540 goto done;
7541 }
7542
7543 rc = copyin(fw->data, fw_data, fw->len);
7544 if (rc == 0)
7545 rc = -t4_load_fw(sc, fw_data, fw->len);
7546
7547 free(fw_data, M_CXGBE);
7548 done:
7549 end_synchronized_op(sc, 0);
7550 return (rc);
7551 }
7552
7553 static int
read_card_mem(struct adapter * sc,int win,struct t4_mem_range * mr)7554 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
7555 {
7556 uint32_t addr, off, remaining, i, n;
7557 uint32_t *buf, *b;
7558 uint32_t mw_base, mw_aperture;
7559 int rc;
7560 uint8_t *dst;
7561
7562 rc = validate_mem_range(sc, mr->addr, mr->len);
7563 if (rc != 0)
7564 return (rc);
7565
7566 memwin_info(sc, win, &mw_base, &mw_aperture);
7567 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK);
7568 addr = mr->addr;
7569 remaining = mr->len;
7570 dst = (void *)mr->data;
7571
7572 while (remaining) {
7573 off = position_memwin(sc, win, addr);
7574
7575 /* number of bytes that we'll copy in the inner loop */
7576 n = min(remaining, mw_aperture - off);
7577 for (i = 0; i < n; i += 4)
7578 *b++ = t4_read_reg(sc, mw_base + off + i);
7579
7580 rc = copyout(buf, dst, n);
7581 if (rc != 0)
7582 break;
7583
7584 b = buf;
7585 dst += n;
7586 remaining -= n;
7587 addr += n;
7588 }
7589
7590 free(buf, M_CXGBE);
7591 return (rc);
7592 }
7593
7594 static int
read_i2c(struct adapter * sc,struct t4_i2c_data * i2cd)7595 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
7596 {
7597 int rc;
7598
7599 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
7600 return (EINVAL);
7601
7602 if (i2cd->len > sizeof(i2cd->data))
7603 return (EFBIG);
7604
7605 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
7606 if (rc)
7607 return (rc);
7608 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
7609 i2cd->offset, i2cd->len, &i2cd->data[0]);
7610 end_synchronized_op(sc, 0);
7611
7612 return (rc);
7613 }
7614
7615 static int
in_range(int val,int lo,int hi)7616 in_range(int val, int lo, int hi)
7617 {
7618
7619 return (val < 0 || (val <= hi && val >= lo));
7620 }
7621
7622 static int
set_sched_class(struct adapter * sc,struct t4_sched_params * p)7623 set_sched_class(struct adapter *sc, struct t4_sched_params *p)
7624 {
7625 int fw_subcmd, fw_type, rc;
7626
7627 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc");
7628 if (rc)
7629 return (rc);
7630
7631 if (!(sc->flags & FULL_INIT_DONE)) {
7632 rc = EAGAIN;
7633 goto done;
7634 }
7635
7636 /*
7637 * Translate the cxgbetool parameters into T4 firmware parameters. (The
7638 * sub-command and type are in common locations.)
7639 */
7640 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
7641 fw_subcmd = FW_SCHED_SC_CONFIG;
7642 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
7643 fw_subcmd = FW_SCHED_SC_PARAMS;
7644 else {
7645 rc = EINVAL;
7646 goto done;
7647 }
7648 if (p->type == SCHED_CLASS_TYPE_PACKET)
7649 fw_type = FW_SCHED_TYPE_PKTSCHED;
7650 else {
7651 rc = EINVAL;
7652 goto done;
7653 }
7654
7655 if (fw_subcmd == FW_SCHED_SC_CONFIG) {
7656 /* Vet our parameters ..*/
7657 if (p->u.config.minmax < 0) {
7658 rc = EINVAL;
7659 goto done;
7660 }
7661
7662 /* And pass the request to the firmware ...*/
7663 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1);
7664 goto done;
7665 }
7666
7667 if (fw_subcmd == FW_SCHED_SC_PARAMS) {
7668 int fw_level;
7669 int fw_mode;
7670 int fw_rateunit;
7671 int fw_ratemode;
7672
7673 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL)
7674 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
7675 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR)
7676 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
7677 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL)
7678 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
7679 else {
7680 rc = EINVAL;
7681 goto done;
7682 }
7683
7684 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS)
7685 fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
7686 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW)
7687 fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
7688 else {
7689 rc = EINVAL;
7690 goto done;
7691 }
7692
7693 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS)
7694 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
7695 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS)
7696 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
7697 else {
7698 rc = EINVAL;
7699 goto done;
7700 }
7701
7702 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL)
7703 fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
7704 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS)
7705 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
7706 else {
7707 rc = EINVAL;
7708 goto done;
7709 }
7710
7711 /* Vet our parameters ... */
7712 if (!in_range(p->u.params.channel, 0, 3) ||
7713 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) ||
7714 !in_range(p->u.params.minrate, 0, 10000000) ||
7715 !in_range(p->u.params.maxrate, 0, 10000000) ||
7716 !in_range(p->u.params.weight, 0, 100)) {
7717 rc = ERANGE;
7718 goto done;
7719 }
7720
7721 /*
7722 * Translate any unset parameters into the firmware's
7723 * nomenclature and/or fail the call if the parameters
7724 * are required ...
7725 */
7726 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 ||
7727 p->u.params.channel < 0 || p->u.params.cl < 0) {
7728 rc = EINVAL;
7729 goto done;
7730 }
7731 if (p->u.params.minrate < 0)
7732 p->u.params.minrate = 0;
7733 if (p->u.params.maxrate < 0) {
7734 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7735 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7736 rc = EINVAL;
7737 goto done;
7738 } else
7739 p->u.params.maxrate = 0;
7740 }
7741 if (p->u.params.weight < 0) {
7742 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) {
7743 rc = EINVAL;
7744 goto done;
7745 } else
7746 p->u.params.weight = 0;
7747 }
7748 if (p->u.params.pktsize < 0) {
7749 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL ||
7750 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) {
7751 rc = EINVAL;
7752 goto done;
7753 } else
7754 p->u.params.pktsize = 0;
7755 }
7756
7757 /* See what the firmware thinks of the request ... */
7758 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode,
7759 fw_rateunit, fw_ratemode, p->u.params.channel,
7760 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate,
7761 p->u.params.weight, p->u.params.pktsize, 1);
7762 goto done;
7763 }
7764
7765 rc = EINVAL;
7766 done:
7767 end_synchronized_op(sc, 0);
7768 return (rc);
7769 }
7770
7771 static int
set_sched_queue(struct adapter * sc,struct t4_sched_queue * p)7772 set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
7773 {
7774 struct port_info *pi = NULL;
7775 struct sge_txq *txq;
7776 uint32_t fw_mnem, fw_queue, fw_class;
7777 int i, rc;
7778
7779 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
7780 if (rc)
7781 return (rc);
7782
7783 if (!(sc->flags & FULL_INIT_DONE)) {
7784 rc = EAGAIN;
7785 goto done;
7786 }
7787
7788 if (p->port >= sc->params.nports) {
7789 rc = EINVAL;
7790 goto done;
7791 }
7792
7793 pi = sc->port[p->port];
7794 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) {
7795 rc = EINVAL;
7796 goto done;
7797 }
7798
7799 /*
7800 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
7801 * Scheduling Class in this case).
7802 */
7803 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
7804 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
7805 fw_class = p->cl < 0 ? 0xffffffff : p->cl;
7806
7807 /*
7808 * If op.queue is non-negative, then we're only changing the scheduling
7809 * on a single specified TX queue.
7810 */
7811 if (p->queue >= 0) {
7812 txq = &sc->sge.txq[pi->first_txq + p->queue];
7813 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7814 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7815 &fw_class);
7816 goto done;
7817 }
7818
7819 /*
7820 * Change the scheduling on all the TX queues for the
7821 * interface.
7822 */
7823 for_each_txq(pi, i, txq) {
7824 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
7825 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
7826 &fw_class);
7827 if (rc)
7828 goto done;
7829 }
7830
7831 rc = 0;
7832 done:
7833 end_synchronized_op(sc, 0);
7834 return (rc);
7835 }
7836
7837 int
t4_os_find_pci_capability(struct adapter * sc,int cap)7838 t4_os_find_pci_capability(struct adapter *sc, int cap)
7839 {
7840 int i;
7841
7842 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
7843 }
7844
7845 int
t4_os_pci_save_state(struct adapter * sc)7846 t4_os_pci_save_state(struct adapter *sc)
7847 {
7848 device_t dev;
7849 struct pci_devinfo *dinfo;
7850
7851 dev = sc->dev;
7852 dinfo = device_get_ivars(dev);
7853
7854 pci_cfg_save(dev, dinfo, 0);
7855 return (0);
7856 }
7857
7858 int
t4_os_pci_restore_state(struct adapter * sc)7859 t4_os_pci_restore_state(struct adapter *sc)
7860 {
7861 device_t dev;
7862 struct pci_devinfo *dinfo;
7863
7864 dev = sc->dev;
7865 dinfo = device_get_ivars(dev);
7866
7867 pci_cfg_restore(dev, dinfo);
7868 return (0);
7869 }
7870
7871 void
t4_os_portmod_changed(const struct adapter * sc,int idx)7872 t4_os_portmod_changed(const struct adapter *sc, int idx)
7873 {
7874 struct port_info *pi = sc->port[idx];
7875 static const char *mod_str[] = {
7876 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
7877 };
7878
7879 build_medialist(pi, &pi->media);
7880 #ifdef DEV_NETMAP
7881 build_medialist(pi, &pi->nm_media);
7882 #endif
7883
7884 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
7885 if_printf(pi->ifp, "transceiver unplugged.\n");
7886 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
7887 if_printf(pi->ifp, "unknown transceiver inserted.\n");
7888 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
7889 if_printf(pi->ifp, "unsupported transceiver inserted.\n");
7890 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
7891 if_printf(pi->ifp, "%s transceiver inserted.\n",
7892 mod_str[pi->mod_type]);
7893 } else {
7894 if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
7895 pi->mod_type);
7896 }
7897 }
7898
7899 void
t4_os_link_changed(struct adapter * sc,int idx,int link_stat,int reason)7900 t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason)
7901 {
7902 struct port_info *pi = sc->port[idx];
7903 struct ifnet *ifp = pi->ifp;
7904
7905 if (link_stat) {
7906 pi->linkdnrc = -1;
7907 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
7908 if_link_state_change(ifp, LINK_STATE_UP);
7909 } else {
7910 if (reason >= 0)
7911 pi->linkdnrc = reason;
7912 if_link_state_change(ifp, LINK_STATE_DOWN);
7913 }
7914 }
7915
7916 void
t4_iterate(void (* func)(struct adapter *,void *),void * arg)7917 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
7918 {
7919 struct adapter *sc;
7920
7921 sx_slock(&t4_list_lock);
7922 SLIST_FOREACH(sc, &t4_list, link) {
7923 /*
7924 * func should not make any assumptions about what state sc is
7925 * in - the only guarantee is that sc->sc_lock is a valid lock.
7926 */
7927 func(sc, arg);
7928 }
7929 sx_sunlock(&t4_list_lock);
7930 }
7931
7932 static int
t4_open(struct cdev * dev,int flags,int type,struct thread * td)7933 t4_open(struct cdev *dev, int flags, int type, struct thread *td)
7934 {
7935 return (0);
7936 }
7937
7938 static int
t4_close(struct cdev * dev,int flags,int type,struct thread * td)7939 t4_close(struct cdev *dev, int flags, int type, struct thread *td)
7940 {
7941 return (0);
7942 }
7943
7944 static int
t4_ioctl(struct cdev * dev,unsigned long cmd,caddr_t data,int fflag,struct thread * td)7945 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
7946 struct thread *td)
7947 {
7948 int rc;
7949 struct adapter *sc = dev->si_drv1;
7950
7951 rc = priv_check(td, PRIV_DRIVER);
7952 if (rc != 0)
7953 return (rc);
7954
7955 switch (cmd) {
7956 case CHELSIO_T4_GETREG: {
7957 struct t4_reg *edata = (struct t4_reg *)data;
7958
7959 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7960 return (EFAULT);
7961
7962 if (edata->size == 4)
7963 edata->val = t4_read_reg(sc, edata->addr);
7964 else if (edata->size == 8)
7965 edata->val = t4_read_reg64(sc, edata->addr);
7966 else
7967 return (EINVAL);
7968
7969 break;
7970 }
7971 case CHELSIO_T4_SETREG: {
7972 struct t4_reg *edata = (struct t4_reg *)data;
7973
7974 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
7975 return (EFAULT);
7976
7977 if (edata->size == 4) {
7978 if (edata->val & 0xffffffff00000000)
7979 return (EINVAL);
7980 t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
7981 } else if (edata->size == 8)
7982 t4_write_reg64(sc, edata->addr, edata->val);
7983 else
7984 return (EINVAL);
7985 break;
7986 }
7987 case CHELSIO_T4_REGDUMP: {
7988 struct t4_regdump *regs = (struct t4_regdump *)data;
7989 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE;
7990 uint8_t *buf;
7991
7992 if (regs->len < reglen) {
7993 regs->len = reglen; /* hint to the caller */
7994 return (ENOBUFS);
7995 }
7996
7997 regs->len = reglen;
7998 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
7999 t4_get_regs(sc, regs, buf);
8000 rc = copyout(buf, regs->data, reglen);
8001 free(buf, M_CXGBE);
8002 break;
8003 }
8004 case CHELSIO_T4_GET_FILTER_MODE:
8005 rc = get_filter_mode(sc, (uint32_t *)data);
8006 break;
8007 case CHELSIO_T4_SET_FILTER_MODE:
8008 rc = set_filter_mode(sc, *(uint32_t *)data);
8009 break;
8010 case CHELSIO_T4_GET_FILTER:
8011 rc = get_filter(sc, (struct t4_filter *)data);
8012 break;
8013 case CHELSIO_T4_SET_FILTER:
8014 rc = set_filter(sc, (struct t4_filter *)data);
8015 break;
8016 case CHELSIO_T4_DEL_FILTER:
8017 rc = del_filter(sc, (struct t4_filter *)data);
8018 break;
8019 case CHELSIO_T4_GET_SGE_CONTEXT:
8020 rc = get_sge_context(sc, (struct t4_sge_context *)data);
8021 break;
8022 case CHELSIO_T4_LOAD_FW:
8023 rc = load_fw(sc, (struct t4_data *)data);
8024 break;
8025 case CHELSIO_T4_GET_MEM:
8026 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
8027 break;
8028 case CHELSIO_T4_GET_I2C:
8029 rc = read_i2c(sc, (struct t4_i2c_data *)data);
8030 break;
8031 case CHELSIO_T4_CLEAR_STATS: {
8032 int i;
8033 u_int port_id = *(uint32_t *)data;
8034 struct port_info *pi;
8035
8036 if (port_id >= sc->params.nports)
8037 return (EINVAL);
8038 pi = sc->port[port_id];
8039
8040 /* MAC stats */
8041 t4_clr_port_stats(sc, pi->tx_chan);
8042 pi->tx_parse_error = 0;
8043
8044 if (pi->flags & PORT_INIT_DONE) {
8045 struct sge_rxq *rxq;
8046 struct sge_txq *txq;
8047 struct sge_wrq *wrq;
8048
8049 for_each_rxq(pi, i, rxq) {
8050 #if defined(INET) || defined(INET6)
8051 rxq->lro.lro_queued = 0;
8052 rxq->lro.lro_flushed = 0;
8053 #endif
8054 rxq->rxcsum = 0;
8055 rxq->vlan_extraction = 0;
8056 }
8057
8058 for_each_txq(pi, i, txq) {
8059 txq->txcsum = 0;
8060 txq->tso_wrs = 0;
8061 txq->vlan_insertion = 0;
8062 txq->imm_wrs = 0;
8063 txq->sgl_wrs = 0;
8064 txq->txpkt_wrs = 0;
8065 txq->txpkts0_wrs = 0;
8066 txq->txpkts1_wrs = 0;
8067 txq->txpkts0_pkts = 0;
8068 txq->txpkts1_pkts = 0;
8069 mp_ring_reset_stats(txq->r);
8070 }
8071
8072 #ifdef TCP_OFFLOAD
8073 /* nothing to clear for each ofld_rxq */
8074
8075 for_each_ofld_txq(pi, i, wrq) {
8076 wrq->tx_wrs_direct = 0;
8077 wrq->tx_wrs_copied = 0;
8078 }
8079 #endif
8080 wrq = &sc->sge.ctrlq[pi->port_id];
8081 wrq->tx_wrs_direct = 0;
8082 wrq->tx_wrs_copied = 0;
8083 }
8084 break;
8085 }
8086 case CHELSIO_T4_SCHED_CLASS:
8087 rc = set_sched_class(sc, (struct t4_sched_params *)data);
8088 break;
8089 case CHELSIO_T4_SCHED_QUEUE:
8090 rc = set_sched_queue(sc, (struct t4_sched_queue *)data);
8091 break;
8092 case CHELSIO_T4_GET_TRACER:
8093 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
8094 break;
8095 case CHELSIO_T4_SET_TRACER:
8096 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
8097 break;
8098 default:
8099 rc = EINVAL;
8100 }
8101
8102 return (rc);
8103 }
8104
8105 #ifdef TCP_OFFLOAD
8106 void
t4_iscsi_init(struct ifnet * ifp,unsigned int tag_mask,const unsigned int * pgsz_order)8107 t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask,
8108 const unsigned int *pgsz_order)
8109 {
8110 struct port_info *pi = ifp->if_softc;
8111 struct adapter *sc = pi->adapter;
8112
8113 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
8114 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
8115 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
8116 V_HPZ3(pgsz_order[3]));
8117 }
8118
8119 static int
toe_capability(struct port_info * pi,int enable)8120 toe_capability(struct port_info *pi, int enable)
8121 {
8122 int rc;
8123 struct adapter *sc = pi->adapter;
8124
8125 ASSERT_SYNCHRONIZED_OP(sc);
8126
8127 if (!is_offload(sc))
8128 return (ENODEV);
8129
8130 if (enable) {
8131 /*
8132 * We need the port's queues around so that we're able to send
8133 * and receive CPLs to/from the TOE even if the ifnet for this
8134 * port has never been UP'd administratively.
8135 */
8136 if (!(pi->flags & PORT_INIT_DONE)) {
8137 rc = cxgbe_init_synchronized(pi);
8138 if (rc)
8139 return (rc);
8140 }
8141
8142 if (isset(&sc->offload_map, pi->port_id))
8143 return (0);
8144
8145 if (!uld_active(sc, ULD_TOM)) {
8146 rc = t4_activate_uld(sc, ULD_TOM);
8147 if (rc == EAGAIN) {
8148 log(LOG_WARNING,
8149 "You must kldload t4_tom.ko before trying "
8150 "to enable TOE on a cxgbe interface.\n");
8151 }
8152 if (rc != 0)
8153 return (rc);
8154 KASSERT(sc->tom_softc != NULL,
8155 ("%s: TOM activated but softc NULL", __func__));
8156 KASSERT(uld_active(sc, ULD_TOM),
8157 ("%s: TOM activated but flag not set", __func__));
8158 }
8159
8160 /* Activate iWARP and iSCSI too, if the modules are loaded. */
8161 if (!uld_active(sc, ULD_IWARP))
8162 (void) t4_activate_uld(sc, ULD_IWARP);
8163 if (!uld_active(sc, ULD_ISCSI))
8164 (void) t4_activate_uld(sc, ULD_ISCSI);
8165
8166 setbit(&sc->offload_map, pi->port_id);
8167 } else {
8168 if (!isset(&sc->offload_map, pi->port_id))
8169 return (0);
8170
8171 KASSERT(uld_active(sc, ULD_TOM),
8172 ("%s: TOM never initialized?", __func__));
8173 clrbit(&sc->offload_map, pi->port_id);
8174 }
8175
8176 return (0);
8177 }
8178
8179 /*
8180 * Add an upper layer driver to the global list.
8181 */
8182 int
t4_register_uld(struct uld_info * ui)8183 t4_register_uld(struct uld_info *ui)
8184 {
8185 int rc = 0;
8186 struct uld_info *u;
8187
8188 sx_xlock(&t4_uld_list_lock);
8189 SLIST_FOREACH(u, &t4_uld_list, link) {
8190 if (u->uld_id == ui->uld_id) {
8191 rc = EEXIST;
8192 goto done;
8193 }
8194 }
8195
8196 SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
8197 ui->refcount = 0;
8198 done:
8199 sx_xunlock(&t4_uld_list_lock);
8200 return (rc);
8201 }
8202
8203 int
t4_unregister_uld(struct uld_info * ui)8204 t4_unregister_uld(struct uld_info *ui)
8205 {
8206 int rc = EINVAL;
8207 struct uld_info *u;
8208
8209 sx_xlock(&t4_uld_list_lock);
8210
8211 SLIST_FOREACH(u, &t4_uld_list, link) {
8212 if (u == ui) {
8213 if (ui->refcount > 0) {
8214 rc = EBUSY;
8215 goto done;
8216 }
8217
8218 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
8219 rc = 0;
8220 goto done;
8221 }
8222 }
8223 done:
8224 sx_xunlock(&t4_uld_list_lock);
8225 return (rc);
8226 }
8227
8228 int
t4_activate_uld(struct adapter * sc,int id)8229 t4_activate_uld(struct adapter *sc, int id)
8230 {
8231 int rc;
8232 struct uld_info *ui;
8233
8234 ASSERT_SYNCHRONIZED_OP(sc);
8235
8236 if (id < 0 || id > ULD_MAX)
8237 return (EINVAL);
8238 rc = EAGAIN; /* kldoad the module with this ULD and try again. */
8239
8240 sx_slock(&t4_uld_list_lock);
8241
8242 SLIST_FOREACH(ui, &t4_uld_list, link) {
8243 if (ui->uld_id == id) {
8244 if (!(sc->flags & FULL_INIT_DONE)) {
8245 rc = adapter_full_init(sc);
8246 if (rc != 0)
8247 break;
8248 }
8249
8250 rc = ui->activate(sc);
8251 if (rc == 0) {
8252 setbit(&sc->active_ulds, id);
8253 ui->refcount++;
8254 }
8255 break;
8256 }
8257 }
8258
8259 sx_sunlock(&t4_uld_list_lock);
8260
8261 return (rc);
8262 }
8263
8264 int
t4_deactivate_uld(struct adapter * sc,int id)8265 t4_deactivate_uld(struct adapter *sc, int id)
8266 {
8267 int rc;
8268 struct uld_info *ui;
8269
8270 ASSERT_SYNCHRONIZED_OP(sc);
8271
8272 if (id < 0 || id > ULD_MAX)
8273 return (EINVAL);
8274 rc = ENXIO;
8275
8276 sx_slock(&t4_uld_list_lock);
8277
8278 SLIST_FOREACH(ui, &t4_uld_list, link) {
8279 if (ui->uld_id == id) {
8280 rc = ui->deactivate(sc);
8281 if (rc == 0) {
8282 clrbit(&sc->active_ulds, id);
8283 ui->refcount--;
8284 }
8285 break;
8286 }
8287 }
8288
8289 sx_sunlock(&t4_uld_list_lock);
8290
8291 return (rc);
8292 }
8293
8294 int
uld_active(struct adapter * sc,int uld_id)8295 uld_active(struct adapter *sc, int uld_id)
8296 {
8297
8298 MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
8299
8300 return (isset(&sc->active_ulds, uld_id));
8301 }
8302 #endif
8303
8304 /*
8305 * Come up with reasonable defaults for some of the tunables, provided they're
8306 * not set by the user (in which case we'll use the values as is).
8307 */
8308 static void
tweak_tunables(void)8309 tweak_tunables(void)
8310 {
8311 int nc = mp_ncpus; /* our snapshot of the number of CPUs */
8312
8313 if (t4_ntxq10g < 1)
8314 t4_ntxq10g = min(nc, NTXQ_10G);
8315
8316 if (t4_ntxq1g < 1)
8317 t4_ntxq1g = min(nc, NTXQ_1G);
8318
8319 if (t4_nrxq10g < 1)
8320 t4_nrxq10g = min(nc, NRXQ_10G);
8321
8322 if (t4_nrxq1g < 1)
8323 t4_nrxq1g = min(nc, NRXQ_1G);
8324
8325 #ifdef TCP_OFFLOAD
8326 if (t4_nofldtxq10g < 1)
8327 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
8328
8329 if (t4_nofldtxq1g < 1)
8330 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
8331
8332 if (t4_nofldrxq10g < 1)
8333 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
8334
8335 if (t4_nofldrxq1g < 1)
8336 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
8337
8338 if (t4_toecaps_allowed == -1)
8339 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
8340 #else
8341 if (t4_toecaps_allowed == -1)
8342 t4_toecaps_allowed = 0;
8343 #endif
8344
8345 #ifdef DEV_NETMAP
8346 if (t4_nnmtxq10g < 1)
8347 t4_nnmtxq10g = min(nc, NNMTXQ_10G);
8348
8349 if (t4_nnmtxq1g < 1)
8350 t4_nnmtxq1g = min(nc, NNMTXQ_1G);
8351
8352 if (t4_nnmrxq10g < 1)
8353 t4_nnmrxq10g = min(nc, NNMRXQ_10G);
8354
8355 if (t4_nnmrxq1g < 1)
8356 t4_nnmrxq1g = min(nc, NNMRXQ_1G);
8357 #endif
8358
8359 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
8360 t4_tmr_idx_10g = TMR_IDX_10G;
8361
8362 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
8363 t4_pktc_idx_10g = PKTC_IDX_10G;
8364
8365 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
8366 t4_tmr_idx_1g = TMR_IDX_1G;
8367
8368 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
8369 t4_pktc_idx_1g = PKTC_IDX_1G;
8370
8371 if (t4_qsize_txq < 128)
8372 t4_qsize_txq = 128;
8373
8374 if (t4_qsize_rxq < 128)
8375 t4_qsize_rxq = 128;
8376 while (t4_qsize_rxq & 7)
8377 t4_qsize_rxq++;
8378
8379 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
8380 }
8381
8382 static struct sx mlu; /* mod load unload */
8383 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
8384
8385 static int
mod_event(module_t mod,int cmd,void * arg)8386 mod_event(module_t mod, int cmd, void *arg)
8387 {
8388 int rc = 0;
8389 static int loaded = 0;
8390
8391 switch (cmd) {
8392 case MOD_LOAD:
8393 sx_xlock(&mlu);
8394 if (loaded++ == 0) {
8395 t4_sge_modload();
8396 sx_init(&t4_list_lock, "T4/T5 adapters");
8397 SLIST_INIT(&t4_list);
8398 #ifdef TCP_OFFLOAD
8399 sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
8400 SLIST_INIT(&t4_uld_list);
8401 #endif
8402 t4_tracer_modload();
8403 tweak_tunables();
8404 }
8405 sx_xunlock(&mlu);
8406 break;
8407
8408 case MOD_UNLOAD:
8409 sx_xlock(&mlu);
8410 if (--loaded == 0) {
8411 int tries;
8412
8413 sx_slock(&t4_list_lock);
8414 if (!SLIST_EMPTY(&t4_list)) {
8415 rc = EBUSY;
8416 sx_sunlock(&t4_list_lock);
8417 goto done_unload;
8418 }
8419 #ifdef TCP_OFFLOAD
8420 sx_slock(&t4_uld_list_lock);
8421 if (!SLIST_EMPTY(&t4_uld_list)) {
8422 rc = EBUSY;
8423 sx_sunlock(&t4_uld_list_lock);
8424 sx_sunlock(&t4_list_lock);
8425 goto done_unload;
8426 }
8427 #endif
8428 tries = 0;
8429 while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
8430 uprintf("%ju clusters with custom free routine "
8431 "still is use.\n", t4_sge_extfree_refs());
8432 pause("t4unload", 2 * hz);
8433 }
8434 #ifdef TCP_OFFLOAD
8435 sx_sunlock(&t4_uld_list_lock);
8436 #endif
8437 sx_sunlock(&t4_list_lock);
8438
8439 if (t4_sge_extfree_refs() == 0) {
8440 t4_tracer_modunload();
8441 #ifdef TCP_OFFLOAD
8442 sx_destroy(&t4_uld_list_lock);
8443 #endif
8444 sx_destroy(&t4_list_lock);
8445 t4_sge_modunload();
8446 loaded = 0;
8447 } else {
8448 rc = EBUSY;
8449 loaded++; /* undo earlier decrement */
8450 }
8451 }
8452 done_unload:
8453 sx_xunlock(&mlu);
8454 break;
8455 }
8456
8457 return (rc);
8458 }
8459
8460 static devclass_t t4_devclass, t5_devclass;
8461 static devclass_t cxgbe_devclass, cxl_devclass;
8462
8463 DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0);
8464 MODULE_VERSION(t4nex, 1);
8465 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
8466
8467 DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0);
8468 MODULE_VERSION(t5nex, 1);
8469 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
8470
8471 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
8472 MODULE_VERSION(cxgbe, 1);
8473
8474 DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0);
8475 MODULE_VERSION(cxl, 1);
8476