1 /*-
2 * Copyright (c) 2014-2015, Matthew Macy <mmacy@nextbsd.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 *
11 * 2. Neither the name of Matthew Macy nor the names of its
12 * contributors may be used to endorse or promote products derived from
13 * this software without specific prior written permission.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
19 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
20 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
21 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
22 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
24 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
25 * POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include <sys/param.h>
29 #include <sys/types.h>
30 #include <sys/bus.h>
31 #include <sys/eventhandler.h>
32 #include <sys/sockio.h>
33 #include <sys/kernel.h>
34 #include <sys/lock.h>
35 #include <sys/mutex.h>
36 #include <sys/module.h>
37 #include <sys/kobj.h>
38 #include <sys/rman.h>
39 #include <sys/smp.h>
40 #include <sys/socket.h>
41 #include <sys/sysctl.h>
42 #include <sys/syslog.h>
43 #include <sys/taskqueue.h>
44
45
46 #include <net/if.h>
47 #include <net/if_var.h>
48 #include <net/if_types.h>
49 #include <net/if_media.h>
50 #include <net/bpf.h>
51 #include <net/ethernet.h>
52 #include <net/mp_ring.h>
53
54 #include <netinet/in.h>
55 #include <netinet/in_pcb.h>
56 #include <netinet/tcp_lro.h>
57
58 #include <machine/bus.h>
59
60 #include <vm/vm.h>
61 #include <vm/pmap.h>
62
63 #include <dev/led/led.h>
64 #include <dev/pci/pcireg.h>
65 #include <dev/pci/pcivar.h>
66
67 #include <net/iflib.h>
68
69 #include "ifdi_if.h"
70
71 /*
72 * File organization:
73 * - private structures
74 * - iflib private utility functions
75 * - ifnet functions
76 * - vlan registry and other exported functions
77 * - iflib public core functions
78 *
79 *
80 * Next steps:
81 *
82 * - validate queue teardown
83 * - validate that all structure fields are initialized
84
85 * - add SW RSS to demux received data packets to buf_rings for deferred processing
86 * look at handling tx ack processing
87 *
88 */
89 static MALLOC_DEFINE(M_IFLIB, "iflib", "ifnet library");
90
91 struct iflib_txq;
92 typedef struct iflib_txq *iflib_txq_t;
93 struct iflib_rxq;
94 typedef struct iflib_rxq *iflib_rxq_t;
95 struct iflib_qset;
96 typedef struct iflib_qset *iflib_qset_t;
97 struct iflib_fl;
98 typedef struct iflib_fl *iflib_fl_t;
99
100 typedef struct iflib_filter_info {
101 driver_filter_t *ifi_filter;
102 void *ifi_filter_arg;
103 struct grouptask *ifi_task;
104 } *iflib_filter_info_t;
105
106 struct iflib_ctx {
107 KOBJ_FIELDS;
108
109 /*
110 * Pointer to hardware driver's softc
111 */
112
113 void *ifc_softc;
114 device_t ifc_dev;
115 if_t ifc_ifp;
116
117 cpuset_t ifc_cpus;
118 if_shared_ctx_t ifc_sctx;
119
120 struct mtx ifc_mtx;
121 char ifc_mtx_name[16];
122 iflib_txq_t ifc_txqs;
123 iflib_rxq_t ifc_rxqs;
124 iflib_qset_t ifc_qsets;
125 uint32_t ifc_if_flags;
126 uint32_t ifc_flags;
127 int ifc_in_detach;
128
129 int ifc_link_state;
130 int ifc_link_irq;
131 eventhandler_tag ifc_vlan_attach_event;
132 eventhandler_tag ifc_vlan_detach_event;
133 int ifc_pause_frames;
134 int ifc_watchdog_events;
135 uint8_t ifc_mac[ETHER_ADDR_LEN];
136 struct cdev *ifc_led_dev;
137 struct resource *ifc_msix_mem;
138
139 struct if_irq ifc_legacy_irq;
140 struct grouptask ifc_admin_task;
141 struct grouptask ifc_vflr_task;
142 struct iflib_filter_info ifc_filter_info;
143 struct ifmedia ifc_media;
144
145 struct if_txrx ifc_txrx;
146 #define isc_txd_encap ifc_txrx.ift_txd_encap
147 #define isc_txd_flush ifc_txrx.ift_txd_flush
148 #define isc_txd_credits_update ifc_txrx.ift_txd_credits_update
149 #define isc_rxd_available ifc_txrx.ift_rxd_available
150 #define isc_rxd_pkt_get ifc_txrx.ift_rxd_pkt_get
151 #define isc_rxd_refill ifc_txrx.ift_rxd_refill
152 #define isc_rxd_flush ifc_txrx.ift_rxd_flush
153 #define isc_rxd_refill ifc_txrx.ift_rxd_refill
154 #define isc_rxd_refill ifc_txrx.ift_rxd_refill
155 #define isc_legacy_intr ifc_txrx.ift_legacy_intr
156 struct if_softc_ctx ifc_softc_ctx;
157 };
158
159
160 void *
iflib_get_softc(if_ctx_t ctx)161 iflib_get_softc(if_ctx_t ctx)
162 {
163
164 return (ctx->ifc_softc);
165 }
166
167 device_t
iflib_get_dev(if_ctx_t ctx)168 iflib_get_dev(if_ctx_t ctx)
169 {
170
171 return (ctx->ifc_dev);
172 }
173
174 if_t
iflib_get_ifp(if_ctx_t ctx)175 iflib_get_ifp(if_ctx_t ctx)
176 {
177
178 return (ctx->ifc_ifp);
179 }
180
181 struct ifmedia *
iflib_get_media(if_ctx_t ctx)182 iflib_get_media(if_ctx_t ctx)
183 {
184
185 return (&ctx->ifc_media);
186 }
187
188 void
iflib_set_mac(if_ctx_t ctx,uint8_t mac[ETHER_ADDR_LEN])189 iflib_set_mac(if_ctx_t ctx, uint8_t mac[ETHER_ADDR_LEN])
190 {
191
192 bcopy(mac, ctx->ifc_mac, ETHER_ADDR_LEN);
193 }
194
195 if_softc_ctx_t
iflib_get_softc_ctx(if_ctx_t ctx)196 iflib_get_softc_ctx(if_ctx_t ctx)
197 {
198
199 return (&ctx->ifc_softc_ctx);
200 }
201
202 if_shared_ctx_t
iflib_get_sctx(if_ctx_t ctx)203 iflib_get_sctx(if_ctx_t ctx)
204 {
205
206 return (ctx->ifc_sctx);
207 }
208
209
210 #define LINK_ACTIVE(ctx) ((ctx)->ifc_link_state == LINK_STATE_UP)
211
212 typedef struct iflib_dma_info {
213 bus_addr_t idi_paddr;
214 caddr_t idi_vaddr;
215 bus_dma_tag_t idi_tag;
216 bus_dmamap_t idi_map;
217 bus_dma_segment_t idi_seg;
218 int idi_nseg;
219 uint32_t idi_size;
220 } *iflib_dma_info_t;
221
222 struct iflib_qset {
223 iflib_dma_info_t ifq_ifdi;
224 uint16_t ifq_nhwqs;
225 };
226
227 #define RX_SW_DESC_MAP_CREATED (1 << 0)
228 #define TX_SW_DESC_MAP_CREATED (1 << 1)
229 #define RX_SW_DESC_INUSE (1 << 3)
230 #define TX_SW_DESC_MAPPED (1 << 4)
231
232 typedef struct iflib_sw_desc {
233 bus_dmamap_t ifsd_map; /* bus_dma map for packet */
234 struct mbuf *ifsd_m; /* rx: uninitialized mbuf
235 * tx: pkthdr for the packet
236 */
237 caddr_t ifsd_cl; /* direct cluster pointer for rx */
238 int ifsd_flags;
239
240 struct mbuf *ifsd_mh;
241 struct mbuf *ifsd_mt;
242 } *iflib_sd_t;
243
244 /* magic number that should be high enough for any hardware */
245 #define IFLIB_MAX_TX_SEGS 128
246 #define IFLIB_RX_COPY_THRESH 128
247 #define IFLIB_QUEUE_IDLE 0
248 #define IFLIB_QUEUE_HUNG 1
249 #define IFLIB_QUEUE_WORKING 2
250
251 #define IFLIB_BUDGET 64
252 #define IFLIB_RESTART_BUDGET 8
253
254 #define IFC_LEGACY 0x1
255 #define IFC_QFLUSH 0x2
256
257 struct iflib_txq {
258 if_ctx_t ift_ctx;
259 uint64_t ift_flags;
260 uint32_t ift_in_use;
261 uint32_t ift_size;
262 uint32_t ift_processed; /* need to have device tx interrupt update this with credits */
263 uint32_t ift_cleaned;
264 uint32_t ift_stop_thres;
265 uint32_t ift_cidx;
266 uint32_t ift_cidx_processed;
267 uint32_t ift_pidx;
268 uint32_t ift_gen;
269 uint32_t ift_db_pending;
270 uint32_t ift_npending;
271 uint32_t ift_tqid;
272 uint64_t ift_tx_direct_packets;
273 uint64_t ift_tx_direct_bytes;
274 uint64_t ift_no_tx_dma_setup;
275 uint64_t ift_no_desc_avail;
276 uint64_t ift_mbuf_defrag_failed;
277 uint64_t ift_tx_irq;
278 bus_dma_tag_t ift_desc_tag;
279 bus_dma_segment_t ift_segs[IFLIB_MAX_TX_SEGS];
280 struct callout ift_timer;
281 struct callout ift_db_check;
282
283 struct mtx ift_mtx;
284 #define MTX_NAME_LEN 16
285 char ift_mtx_name[MTX_NAME_LEN];
286 #define BATCH_SIZE 32
287 struct mbuf *ift_mp[BATCH_SIZE];
288 int ift_id;
289 iflib_sd_t ift_sds;
290 int ift_nbr;
291 struct mp_ring **ift_br;
292 struct grouptask ift_task;
293 int ift_qstatus;
294 int ift_active;
295 int ift_watchdog_time;
296 struct iflib_filter_info ift_filter_info;
297 iflib_dma_info_t ift_ifdi;
298 int ift_closed;
299 };
300
301 struct iflib_fl {
302 uint32_t ifl_cidx;
303 uint32_t ifl_pidx;
304 uint32_t ifl_gen;
305 uint32_t ifl_size;
306 uint32_t ifl_credits;
307 uint32_t ifl_buf_size;
308 int ifl_cltype;
309 uma_zone_t ifl_zone;
310
311 iflib_sd_t ifl_sds;
312 iflib_rxq_t ifl_rxq;
313 uint8_t ifl_id;
314 iflib_dma_info_t ifl_ifdi;
315 uint64_t ifl_phys_addrs[256];
316 caddr_t ifl_vm_addrs[256];
317 };
318
319 static inline int
get_inuse(int size,int cidx,int pidx,int gen)320 get_inuse(int size, int cidx, int pidx, int gen)
321 {
322 int used;
323
324 if (pidx > cidx)
325 used = pidx - cidx;
326 else if (pidx < cidx)
327 used = size - cidx + pidx;
328 else if (gen == 0 && pidx == cidx)
329 used = 0;
330 else if (gen == 1 && pidx == cidx)
331 used = size;
332 else
333 panic("bad state");
334
335 return (used);
336 }
337
338 #define TXQ_AVAIL(txq) (txq->ift_size - get_inuse(txq->ift_size, txq->ift_cidx, txq->ift_pidx, txq->ift_gen))
339
340 #define IDXDIFF(head, tail, wrap) \
341 ((head) >= (tail) ? (head) - (tail) : (wrap) - (tail) + (head))
342
343 typedef struct iflib_global_context {
344 struct taskqgroup *igc_io_tqg; /* per-cpu taskqueues for io */
345 struct taskqgroup *igc_config_tqg; /* taskqueue for config operations */
346 } iflib_global_context_t;
347
348 struct iflib_global_context global_ctx, *gctx;
349
350 struct iflib_rxq {
351 if_ctx_t ifr_ctx;
352 uint32_t ifr_size;
353 uint32_t ifr_cidx;
354 uint32_t ifr_pidx; /* if there is a separate completion queue -
355 * these are the cq cidx and pidx otherwise
356 * these are unused
357 */
358 uint32_t ifr_gen;
359 uint64_t ifr_rx_irq;
360 uint16_t ifr_id;
361 int ifr_lro_enabled;
362 iflib_fl_t ifr_fl;
363 uint8_t ifr_nfl;
364 struct lro_ctrl ifr_lc;
365 struct mtx ifr_mtx;
366 char ifr_mtx_name[MTX_NAME_LEN];
367 struct grouptask ifr_task;
368 bus_dma_tag_t ifr_desc_tag;
369 iflib_dma_info_t ifr_ifdi;
370 struct iflib_filter_info ifr_filter_info;
371 };
372
373
374 static int enable_msix = 1;
375
376 #define mtx_held(m) (((m)->mtx_lock & ~MTX_FLAGMASK) != (uintptr_t)0)
377
378
379 #define CTX_ACTIVE(ctx) ((if_getdrvflags((ctx)->ifc_ifp) & IFF_DRV_RUNNING))
380
381 #define CTX_LOCK_INIT(_sc, _name) mtx_init(&(_sc)->ifc_mtx, _name, "iflib ctx lock", MTX_DEF)
382
383 #define CTX_LOCK(ctx) mtx_lock(&(ctx)->ifc_mtx)
384 #define CTX_UNLOCK(ctx) mtx_unlock(&(ctx)->ifc_mtx)
385 #define CTX_LOCK_DESTROY(ctx) mtx_destroy(&(ctx)->ifc_mtx)
386
387
388 #define TX_LOCK(txq) mtx_lock(&txq->ift_mtx)
389 #define TX_TRY_LOCK(txq) mtx_trylock(&txq->ift_mtx)
390 #define TX_UNLOCK(txq) mtx_unlock(&txq->ift_mtx)
391
392
393 /* Our boot-time initialization hook */
394 static int iflib_module_event_handler(module_t, int, void *);
395
396 static moduledata_t iflib_moduledata = {
397 "iflib",
398 iflib_module_event_handler,
399 NULL
400 };
401
402 DECLARE_MODULE(iflib, iflib_moduledata, SI_SUB_SMP, SI_ORDER_ANY);
403 MODULE_VERSION(iflib, 1);
404
405 MODULE_DEPEND(iflib, pci, 1, 1, 1);
406 MODULE_DEPEND(iflib, ether, 1, 1, 1);
407
408 TASKQGROUP_DEFINE(if_io_tqg, mp_ncpus, 1);
409 TASKQGROUP_DEFINE(if_config_tqg, 1, 1);
410
411 #ifndef IFLIB_DEBUG_COUNTERS
412 #ifdef INVARIANTS
413 #define IFLIB_DEBUG_COUNTERS 1
414 #else
415 #define IFLIB_DEBUG_COUNTERS 0
416 #endif /* !INVARIANTS */
417 #endif
418
419 static SYSCTL_NODE(_net, OID_AUTO, iflib, CTLFLAG_RD, 0,
420 "iflib driver parameters");
421
422 static int iflib_min_tx_latency;
423
424 SYSCTL_INT(_net_iflib, OID_AUTO, min_tx_latency, CTLFLAG_RW,
425 &iflib_min_tx_latency, 0, "minimize transmit latency at the possibel expense of throughput");
426
427
428 #if IFLIB_DEBUG_COUNTERS
429
430 static int iflib_tx_seen;
431 static int iflib_tx_sent;
432 static int iflib_tx_encap;
433 static int iflib_rx_allocs;
434 static int iflib_fl_refills;
435 static int iflib_fl_refills_large;
436 static int iflib_tx_frees;
437
438 SYSCTL_INT(_net_iflib, OID_AUTO, tx_seen, CTLFLAG_RD,
439 &iflib_tx_seen, 0, "# tx mbufs seen");
440 SYSCTL_INT(_net_iflib, OID_AUTO, tx_sent, CTLFLAG_RD,
441 &iflib_tx_sent, 0, "# tx mbufs sent");
442 SYSCTL_INT(_net_iflib, OID_AUTO, tx_encap, CTLFLAG_RD,
443 &iflib_tx_encap, 0, "# tx mbufs encapped");
444 SYSCTL_INT(_net_iflib, OID_AUTO, tx_frees, CTLFLAG_RD,
445 &iflib_tx_frees, 0, "# tx frees");
446 SYSCTL_INT(_net_iflib, OID_AUTO, rx_allocs, CTLFLAG_RD,
447 &iflib_rx_allocs, 0, "# rx allocations");
448 SYSCTL_INT(_net_iflib, OID_AUTO, fl_refills, CTLFLAG_RD,
449 &iflib_fl_refills, 0, "# refills");
450 SYSCTL_INT(_net_iflib, OID_AUTO, fl_refills_large, CTLFLAG_RD,
451 &iflib_fl_refills_large, 0, "# large refills");
452
453
454 static int iflib_txq_drain_flushing;
455 static int iflib_txq_drain_oactive;
456 static int iflib_txq_drain_notready;
457 static int iflib_txq_drain_encapfail;
458
459 SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_flushing, CTLFLAG_RD,
460 &iflib_txq_drain_flushing, 0, "# drain flushes");
461 SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_oactive, CTLFLAG_RD,
462 &iflib_txq_drain_oactive, 0, "# drain oactives");
463 SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_notready, CTLFLAG_RD,
464 &iflib_txq_drain_notready, 0, "# drain notready");
465 SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_encapfail, CTLFLAG_RD,
466 &iflib_txq_drain_encapfail, 0, "# drain encap fails");
467
468
469 static int iflib_encap_load_mbuf_fail;
470 static int iflib_encap_txq_avail_fail;
471 static int iflib_encap_txd_encap_fail;
472
473 SYSCTL_INT(_net_iflib, OID_AUTO, encap_load_mbuf_fail, CTLFLAG_RD,
474 &iflib_encap_load_mbuf_fail, 0, "# busdma load failures");
475 SYSCTL_INT(_net_iflib, OID_AUTO, encap_txq_avail_fail, CTLFLAG_RD,
476 &iflib_encap_txq_avail_fail, 0, "# txq avail failures");
477 SYSCTL_INT(_net_iflib, OID_AUTO, encap_txd_encap_fail, CTLFLAG_RD,
478 &iflib_encap_txd_encap_fail, 0, "# driver encap failures");
479
480 static int iflib_task_fn_rxs;
481 static int iflib_rx_intr_enables;
482 static int iflib_fast_intrs;
483 static int iflib_intr_link;
484 static int iflib_intr_msix;
485 static int iflib_rx_unavail;
486 static int iflib_rx_ctx_inactive;
487 static int iflib_rx_zero_len;
488 static int iflib_rx_if_input;
489 static int iflib_rx_mbuf_null;
490 static int iflib_rxd_flush;
491
492 SYSCTL_INT(_net_iflib, OID_AUTO, intr_link, CTLFLAG_RD,
493 &iflib_intr_link, 0, "# intr link calls");
494 SYSCTL_INT(_net_iflib, OID_AUTO, intr_msix, CTLFLAG_RD,
495 &iflib_intr_msix, 0, "# intr msix calls");
496 SYSCTL_INT(_net_iflib, OID_AUTO, task_fn_rx, CTLFLAG_RD,
497 &iflib_task_fn_rxs, 0, "# task_fn_rx calls");
498 SYSCTL_INT(_net_iflib, OID_AUTO, rx_intr_enables, CTLFLAG_RD,
499 &iflib_rx_intr_enables, 0, "# rx intr enables");
500 SYSCTL_INT(_net_iflib, OID_AUTO, fast_intrs, CTLFLAG_RD,
501 &iflib_fast_intrs, 0, "# fast_intr calls");
502 SYSCTL_INT(_net_iflib, OID_AUTO, rx_unavail, CTLFLAG_RD,
503 &iflib_rx_unavail, 0, "# times rxeof called with no available data");
504 SYSCTL_INT(_net_iflib, OID_AUTO, rx_ctx_inactive, CTLFLAG_RD,
505 &iflib_rx_ctx_inactive, 0, "# times rxeof called with inactive context");
506 SYSCTL_INT(_net_iflib, OID_AUTO, rx_zero_len, CTLFLAG_RD,
507 &iflib_rx_zero_len, 0, "# times rxeof saw zero len mbuf");
508 SYSCTL_INT(_net_iflib, OID_AUTO, rx_if_input, CTLFLAG_RD,
509 &iflib_rx_if_input, 0, "# times rxeof called if_input");
510 SYSCTL_INT(_net_iflib, OID_AUTO, rx_mbuf_null, CTLFLAG_RD,
511 &iflib_rx_mbuf_null, 0, "# times rxeof got null mbuf");
512 SYSCTL_INT(_net_iflib, OID_AUTO, rxd_flush, CTLFLAG_RD,
513 &iflib_rxd_flush, 0, "# times rxd_flush called");
514
515 #define DBG_COUNTER_INC(name) atomic_add_int(&(iflib_ ## name), 1)
516 #else
517 #define DBG_COUNTER_INC(name)
518 #endif
519
520 #define IFLIB_DEBUG 0
521
522 static void iflib_tx_structures_free(if_ctx_t ctx);
523 static void iflib_rx_structures_free(if_ctx_t ctx);
524 static int iflib_queues_alloc(if_ctx_t ctx);
525 static int iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq);
526 static int iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, int cidx);
527 static int iflib_qset_structures_setup(if_ctx_t ctx);
528 static int iflib_msix_init(if_ctx_t ctx);
529 static int iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filterarg, int *rid, char *str);
530 static void iflib_txq_check_drain(iflib_txq_t txq, int budget);
531 static uint32_t iflib_txq_can_drain(struct mp_ring *);
532 static int iflib_register(if_ctx_t);
533
534
535 #if IFLIB_DEBUG
536 static void *
if_dbg_malloc(unsigned long size,struct malloc_type * type,int flags)537 if_dbg_malloc(unsigned long size, struct malloc_type *type, int flags)
538 {
539 caddr_t p, ptmp;
540 char buf[4] = {0, 0, 0, 0};
541 int i;
542
543 ptmp = p = malloc(size, type, flags);
544
545 if ((flags & M_ZERO) == 0)
546 return (p);
547
548 for (i = 0; i < size; i += 4, ptmp += 4) {
549 if (bcmp(buf, ptmp, 4) != 0)
550 panic("received non-zero memory from malloc");
551 }
552 return (p);
553 }
554
555 #define malloc if_dbg_malloc
556 #endif
557
558
559
560 #ifdef DEV_NETMAP
561 #include <sys/selinfo.h>
562 #include <net/netmap.h>
563 #include <dev/netmap/netmap_kern.h>
564
565 MODULE_DEPEND(iflib, netmap, 1, 1, 1);
566
567 /*
568 * device-specific sysctl variables:
569 *
570 * ixl_crcstrip: 0: keep CRC in rx frames (default), 1: strip it.
571 * During regular operations the CRC is stripped, but on some
572 * hardware reception of frames not multiple of 64 is slower,
573 * so using crcstrip=0 helps in benchmarks.
574 *
575 * ixl_rx_miss, ixl_rx_miss_bufs:
576 * count packets that might be missed due to lost interrupts.
577 */
578 SYSCTL_DECL(_dev_netmap);
579 /*
580 * The xl driver by default strips CRCs and we do not override it.
581 */
582 int ixl_rx_miss, ixl_rx_miss_bufs, ixl_crcstrip = 1;
583 #if 0
584 SYSCTL_INT(_dev_netmap, OID_AUTO, ixl_crcstrip,
585 CTLFLAG_RW, &ixl_crcstrip, 1, "strip CRC on rx frames");
586 #endif
587 SYSCTL_INT(_dev_netmap, OID_AUTO, ixl_rx_miss,
588 CTLFLAG_RW, &ixl_rx_miss, 0, "potentially missed rx intr");
589 SYSCTL_INT(_dev_netmap, OID_AUTO, ixl_rx_miss_bufs,
590 CTLFLAG_RW, &ixl_rx_miss_bufs, 0, "potentially missed rx intr bufs");
591
592
593 /*
594 * Register/unregister. We are already under netmap lock.
595 * Only called on the first register or the last unregister.
596 */
597 static int
iflib_netmap_register(struct netmap_adapter * na,int onoff)598 iflib_netmap_register(struct netmap_adapter *na, int onoff)
599 {
600 struct ifnet *ifp = na->ifp;
601 if_ctx_t ctx = ifp->if_softc;
602
603 CTX_LOCK(ctx);
604 IFDI_INTR_DISABLE(ctx);
605
606 /* Tell the stack that the interface is no longer active */
607 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
608
609 //set_crcstrip(&adapter->hw, onoff);
610 /* enable or disable flags and callbacks in na and ifp */
611 if (onoff) {
612 nm_set_native_flags(na);
613 } else {
614 nm_clear_native_flags(na);
615 }
616 IFDI_INIT(ctx);
617 //set_crcstrip(&adapter->hw, onoff); // XXX why twice ?
618 CTX_UNLOCK(ctx);
619 return (ifp->if_drv_flags & IFF_DRV_RUNNING ? 0 : 1);
620 }
621
622 /*
623 * Reconcile kernel and user view of the transmit ring.
624 *
625 * All information is in the kring.
626 * Userspace wants to send packets up to the one before kring->rhead,
627 * kernel knows kring->nr_hwcur is the first unsent packet.
628 *
629 * Here we push packets out (as many as possible), and possibly
630 * reclaim buffers from previously completed transmission.
631 *
632 * The caller (netmap) guarantees that there is only one instance
633 * running at any time. Any interference with other driver
634 * methods should be handled by the individual drivers.
635 */
636 static int
iflib_netmap_txsync(struct netmap_kring * kring,int flags)637 iflib_netmap_txsync(struct netmap_kring *kring, int flags)
638 {
639 struct netmap_adapter *na = kring->na;
640 struct ifnet *ifp = na->ifp;
641 struct netmap_ring *ring = kring->ring;
642 u_int nm_i; /* index into the netmap ring */
643 u_int nic_i; /* index into the NIC ring */
644 u_int n;
645 u_int const lim = kring->nkr_num_slots - 1;
646 u_int const head = kring->rhead;
647 struct if_pkt_info pi;
648
649 #ifdef notyet
650 /* XXX need to pass in pi */
651 /*
652 * interrupts on every tx packet are expensive so request
653 * them every half ring, or where NS_REPORT is set
654 */
655 u_int report_frequency = kring->nkr_num_slots >> 1;
656 #endif
657 /* device-specific */
658 if_ctx_t ctx = ifp->if_softc;
659 iflib_txq_t txq = &ctx->ifc_txqs[kring->ring_id];
660
661 pi.ipi_m = NULL;
662 pi.ipi_segs = txq->ift_segs;
663 pi.ipi_qsidx = kring->ring_id;
664 pi.ipi_ndescs = 0;
665
666 bus_dmamap_sync(txq->ift_desc_tag, txq->ift_ifdi->idi_map,
667 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
668
669
670 /*
671 * First part: process new packets to send.
672 * nm_i is the current index in the netmap ring,
673 * nic_i is the corresponding index in the NIC ring.
674 *
675 * If we have packets to send (nm_i != head)
676 * iterate over the netmap ring, fetch length and update
677 * the corresponding slot in the NIC ring. Some drivers also
678 * need to update the buffer's physical address in the NIC slot
679 * even NS_BUF_CHANGED is not set (PNMB computes the addresses).
680 *
681 * The netmap_reload_map() calls is especially expensive,
682 * even when (as in this case) the tag is 0, so do only
683 * when the buffer has actually changed.
684 *
685 * If possible do not set the report/intr bit on all slots,
686 * but only a few times per ring or when NS_REPORT is set.
687 *
688 * Finally, on 10G and faster drivers, it might be useful
689 * to prefetch the next slot and txr entry.
690 */
691
692 nm_i = kring->nr_hwcur;
693 if (nm_i != head) { /* we have new packets to send */
694 nic_i = netmap_idx_k2n(kring, nm_i);
695
696 __builtin_prefetch(&ring->slot[nm_i]);
697 __builtin_prefetch(&txq->ift_sds[nic_i]);
698
699 for (n = 0; nm_i != head; n++) {
700 struct netmap_slot *slot = &ring->slot[nm_i];
701 u_int len = slot->len;
702 uint64_t paddr;
703 void *addr = PNMB(na, slot, &paddr);
704
705 /* device-specific */
706 pi.ipi_pidx = nic_i;
707 ctx->isc_txd_encap(ctx->ifc_softc, &pi);
708 /* prefetch for next round */
709 __builtin_prefetch(&ring->slot[nm_i + 1]);
710 __builtin_prefetch(&txq->ift_sds[nic_i + 1]);
711
712 NM_CHECK_ADDR_LEN(na, addr, len);
713
714 if (slot->flags & NS_BUF_CHANGED) {
715 /* buffer has changed, reload map */
716 netmap_reload_map(na, txq->ift_desc_tag, txq->ift_sds[nic_i].ifsd_map, addr);
717 }
718 slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED);
719
720 /* make sure changes to the buffer are synced */
721 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_sds[nic_i].ifsd_map,
722 BUS_DMASYNC_PREWRITE);
723
724 nm_i = nm_next(nm_i, lim);
725 nic_i = nm_next(nic_i, lim);
726 }
727 kring->nr_hwcur = head;
728 bus_dmamap_sync(txq->ift_desc_tag, txq->ift_ifdi->idi_map,
729 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
730
731 /* (re)start the tx unit up to slot nic_i (excluded) */
732 ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, nic_i);
733 }
734
735 /*
736 * Second part: reclaim buffers for completed transmissions.
737 */
738 if (iflib_tx_credits_update(ctx, txq)) {
739 /* some tx completed, increment avail */
740 nic_i = txq->ift_cidx_processed;
741 kring->nr_hwtail = nm_prev(netmap_idx_n2k(kring, nic_i), lim);
742 }
743 return (0);
744 }
745
746 /*
747 * Reconcile kernel and user view of the receive ring.
748 * Same as for the txsync, this routine must be efficient.
749 * The caller guarantees a single invocations, but races against
750 * the rest of the driver should be handled here.
751 *
752 * On call, kring->rhead is the first packet that userspace wants
753 * to keep, and kring->rcur is the wakeup point.
754 * The kernel has previously reported packets up to kring->rtail.
755 *
756 * If (flags & NAF_FORCE_READ) also check for incoming packets irrespective
757 * of whether or not we received an interrupt.
758 */
759 static int
iflib_netmap_rxsync(struct netmap_kring * kring,int flags)760 iflib_netmap_rxsync(struct netmap_kring *kring, int flags)
761 {
762 struct netmap_adapter *na = kring->na;
763 struct ifnet *ifp = na->ifp;
764 struct netmap_ring *ring = kring->ring;
765 u_int nm_i; /* index into the netmap ring */
766 u_int nic_i; /* index into the NIC ring */
767 u_int i, n;
768 u_int const lim = kring->nkr_num_slots - 1;
769 u_int const head = kring->rhead;
770 int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;
771 struct if_rxd_info ri;
772 /* device-specific */
773 if_ctx_t ctx = ifp->if_softc;
774 iflib_rxq_t rxq = &ctx->ifc_rxqs[kring->ring_id];
775 iflib_fl_t fl = rxq->ifr_fl;
776 if (head > lim)
777 return netmap_ring_reinit(kring);
778
779 bzero(&ri, sizeof(ri));
780 ri.iri_qsidx = kring->ring_id;
781 ri.iri_ifp = ctx->ifc_ifp;
782 /* XXX check sync modes */
783 for (i = 0, fl = rxq->ifr_fl; i < rxq->ifr_nfl; i++, fl++)
784 bus_dmamap_sync(rxq->ifr_desc_tag, fl->ifl_ifdi->idi_map,
785 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
786
787 /*
788 * First part: import newly received packets.
789 *
790 * nm_i is the index of the next free slot in the netmap ring,
791 * nic_i is the index of the next received packet in the NIC ring,
792 * and they may differ in case if_init() has been called while
793 * in netmap mode. For the receive ring we have
794 *
795 * nic_i = rxr->next_check;
796 * nm_i = kring->nr_hwtail (previous)
797 * and
798 * nm_i == (nic_i + kring->nkr_hwofs) % ring_size
799 *
800 * rxr->next_check is set to 0 on a ring reinit
801 */
802 if (netmap_no_pendintr || force_update) {
803 #if 0
804 int crclen = ixl_crcstrip ? 0 : 4;
805 #endif
806 int error, avail;
807 uint16_t slot_flags = kring->nkr_slot_flags;
808
809 for (fl = rxq->ifr_fl, i = 0; i < rxq->ifr_nfl; i++, fl++) {
810 nic_i = fl->ifl_cidx;
811 nm_i = netmap_idx_n2k(kring, nic_i);
812 avail = ctx->isc_rxd_available(ctx->ifc_softc, kring->ring_id, nic_i);
813 for (n = 0; avail > 0; n++, avail--) {
814 error = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri);
815 if (error)
816 ring->slot[nm_i].len = 0;
817 else
818 ring->slot[nm_i].len = ri.iri_len;
819 ring->slot[nm_i].flags = slot_flags;
820 bus_dmamap_sync(fl->ifl_ifdi->idi_tag,
821 fl->ifl_sds[nic_i].ifsd_map, BUS_DMASYNC_POSTREAD);
822 nm_i = nm_next(nm_i, lim);
823 nic_i = nm_next(nic_i, lim);
824 }
825 if (n) { /* update the state variables */
826 #if 0
827 if (netmap_no_pendintr && !force_update) {
828 /* diagnostics */
829 ixl_rx_miss ++;
830 ixl_rx_miss_bufs += n;
831 }
832 #endif
833 fl->ifl_cidx = nic_i;
834 kring->nr_hwtail = nm_i;
835 }
836 kring->nr_kflags &= ~NKR_PENDINTR;
837 }
838 }
839 /*
840 * Second part: skip past packets that userspace has released.
841 * (kring->nr_hwcur to head excluded),
842 * and make the buffers available for reception.
843 * As usual nm_i is the index in the netmap ring,
844 * nic_i is the index in the NIC ring, and
845 * nm_i == (nic_i + kring->nkr_hwofs) % ring_size
846 */
847 /* XXX not sure how this will work with multiple free lists */
848 nm_i = kring->nr_hwcur;
849 if (nm_i != head) {
850 nic_i = netmap_idx_k2n(kring, nm_i);
851 for (n = 0; nm_i != head; n++) {
852 struct netmap_slot *slot = &ring->slot[nm_i];
853 uint64_t paddr;
854 caddr_t vaddr;
855 void *addr = PNMB(na, slot, &paddr);
856
857 if (addr == NETMAP_BUF_BASE(na)) /* bad buf */
858 goto ring_reset;
859
860 vaddr = addr;
861 if (slot->flags & NS_BUF_CHANGED) {
862 /* buffer has changed, reload map */
863 netmap_reload_map(na, fl->ifl_ifdi->idi_tag, fl->ifl_sds[nic_i].ifsd_map, addr);
864 slot->flags &= ~NS_BUF_CHANGED;
865 }
866 /*
867 * XXX we should be batching this operation - TODO
868 */
869 ctx->isc_rxd_refill(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, nic_i, &paddr, &vaddr, 1);
870 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_sds[nic_i].ifsd_map,
871 BUS_DMASYNC_PREREAD);
872 nm_i = nm_next(nm_i, lim);
873 nic_i = nm_next(nic_i, lim);
874 }
875 kring->nr_hwcur = head;
876
877 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
878 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
879 /*
880 * IMPORTANT: we must leave one free slot in the ring,
881 * so move nic_i back by one unit
882 */
883 nic_i = nm_prev(nic_i, lim);
884 ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, nic_i);
885 }
886
887 return 0;
888
889 ring_reset:
890 return netmap_ring_reinit(kring);
891 }
892
893 static int
iflib_netmap_attach(if_ctx_t ctx)894 iflib_netmap_attach(if_ctx_t ctx)
895 {
896 struct netmap_adapter na;
897
898 bzero(&na, sizeof(na));
899
900 na.ifp = ctx->ifc_ifp;
901 na.na_flags = NAF_BDG_MAYSLEEP;
902 MPASS(ctx->ifc_softc_ctx.isc_nqsets);
903
904 na.num_tx_desc = ctx->ifc_sctx->isc_ntxd;
905 na.num_rx_desc = ctx->ifc_sctx->isc_ntxd;
906 na.nm_txsync = iflib_netmap_txsync;
907 na.nm_rxsync = iflib_netmap_rxsync;
908 na.nm_register = iflib_netmap_register;
909 na.num_tx_rings = na.num_rx_rings = ctx->ifc_softc_ctx.isc_nqsets;
910 return (netmap_attach(&na));
911 }
912
913 static void
iflib_netmap_txq_init(if_ctx_t ctx,iflib_txq_t txq)914 iflib_netmap_txq_init(if_ctx_t ctx, iflib_txq_t txq)
915 {
916 struct netmap_adapter *na = NA(ctx->ifc_ifp);
917 struct netmap_slot *slot;
918 iflib_sd_t sd;
919
920 slot = netmap_reset(na, NR_TX, txq->ift_id, 0);
921 if (slot == 0)
922 return;
923
924 sd = txq->ift_sds;
925 for (int i = 0; i < ctx->ifc_sctx->isc_ntxd; i++, sd++) {
926
927 /*
928 * In netmap mode, set the map for the packet buffer.
929 * NOTE: Some drivers (not this one) also need to set
930 * the physical buffer address in the NIC ring.
931 * netmap_idx_n2k() maps a nic index, i, into the corresponding
932 * netmap slot index, si
933 */
934 int si = netmap_idx_n2k(&na->tx_rings[txq->ift_id], i);
935 netmap_load_map(na, txq->ift_desc_tag, sd->ifsd_map, NMB(na, slot + si));
936 }
937 }
938 static void
iflib_netmap_rxq_init(if_ctx_t ctx,iflib_rxq_t rxq)939 iflib_netmap_rxq_init(if_ctx_t ctx, iflib_rxq_t rxq)
940 {
941 struct netmap_adapter *na = NA(ctx->ifc_ifp);
942 struct netmap_slot *slot;
943 iflib_sd_t sd;
944 int nrxd;
945
946 slot = netmap_reset(na, NR_RX, rxq->ifr_id, 0);
947 if (slot == 0)
948 return;
949 sd = rxq->ifr_fl[0].ifl_sds;
950 nrxd = ctx->ifc_sctx->isc_nrxd;
951 for (int i = 0; i < nrxd; i++, sd++) {
952 int sj = netmap_idx_n2k(&na->rx_rings[rxq->ifr_id], i);
953 uint64_t paddr;
954 void *addr;
955 caddr_t vaddr;
956
957 vaddr = addr = PNMB(na, slot + sj, &paddr);
958 netmap_load_map(na, rxq->ifr_fl[0].ifl_ifdi->idi_tag, sd->ifsd_map, addr);
959 /* Update descriptor and the cached value */
960 ctx->isc_rxd_refill(ctx->ifc_softc, rxq->ifr_id, 0 /* fl_id */, i, &paddr, &vaddr, 1);
961 }
962 /* preserve queue */
963 if (ctx->ifc_ifp->if_capenable & IFCAP_NETMAP) {
964 struct netmap_kring *kring = &na->rx_rings[rxq->ifr_id];
965 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
966 ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, 0 /* fl_id */, t);
967 } else
968 ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, 0 /* fl_id */, nrxd-1);
969 }
970
971 #define iflib_netmap_detach(ifp) netmap_detach(ifp)
972
973 #else
974 #define iflib_netmap_txq_init(ctx, txq)
975 #define iflib_netmap_rxq_init(ctx, rxq)
976 #define iflib_netmap_detach(ifp)
977
978 #define iflib_netmap_attach(ctx) (0)
979 #define netmap_rx_irq(ifp, qid, budget) (0)
980
981 #endif
982
983 #if defined(__i386__) || defined(__amd64__)
984 static __inline void
prefetch(void * x)985 prefetch(void *x)
986 {
987 __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x));
988 }
989 #else
990 #define prefetch(x)
991 #endif
992
993 static void
_iflib_dmamap_cb(void * arg,bus_dma_segment_t * segs,int nseg,int err)994 _iflib_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err)
995 {
996 if (err)
997 return;
998 *(bus_addr_t *) arg = segs[0].ds_addr;
999 }
1000
1001 static int
iflib_dma_alloc(if_ctx_t ctx,bus_size_t size,iflib_dma_info_t dma,int mapflags)1002 iflib_dma_alloc(if_ctx_t ctx, bus_size_t size, iflib_dma_info_t dma,
1003 int mapflags)
1004 {
1005 int err;
1006 if_shared_ctx_t sctx = ctx->ifc_sctx;
1007 device_t dev = ctx->ifc_dev;
1008
1009 KASSERT(sctx->isc_q_align != 0, ("alignment value not initialized"));
1010
1011 err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
1012 sctx->isc_q_align, 0, /* alignment, bounds */
1013 BUS_SPACE_MAXADDR, /* lowaddr */
1014 BUS_SPACE_MAXADDR, /* highaddr */
1015 NULL, NULL, /* filter, filterarg */
1016 size, /* maxsize */
1017 1, /* nsegments */
1018 size, /* maxsegsize */
1019 BUS_DMA_ALLOCNOW, /* flags */
1020 NULL, /* lockfunc */
1021 NULL, /* lockarg */
1022 &dma->idi_tag);
1023 if (err) {
1024 device_printf(dev,
1025 "%s: bus_dma_tag_create failed: %d\n",
1026 __func__, err);
1027 goto fail_0;
1028 }
1029
1030 err = bus_dmamem_alloc(dma->idi_tag, (void**) &dma->idi_vaddr,
1031 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->idi_map);
1032 if (err) {
1033 device_printf(dev,
1034 "%s: bus_dmamem_alloc(%ju) failed: %d\n",
1035 __func__, (uintmax_t)size, err);
1036 goto fail_2;
1037 }
1038
1039 dma->idi_paddr = 0;
1040 err = bus_dmamap_load(dma->idi_tag, dma->idi_map, dma->idi_vaddr,
1041 size, _iflib_dmamap_cb, &dma->idi_paddr, mapflags | BUS_DMA_NOWAIT);
1042 if (err || dma->idi_paddr == 0) {
1043 device_printf(dev,
1044 "%s: bus_dmamap_load failed: %d\n",
1045 __func__, err);
1046 goto fail_3;
1047 }
1048
1049 dma->idi_size = size;
1050 return (0);
1051
1052 fail_3:
1053 bus_dmamap_unload(dma->idi_tag, dma->idi_map);
1054 fail_2:
1055 bus_dmamem_free(dma->idi_tag, dma->idi_vaddr, dma->idi_map);
1056 bus_dma_tag_destroy(dma->idi_tag);
1057 fail_0:
1058 dma->idi_tag = NULL;
1059
1060 return (err);
1061 }
1062
1063 static void
iflib_dma_free(iflib_dma_info_t dma)1064 iflib_dma_free(iflib_dma_info_t dma)
1065 {
1066 if (dma->idi_tag == NULL)
1067 return;
1068 if (dma->idi_paddr != 0) {
1069 bus_dmamap_sync(dma->idi_tag, dma->idi_map,
1070 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1071 bus_dmamap_unload(dma->idi_tag, dma->idi_map);
1072 dma->idi_paddr = 0;
1073 }
1074 if (dma->idi_vaddr != NULL) {
1075 bus_dmamem_free(dma->idi_tag, dma->idi_vaddr, dma->idi_map);
1076 dma->idi_vaddr = NULL;
1077 }
1078 bus_dma_tag_destroy(dma->idi_tag);
1079 dma->idi_tag = NULL;
1080 }
1081
1082 static int
iflib_fast_intr(void * arg)1083 iflib_fast_intr(void *arg)
1084 {
1085 iflib_filter_info_t info = arg;
1086 struct grouptask *gtask = info->ifi_task;
1087
1088 DBG_COUNTER_INC(fast_intrs);
1089 if (info->ifi_filter != NULL && info->ifi_filter(info->ifi_filter_arg) == FILTER_HANDLED)
1090 return (FILTER_HANDLED);
1091
1092 GROUPTASK_ENQUEUE(gtask);
1093 return (FILTER_HANDLED);
1094 }
1095
1096 static int
_iflib_irq_alloc(if_ctx_t ctx,if_irq_t irq,int rid,driver_filter_t filter,driver_intr_t handler,void * arg,char * name)1097 _iflib_irq_alloc(if_ctx_t ctx, if_irq_t irq, int rid,
1098 driver_filter_t filter, driver_intr_t handler, void *arg,
1099 char *name)
1100 {
1101 int rc;
1102 struct resource *res;
1103 void *tag;
1104 device_t dev = ctx->ifc_dev;
1105
1106 MPASS(rid < 512);
1107 irq->ii_rid = rid;
1108 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &irq->ii_rid,
1109 RF_SHAREABLE | RF_ACTIVE);
1110 if (res == NULL) {
1111 device_printf(dev,
1112 "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
1113 return (ENOMEM);
1114 }
1115 irq->ii_res = res;
1116 KASSERT(filter == NULL || handler == NULL, ("filter and handler can't both be non-NULL"));
1117 rc = bus_setup_intr(dev, res, INTR_MPSAFE | INTR_TYPE_NET,
1118 filter, handler, arg, &tag);
1119 if (rc != 0) {
1120 device_printf(dev,
1121 "failed to setup interrupt for rid %d, name %s: %d\n",
1122 rid, name ? name : "unknown", rc);
1123 return (rc);
1124 } else if (name)
1125 bus_describe_intr(dev, res, tag, name);
1126
1127 irq->ii_tag = tag;
1128 return (0);
1129 }
1130
1131
1132 /*********************************************************************
1133 *
1134 * Allocate memory for tx_buffer structures. The tx_buffer stores all
1135 * the information needed to transmit a packet on the wire. This is
1136 * called only once at attach, setup is done every reset.
1137 *
1138 **********************************************************************/
1139
1140 static int
iflib_txsd_alloc(iflib_txq_t txq)1141 iflib_txsd_alloc(iflib_txq_t txq)
1142 {
1143 if_ctx_t ctx = txq->ift_ctx;
1144 if_shared_ctx_t sctx = ctx->ifc_sctx;
1145 device_t dev = ctx->ifc_dev;
1146 iflib_sd_t txsd;
1147 int err, i, nsegments;
1148
1149 nsegments = ctx->ifc_softc_ctx.isc_tx_nsegments;
1150 MPASS(sctx->isc_ntxd > 0);
1151 MPASS(nsegments > 0);
1152 /*
1153 * Setup DMA descriptor areas.
1154 */
1155 if ((err = bus_dma_tag_create(bus_get_dma_tag(dev),
1156 1, 0, /* alignment, bounds */
1157 BUS_SPACE_MAXADDR, /* lowaddr */
1158 BUS_SPACE_MAXADDR, /* highaddr */
1159 NULL, NULL, /* filter, filterarg */
1160 sctx->isc_tx_maxsize, /* maxsize */
1161 nsegments, /* nsegments */
1162 sctx->isc_tx_maxsegsize, /* maxsegsize */
1163 0, /* flags */
1164 NULL, /* lockfunc */
1165 NULL, /* lockfuncarg */
1166 &txq->ift_desc_tag))) {
1167 device_printf(dev,"Unable to allocate TX DMA tag: %d\n", err);
1168 device_printf(dev,"maxsize: %ld nsegments: %d maxsegsize: %ld\n",
1169 sctx->isc_tx_maxsize, nsegments, sctx->isc_tx_maxsegsize);
1170 goto fail;
1171 }
1172
1173 if (!(txq->ift_sds =
1174 (iflib_sd_t) malloc(sizeof(struct iflib_sw_desc) *
1175 sctx->isc_ntxd, M_IFLIB, M_NOWAIT | M_ZERO))) {
1176 device_printf(dev, "Unable to allocate tx_buffer memory\n");
1177 err = ENOMEM;
1178 goto fail;
1179 }
1180
1181 /* Create the descriptor buffer dma maps */
1182 txsd = txq->ift_sds;
1183 for (i = 0; i < sctx->isc_ntxd; i++, txsd++) {
1184 err = bus_dmamap_create(txq->ift_desc_tag, 0, &txsd->ifsd_map);
1185 if (err != 0) {
1186 device_printf(dev, "Unable to create TX DMA map\n");
1187 goto fail;
1188 }
1189 }
1190
1191 return 0;
1192 fail:
1193 /* We free all, it handles case where we are in the middle */
1194 iflib_tx_structures_free(ctx);
1195 return (err);
1196 }
1197
1198 /*
1199 * XXX Review tx cleaning and buffer mapping
1200 *
1201 */
1202
1203 static void
iflib_txsd_destroy(if_ctx_t ctx,iflib_txq_t txq,iflib_sd_t txsd)1204 iflib_txsd_destroy(if_ctx_t ctx, iflib_txq_t txq, iflib_sd_t txsd)
1205 {
1206 if (txsd->ifsd_m != NULL) {
1207 if (txsd->ifsd_map != NULL) {
1208 bus_dmamap_destroy(txq->ift_desc_tag, txsd->ifsd_map);
1209 txsd->ifsd_map = NULL;
1210 }
1211 } else if (txsd->ifsd_map != NULL) {
1212 bus_dmamap_unload(txq->ift_desc_tag,
1213 txsd->ifsd_map);
1214 bus_dmamap_destroy(txq->ift_desc_tag,
1215 txsd->ifsd_map);
1216 txsd->ifsd_map = NULL;
1217 }
1218 }
1219
1220 static void
iflib_txq_destroy(iflib_txq_t txq)1221 iflib_txq_destroy(iflib_txq_t txq)
1222 {
1223 if_ctx_t ctx = txq->ift_ctx;
1224 if_shared_ctx_t sctx = ctx->ifc_sctx;
1225 iflib_sd_t sd = txq->ift_sds;
1226
1227 for (int i = 0; i < sctx->isc_ntxd; i++, sd++)
1228 iflib_txsd_destroy(ctx, txq, sd);
1229 if (txq->ift_sds != NULL) {
1230 free(txq->ift_sds, M_IFLIB);
1231 txq->ift_sds = NULL;
1232 }
1233 if (txq->ift_desc_tag != NULL) {
1234 bus_dma_tag_destroy(txq->ift_desc_tag);
1235 txq->ift_desc_tag = NULL;
1236 }
1237 }
1238
1239 static void
iflib_txsd_free(if_ctx_t ctx,iflib_txq_t txq,iflib_sd_t txsd)1240 iflib_txsd_free(if_ctx_t ctx, iflib_txq_t txq, iflib_sd_t txsd)
1241 {
1242 if (txsd->ifsd_m == NULL)
1243 return;
1244 bus_dmamap_sync(txq->ift_desc_tag,
1245 txsd->ifsd_map,
1246 BUS_DMASYNC_POSTWRITE);
1247 bus_dmamap_unload(txq->ift_desc_tag,
1248 txsd->ifsd_map);
1249 m_freem(txsd->ifsd_m);
1250 DBG_COUNTER_INC(tx_frees);
1251 txsd->ifsd_m = NULL;
1252 }
1253
1254 static int
iflib_txq_setup(iflib_txq_t txq)1255 iflib_txq_setup(iflib_txq_t txq)
1256 {
1257 if_ctx_t ctx = txq->ift_ctx;
1258 if_shared_ctx_t sctx = ctx->ifc_sctx;
1259 iflib_qset_t qset = &ctx->ifc_qsets[txq->ift_id];
1260 iflib_sd_t txsd;
1261 iflib_dma_info_t di;
1262 int i;
1263
1264 /* Set number of descriptors available */
1265 txq->ift_qstatus = IFLIB_QUEUE_IDLE;
1266
1267 /* Reset indices */
1268 txq->ift_cidx_processed = txq->ift_pidx = txq->ift_cidx = txq->ift_npending = 0;
1269 txq->ift_size = sctx->isc_ntxd;
1270
1271 /* Free any existing tx buffers. */
1272 txsd = txq->ift_sds;
1273 for (int i = 0; i < sctx->isc_ntxd; i++, txsd++) {
1274 iflib_txsd_free(ctx, txq, txsd);
1275 }
1276 for (i = 0, di = qset->ifq_ifdi; i < qset->ifq_nhwqs; i++, di++)
1277 bzero((void *)di->idi_vaddr, di->idi_size);
1278
1279 IFDI_TXQ_SETUP(ctx, txq->ift_id);
1280 for (i = 0, di = qset->ifq_ifdi; i < qset->ifq_nhwqs; i++, di++)
1281 bus_dmamap_sync(di->idi_tag, di->idi_map,
1282 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1283 return (0);
1284 }
1285
1286 /*********************************************************************
1287 *
1288 * Allocate memory for rx_buffer structures. Since we use one
1289 * rx_buffer per received packet, the maximum number of rx_buffer's
1290 * that we'll need is equal to the number of receive descriptors
1291 * that we've allocated.
1292 *
1293 **********************************************************************/
1294 static int
iflib_rxsd_alloc(iflib_rxq_t rxq)1295 iflib_rxsd_alloc(iflib_rxq_t rxq)
1296 {
1297 if_ctx_t ctx = rxq->ifr_ctx;
1298 if_shared_ctx_t sctx = ctx->ifc_sctx;
1299 device_t dev = ctx->ifc_dev;
1300 iflib_fl_t fl;
1301 iflib_sd_t rxsd;
1302 int err;
1303
1304 MPASS(sctx->isc_nrxd > 0);
1305
1306 fl = rxq->ifr_fl;
1307 for (int i = 0; i < rxq->ifr_nfl; i++, fl++) {
1308 fl->ifl_sds = malloc(sizeof(struct iflib_sw_desc) *
1309 sctx->isc_nrxd, M_IFLIB, M_WAITOK | M_ZERO);
1310 if (fl->ifl_sds == NULL) {
1311 device_printf(dev, "Unable to allocate rx sw desc memory\n");
1312 return (ENOMEM);
1313 }
1314 fl->ifl_size = sctx->isc_nrxd; /* this isn't necessarily the same */
1315 err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
1316 1, 0, /* alignment, bounds */
1317 BUS_SPACE_MAXADDR, /* lowaddr */
1318 BUS_SPACE_MAXADDR, /* highaddr */
1319 NULL, NULL, /* filter, filterarg */
1320 sctx->isc_rx_maxsize, /* maxsize */
1321 sctx->isc_rx_nsegments, /* nsegments */
1322 sctx->isc_rx_maxsegsize, /* maxsegsize */
1323 0, /* flags */
1324 NULL, /* lockfunc */
1325 NULL, /* lockarg */
1326 &rxq->ifr_desc_tag);
1327 if (err) {
1328 device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
1329 __func__, err);
1330 goto fail;
1331 }
1332
1333 rxsd = fl->ifl_sds;
1334 for (int i = 0; i < sctx->isc_nrxd; i++, rxsd++) {
1335 err = bus_dmamap_create(rxq->ifr_desc_tag, 0, &rxsd->ifsd_map);
1336 if (err) {
1337 device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
1338 __func__, err);
1339 goto fail;
1340 }
1341 }
1342 }
1343 return (0);
1344
1345 fail:
1346 iflib_rx_structures_free(ctx);
1347 return (err);
1348 }
1349
1350 /**
1351 * rxq_refill - refill an rxq free-buffer list
1352 * @ctx: the iflib context
1353 * @rxq: the free-list to refill
1354 * @n: the number of new buffers to allocate
1355 *
1356 * (Re)populate an rxq free-buffer list with up to @n new packet buffers.
1357 * The caller must assure that @n does not exceed the queue's capacity.
1358 */
1359 static void
_iflib_fl_refill(if_ctx_t ctx,iflib_fl_t fl,int count)1360 _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count)
1361 {
1362 struct mbuf *m;
1363 int pidx = fl->ifl_pidx;
1364 iflib_sd_t rxsd = &fl->ifl_sds[pidx];
1365 caddr_t cl;
1366 int n, i = 0;
1367 uint64_t phys_addr;
1368
1369 n = count;
1370 MPASS(n > 0);
1371 MPASS(fl->ifl_credits >= 0);
1372 MPASS(fl->ifl_credits + n <= fl->ifl_size);
1373 #ifdef INVARIANTS
1374 if (pidx < fl->ifl_cidx)
1375 MPASS(pidx + n <= fl->ifl_cidx);
1376 if (pidx == fl->ifl_cidx && (fl->ifl_credits < fl->ifl_size))
1377 MPASS(fl->ifl_gen == 0);
1378 if (pidx > fl->ifl_cidx)
1379 MPASS(n <= fl->ifl_size - pidx + fl->ifl_cidx);
1380 #endif
1381 DBG_COUNTER_INC(fl_refills);
1382 if (n > 8)
1383 DBG_COUNTER_INC(fl_refills_large);
1384
1385 while (n--) {
1386 /*
1387 * We allocate an uninitialized mbuf + cluster, mbuf is
1388 * initialized after rx.
1389 *
1390 * If the cluster is still set then we know a minimum sized packet was received
1391 */
1392 if ((cl = rxsd->ifsd_cl) == NULL &&
1393 (cl = rxsd->ifsd_cl = m_cljget(NULL, M_NOWAIT, fl->ifl_buf_size)) == NULL)
1394 break;
1395 if ((m = m_gethdr(M_NOWAIT, MT_NOINIT)) == NULL) {
1396 break;
1397 }
1398 DBG_COUNTER_INC(rx_allocs);
1399 #ifdef notyet
1400 if ((rxsd->ifsd_flags & RX_SW_DESC_MAP_CREATED) == 0) {
1401 int err;
1402
1403 if ((err = bus_dmamap_create(fl->ifl_ifdi->idi_tag, 0, &rxsd->ifsd_map))) {
1404 log(LOG_WARNING, "bus_dmamap_create failed %d\n", err);
1405 uma_zfree(fl->ifl_zone, cl);
1406 n = 0;
1407 goto done;
1408 }
1409 rxsd->ifsd_flags |= RX_SW_DESC_MAP_CREATED;
1410 }
1411 #endif
1412 #if !defined(__i386__) && !defined(__amd64__)
1413 {
1414 struct refill_rxq_cb_arg cb_arg;
1415 cb_arg.error = 0;
1416 err = bus_dmamap_load(q->ifr_desc_tag, sd->ifsd_map,
1417 cl, q->ifr_buf_size, refill_rxq_cb, &cb_arg, 0);
1418
1419 if (err != 0 || cb_arg.error) {
1420 /*
1421 * !zone_pack ?
1422 */
1423 if (q->zone == zone_pack)
1424 uma_zfree(q->ifr_zone, cl);
1425 m_free(m);
1426 n = 0;
1427 goto done;
1428 }
1429 phys_addr = cb_arg.seg.ds_addr;
1430 }
1431 #else
1432 phys_addr = pmap_kextract((vm_offset_t)cl);
1433 #endif
1434 rxsd->ifsd_flags |= RX_SW_DESC_INUSE;
1435
1436 MPASS(rxsd->ifsd_m == NULL);
1437 rxsd->ifsd_cl = cl;
1438 rxsd->ifsd_m = m;
1439 fl->ifl_phys_addrs[i] = phys_addr;
1440 fl->ifl_vm_addrs[i] = cl;
1441 rxsd++;
1442 fl->ifl_credits++;
1443 i++;
1444 MPASS(fl->ifl_credits <= fl->ifl_size);
1445 if (++fl->ifl_pidx == fl->ifl_size) {
1446 fl->ifl_pidx = 0;
1447 fl->ifl_gen = 1;
1448 rxsd = fl->ifl_sds;
1449 }
1450 if (n == 0 || i == 256) {
1451 ctx->isc_rxd_refill(ctx->ifc_softc, fl->ifl_rxq->ifr_id, fl->ifl_id, pidx,
1452 fl->ifl_phys_addrs, fl->ifl_vm_addrs, i);
1453 i = 0;
1454 pidx = fl->ifl_pidx;
1455 }
1456 }
1457 #if !defined(__i386__) && !defined(__amd64__)
1458 done:
1459 #endif
1460 DBG_COUNTER_INC(rxd_flush);
1461 if (fl->ifl_pidx == 0)
1462 pidx = fl->ifl_size - 1;
1463 else
1464 pidx = fl->ifl_pidx - 1;
1465 ctx->isc_rxd_flush(ctx->ifc_softc, fl->ifl_rxq->ifr_id, fl->ifl_id, pidx);
1466 }
1467
1468 static __inline void
__iflib_fl_refill_lt(if_ctx_t ctx,iflib_fl_t fl,int max)1469 __iflib_fl_refill_lt(if_ctx_t ctx, iflib_fl_t fl, int max)
1470 {
1471 /* we avoid allowing pidx to catch up with cidx as it confuses ixl */
1472 int32_t reclaimable = fl->ifl_size - fl->ifl_credits - 1;
1473 #ifdef INVARIANTS
1474 int32_t delta = fl->ifl_size - get_inuse(fl->ifl_size, fl->ifl_cidx, fl->ifl_pidx, fl->ifl_gen) - 1;
1475
1476 MPASS(fl->ifl_credits <= fl->ifl_size);
1477 MPASS(reclaimable == delta);
1478 #endif
1479 if (reclaimable > 0)
1480 _iflib_fl_refill(ctx, fl, min(max, reclaimable));
1481 }
1482
1483 static void
iflib_fl_bufs_free(iflib_fl_t fl)1484 iflib_fl_bufs_free(iflib_fl_t fl)
1485 {
1486 uint32_t cidx = fl->ifl_cidx;
1487 iflib_dma_info_t idi = fl->ifl_rxq->ifr_ifdi;
1488
1489 MPASS(fl->ifl_credits >= 0);
1490 while (fl->ifl_credits) {
1491 iflib_sd_t d = &fl->ifl_sds[cidx];
1492
1493 if (d->ifsd_flags & RX_SW_DESC_INUSE) {
1494 bus_dmamap_unload(fl->ifl_rxq->ifr_desc_tag, d->ifsd_map);
1495 bus_dmamap_destroy(fl->ifl_rxq->ifr_desc_tag, d->ifsd_map);
1496 if (d->ifsd_m != NULL) {
1497 m_init(d->ifsd_m, zone_mbuf, MLEN,
1498 M_NOWAIT, MT_DATA, 0);
1499 uma_zfree(zone_mbuf, d->ifsd_m);
1500 }
1501 if (d->ifsd_cl != NULL)
1502 uma_zfree(fl->ifl_zone, d->ifsd_cl);
1503 }
1504 #ifdef INVARIANTS
1505 else {
1506 MPASS(d->ifsd_cl == NULL);
1507 MPASS(d->ifsd_m == NULL);
1508 }
1509 #endif
1510 d->ifsd_cl = NULL;
1511 d->ifsd_m = NULL;
1512 if (++cidx == fl->ifl_size)
1513 cidx = 0;
1514 fl->ifl_credits--;
1515 }
1516 /*
1517 * Reset free list values
1518 */
1519 fl->ifl_pidx = 0;
1520 fl->ifl_cidx = 0;
1521 fl->ifl_gen = 0;
1522 bzero(idi->idi_vaddr, idi->idi_size);
1523 }
1524
1525 /*********************************************************************
1526 *
1527 * Initialize a receive ring and its buffers.
1528 *
1529 **********************************************************************/
1530 static int
iflib_fl_setup(iflib_fl_t fl)1531 iflib_fl_setup(iflib_fl_t fl)
1532 {
1533 iflib_rxq_t rxq = fl->ifl_rxq;
1534 if_ctx_t ctx = rxq->ifr_ctx;
1535 if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
1536 int err = 0;
1537 /*
1538 * XXX don't set the max_frame_size to larger
1539 * than the hardware can handle
1540 */
1541 if (sctx->isc_max_frame_size <= 2048)
1542 fl->ifl_buf_size = MCLBYTES;
1543 else if (sctx->isc_max_frame_size <= 4096)
1544 fl->ifl_buf_size = MJUMPAGESIZE;
1545 else if (sctx->isc_max_frame_size <= 9216)
1546 fl->ifl_buf_size = MJUM9BYTES;
1547 else
1548 fl->ifl_buf_size = MJUM16BYTES;
1549 fl->ifl_cltype = m_gettype(fl->ifl_buf_size);
1550 fl->ifl_zone = m_getzone(fl->ifl_buf_size);
1551
1552 /*
1553 ** Free current RX buffer structs and their mbufs
1554 */
1555 iflib_fl_bufs_free(fl);
1556
1557 /* Now replenish the mbufs */
1558 MPASS(fl->ifl_credits == 0);
1559 #if 0
1560 _iflib_fl_refill(ctx, fl, fl->ifl_size);
1561 MPASS(fl->ifl_pidx == 0);
1562 MPASS(fl->ifl_size == fl->ifl_credits);
1563 MPASS(fl->ifl_gen == 1);
1564 #endif
1565 /* avoid pre-allocating zillions of clusters to an idle card
1566 * potentially speeding up attach
1567 */
1568 _iflib_fl_refill(ctx, fl, min(128, fl->ifl_size));
1569 MPASS(min(128, fl->ifl_size) == fl->ifl_credits);
1570 /*
1571 * handle failure
1572 */
1573 MPASS(rxq != NULL);
1574 MPASS(rxq->ifr_ifdi != NULL);
1575 bus_dmamap_sync(rxq->ifr_ifdi->idi_tag, rxq->ifr_ifdi->idi_map,
1576 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1577 return (err);
1578 }
1579
1580 /*********************************************************************
1581 *
1582 * Free receive ring data structures
1583 *
1584 **********************************************************************/
1585 static void
iflib_rx_sds_free(iflib_rxq_t rxq)1586 iflib_rx_sds_free(iflib_rxq_t rxq)
1587 {
1588
1589 if (rxq->ifr_fl != NULL) {
1590 if (rxq->ifr_fl->ifl_sds != NULL)
1591 free(rxq->ifr_fl->ifl_sds, M_IFLIB);
1592
1593 free(rxq->ifr_fl, M_IFLIB);
1594 rxq->ifr_fl = NULL;
1595 rxq->ifr_gen = rxq->ifr_cidx = rxq->ifr_pidx = 0;
1596 }
1597
1598 if (rxq->ifr_desc_tag != NULL) {
1599 bus_dma_tag_destroy(rxq->ifr_desc_tag);
1600 rxq->ifr_desc_tag = NULL;
1601 }
1602 }
1603
1604 /*
1605 * MI independent logic
1606 *
1607 */
1608 static void
iflib_timer(void * arg)1609 iflib_timer(void *arg)
1610 {
1611 iflib_txq_t txq = arg;
1612 if_ctx_t ctx = txq->ift_ctx;
1613 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
1614 if_t ifp = ctx->ifc_ifp;
1615
1616 /*
1617 ** Check on the state of the TX queue(s), this
1618 ** can be done without the lock because its RO
1619 ** and the HUNG state will be static if set.
1620 */
1621 IFDI_TIMER(ctx, txq->ift_id);
1622 if ((txq->ift_qstatus == IFLIB_QUEUE_HUNG) &&
1623 (ctx->ifc_pause_frames == 0))
1624 goto hung;
1625
1626 if (TXQ_AVAIL(txq) <= scctx->isc_tx_nsegments)
1627 GROUPTASK_ENQUEUE(&txq->ift_task);
1628
1629 ctx->ifc_pause_frames = 0;
1630 callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, txq->ift_timer.c_cpu);
1631 return;
1632 hung:
1633 CTX_LOCK(ctx);
1634 if_setdrvflagbits(ctx->ifc_ifp, 0, IFF_DRV_RUNNING);
1635 device_printf(ctx->ifc_dev, "TX(%d) desc avail = %d, pidx = %d\n",
1636 txq->ift_id, TXQ_AVAIL(txq), txq->ift_pidx);
1637
1638 IFDI_WATCHDOG_RESET(ctx);
1639 ctx->ifc_watchdog_events++;
1640 ctx->ifc_pause_frames = 0;
1641
1642 /* Set hardware offload abilities */
1643 if_clearhwassist(ifp);
1644 if (if_getcapenable(ifp) & IFCAP_TXCSUM)
1645 if_sethwassistbits(ifp, CSUM_TCP | CSUM_UDP, 0);
1646 if (if_getcapenable(ifp) & IFCAP_TSO4)
1647 if_sethwassistbits(ifp, CSUM_TSO, 0);
1648 if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6)
1649 if_sethwassistbits(ifp, (CSUM_TCP_IPV6 | CSUM_UDP_IPV6), 0);
1650
1651 IFDI_INIT(ctx);
1652 CTX_UNLOCK(ctx);
1653 }
1654
1655 static void
iflib_init_locked(if_ctx_t ctx)1656 iflib_init_locked(if_ctx_t ctx)
1657 {
1658 if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
1659 iflib_fl_t fl;
1660 iflib_txq_t txq;
1661 iflib_rxq_t rxq;
1662 int i, j;
1663
1664 if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
1665 IFDI_INTR_DISABLE(ctx);
1666 for (i = 0, txq = ctx->ifc_txqs, rxq = ctx->ifc_rxqs; i < sctx->isc_nqsets; i++, txq++, rxq++) {
1667 TX_LOCK(txq);
1668 callout_stop(&txq->ift_timer);
1669 callout_stop(&txq->ift_db_check);
1670 TX_UNLOCK(txq);
1671 iflib_netmap_txq_init(ctx, txq);
1672 iflib_netmap_rxq_init(ctx, rxq);
1673 }
1674
1675 IFDI_INIT(ctx);
1676 for (i = 0, rxq = ctx->ifc_rxqs; i < sctx->isc_nqsets; i++, rxq++) {
1677 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) {
1678 if (iflib_fl_setup(fl)) {
1679 device_printf(ctx->ifc_dev, "freelist setup failed - check cluster settings\n");
1680 goto done;
1681 }
1682 }
1683 }
1684 done:
1685 if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
1686 IFDI_INTR_ENABLE(ctx);
1687 txq = ctx->ifc_txqs;
1688 for (i = 0; i < sctx->isc_nqsets; i++, txq++)
1689 callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq,
1690 txq->ift_timer.c_cpu);
1691 }
1692
1693 #define FLOG printf("%s called\n", __FUNCTION__)
1694
1695 static int
iflib_media_change(if_t ifp)1696 iflib_media_change(if_t ifp)
1697 {
1698 if_ctx_t ctx = if_getsoftc(ifp);
1699 int err;
1700
1701 CTX_LOCK(ctx);
1702 if ((err = IFDI_MEDIA_CHANGE(ctx)) == 0)
1703 iflib_init_locked(ctx);
1704 CTX_UNLOCK(ctx);
1705 return (err);
1706 }
1707
1708 static void
iflib_media_status(if_t ifp,struct ifmediareq * ifmr)1709 iflib_media_status(if_t ifp, struct ifmediareq *ifmr)
1710 {
1711 if_ctx_t ctx = if_getsoftc(ifp);
1712
1713 CTX_LOCK(ctx);
1714 IFDI_UPDATE_ADMIN_STATUS(ctx);
1715 IFDI_MEDIA_STATUS(ctx, ifmr);
1716 CTX_UNLOCK(ctx);
1717 }
1718
1719 static void
iflib_stop(if_ctx_t ctx)1720 iflib_stop(if_ctx_t ctx)
1721 {
1722 iflib_txq_t txq = ctx->ifc_txqs;
1723 iflib_rxq_t rxq = ctx->ifc_rxqs;
1724 if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
1725 iflib_fl_t fl;
1726 int i, j;
1727
1728 /* Tell the stack that the interface is no longer active */
1729 if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
1730
1731 IFDI_INTR_DISABLE(ctx);
1732
1733 /* Wait for current tx queue users to exit to disarm watchdog timer. */
1734 for (i = 0; i < sctx->isc_nqsets; i++, txq++, rxq++) {
1735 iflib_txq_check_drain(txq, 0);
1736 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)
1737 iflib_fl_bufs_free(fl);
1738
1739 }
1740 IFDI_STOP(ctx);
1741 }
1742
1743 /*
1744 * Internal service routines
1745 */
1746
1747 #if !defined(__i386__) && !defined(__amd64__)
1748 struct rxq_refill_cb_arg {
1749 int error;
1750 bus_dma_segment_t seg;
1751 int nseg;
1752 };
1753
1754 static void
_rxq_refill_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)1755 _rxq_refill_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
1756 {
1757 struct rxq_refill_cb_arg *cb_arg = arg;
1758
1759 cb_arg->error = error;
1760 cb_arg->seg = segs[0];
1761 cb_arg->nseg = nseg;
1762 }
1763 #endif
1764
1765 /*
1766 * Process one software descriptor
1767 */
1768 static struct mbuf *
iflib_rxd_pkt_get(iflib_fl_t fl,if_rxd_info_t ri)1769 iflib_rxd_pkt_get(iflib_fl_t fl, if_rxd_info_t ri)
1770 {
1771 iflib_sd_t sd_next, sd = &fl->ifl_sds[fl->ifl_cidx];
1772 uint32_t flags = 0;
1773 caddr_t cl;
1774 struct mbuf *m;
1775 int cidx_next, len = ri->iri_len;
1776
1777 MPASS(sd->ifsd_cl != NULL);
1778 MPASS(sd->ifsd_m != NULL);
1779
1780 fl->ifl_credits--;
1781 m = sd->ifsd_m;
1782 sd->ifsd_m = NULL;
1783 if (sd->ifsd_mh == NULL)
1784 flags |= M_PKTHDR;
1785
1786 /* SYNC ? */
1787 if (ri->iri_len <= IFLIB_RX_COPY_THRESH) {
1788 m_init(m, fl->ifl_zone, fl->ifl_buf_size, M_NOWAIT, MT_DATA, flags);
1789 memcpy(m->m_data, sd->ifsd_cl, ri->iri_len);
1790 } else {
1791 bus_dmamap_unload(fl->ifl_rxq->ifr_desc_tag, sd->ifsd_map);
1792 cl = sd->ifsd_cl;
1793 sd->ifsd_cl = NULL;
1794
1795 flags |= M_EXT;
1796 m_init(m, fl->ifl_zone, fl->ifl_buf_size, M_NOWAIT, MT_DATA, flags);
1797 m_cljset(m, cl, fl->ifl_cltype);
1798 }
1799
1800 if (ri->iri_pad) {
1801 m->m_data += ri->iri_pad;
1802 len -= ri->iri_pad;
1803 }
1804 m->m_len = len;
1805 if (sd->ifsd_mh == NULL)
1806 m->m_pkthdr.len = len;
1807 else
1808 sd->ifsd_mh->m_pkthdr.len += len;
1809
1810 if (sd->ifsd_mh != NULL && ri->iri_next_offset != 0) {
1811 /* We're in the middle of a packet and thus
1812 * need to pass this packet's data on to the
1813 * next descriptor
1814 */
1815 cidx_next = ri->iri_cidx + ri->iri_next_offset;
1816 if (cidx_next >= fl->ifl_size)
1817 cidx_next -= fl->ifl_size;
1818 sd_next = &fl->ifl_sds[cidx_next];
1819 sd_next->ifsd_mh = sd->ifsd_mh;
1820 sd_next->ifsd_mt = sd->ifsd_mt;
1821 sd->ifsd_mh = sd->ifsd_mt = NULL;
1822 sd_next->ifsd_mt->m_next = m;
1823 sd_next->ifsd_mt = m;
1824 m = NULL;
1825 } else if (sd->ifsd_mh == NULL && ri->iri_next_offset != 0) {
1826 /*
1827 * We're at the start of a multi-fragment packet
1828 */
1829 cidx_next = ri->iri_cidx + ri->iri_next_offset;
1830 if (cidx_next >= fl->ifl_size)
1831 cidx_next -= fl->ifl_size;
1832 sd_next = &fl->ifl_sds[cidx_next];
1833 sd_next->ifsd_mh = sd_next->ifsd_mt = m;
1834 m = NULL;
1835 } else if (sd->ifsd_mh != NULL && ri->iri_next_offset == 0) {
1836 /*
1837 * We're at the end of a multi-fragment packet
1838 */
1839 sd->ifsd_mt->m_next = m;
1840 sd->ifsd_mt = m;
1841 m = sd->ifsd_mh;
1842 sd->ifsd_mh = sd->ifsd_mt = NULL;
1843 }
1844 if (m == NULL)
1845 return (NULL);
1846
1847 m->m_pkthdr.rcvif = ri->iri_ifp;
1848 m->m_flags |= ri->iri_flags;
1849
1850 if (ri->iri_flags & M_VLANTAG)
1851 if_setvtag(m, ri->iri_vtag);
1852 m->m_pkthdr.flowid = ri->iri_flowid;
1853 M_HASHTYPE_SET(m, ri->iri_rsstype);
1854 m->m_pkthdr.csum_flags = ri->iri_csum_flags;
1855 m->m_pkthdr.csum_data = ri->iri_csum_data;
1856 return (m);
1857 }
1858
1859 static bool
iflib_rxeof(iflib_rxq_t rxq,int budget)1860 iflib_rxeof(iflib_rxq_t rxq, int budget)
1861 {
1862 if_ctx_t ctx = rxq->ifr_ctx;
1863 if_shared_ctx_t sctx = ctx->ifc_sctx;
1864 int avail, fl_cidx, cidx, gen, fl_gen, i;
1865 int *cidxp, *genp;
1866 struct if_rxd_info ri;
1867 iflib_dma_info_t di;
1868 int err, budget_left, rx_bytes, rx_pkts;
1869 iflib_fl_t fl;
1870 struct ifnet *ifp;
1871 struct lro_entry *queued;
1872 int8_t qidx;
1873 /*
1874 * XXX early demux data packets so that if_input processing only handles
1875 * acks in interrupt context
1876 */
1877 struct mbuf *m, *mh, *mt;
1878
1879 if (netmap_rx_irq(ctx->ifc_ifp, rxq->ifr_id, &budget)) {
1880 return (FALSE);
1881 }
1882
1883 ri.iri_qsidx = rxq->ifr_id;
1884 if (sctx->isc_flags & IFLIB_HAS_CQ) {
1885 cidxp = &rxq->ifr_cidx;
1886 genp = &rxq->ifr_gen;
1887 } else {
1888 cidxp = &rxq->ifr_fl[0].ifl_cidx;
1889 genp = &rxq->ifr_fl[0].ifl_gen;
1890 }
1891 cidx = *cidxp;
1892 gen = *genp;
1893 mh = mt = NULL;
1894 MPASS(budget > 0);
1895 rx_pkts = rx_bytes = 0;
1896
1897 if ((avail = iflib_rxd_avail(ctx, rxq, cidx)) == 0) {
1898 for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++)
1899 __iflib_fl_refill_lt(ctx, fl, budget + 8);
1900 DBG_COUNTER_INC(rx_unavail);
1901 return (false);
1902 }
1903
1904 for (budget_left = budget; (budget_left > 0) && (avail > 0); budget_left--, avail--) {
1905 if (__predict_false(!CTX_ACTIVE(ctx))) {
1906 DBG_COUNTER_INC(rx_ctx_inactive);
1907 break;
1908 }
1909 di = rxq->ifr_ifdi;
1910 bus_dmamap_sync(di->idi_tag, di->idi_map,
1911 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1912
1913 ri.iri_cidx = cidx;
1914 /*
1915 * Reset client set fields to their default values
1916 */
1917 ri.iri_flags = 0;
1918 ri.iri_m = NULL;
1919 ri.iri_next_offset = 0;
1920 ri.iri_pad = 0;
1921 ri.iri_qidx = 0;
1922 ri.iri_ifp = ctx->ifc_ifp;
1923 err = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri);
1924
1925 /* in lieu of handling correctly - make sure it isn't being unhandled */
1926 MPASS(err == 0);
1927
1928 qidx = ri.iri_qidx;
1929 if (++cidx == sctx->isc_nrxd) {
1930 cidx = 0;
1931 gen = 0;
1932 }
1933 if (sctx->isc_flags & IFLIB_HAS_CQ) {
1934 if (ri.iri_m != NULL) {
1935 m = ri.iri_m;
1936 ri.iri_m = NULL;
1937 goto imm_pkt;
1938 }
1939 /* was this only a completion queue message? */
1940 if (qidx == -1)
1941 continue;
1942 }
1943 fl = &rxq->ifr_fl[qidx];
1944 fl_cidx = fl->ifl_cidx;
1945 fl_gen = fl->ifl_gen;
1946 bus_dmamap_unload(rxq->ifr_desc_tag, fl->ifl_sds[fl_cidx].ifsd_map);
1947
1948 if (ri.iri_len == 0) {
1949 DBG_COUNTER_INC(rx_zero_len);
1950 m_freem(fl->ifl_sds[fl_cidx].ifsd_m);
1951 fl->ifl_sds[fl_cidx].ifsd_m = NULL;
1952 /*
1953 * XXX Note currently we don't free the initial pieces
1954 * of a multi-fragment packet
1955 */
1956 if (++fl_cidx == fl->ifl_size) {
1957 fl_cidx = 0;
1958 fl_gen = 0;
1959 }
1960 fl->ifl_cidx = fl_cidx;
1961 fl->ifl_gen = fl_gen;
1962 continue;
1963 }
1964 m = iflib_rxd_pkt_get(fl, &ri);
1965 if (++fl_cidx == fl->ifl_size) {
1966 fl_cidx = 0;
1967 fl_gen = 0;
1968 }
1969 fl->ifl_cidx = fl_cidx;
1970 fl->ifl_gen = fl_gen;
1971
1972 if (avail == 0 && budget_left)
1973 avail = iflib_rxd_avail(ctx, rxq, cidx);
1974
1975 if (m == NULL) {
1976 DBG_COUNTER_INC(rx_mbuf_null);
1977 continue;
1978 }
1979 imm_pkt:
1980 if (mh == NULL)
1981 mh = mt = m;
1982 else {
1983 mt->m_nextpkt = m;
1984 mt = m;
1985 }
1986 }
1987 /* make sure that we can refill faster than drain */
1988 for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++)
1989 __iflib_fl_refill_lt(ctx, fl, budget + 8);
1990
1991 ifp = ctx->ifc_ifp;
1992 while (mh != NULL) {
1993 m = mh;
1994 mh = mh->m_nextpkt;
1995 m->m_nextpkt = NULL;
1996 rx_bytes += m->m_pkthdr.len;
1997 rx_pkts++;
1998 if (rxq->ifr_lc.lro_cnt != 0 &&
1999 tcp_lro_rx(&rxq->ifr_lc, m, 0) == 0)
2000 continue;
2001 DBG_COUNTER_INC(rx_if_input);
2002 ifp->if_input(ifp, m);
2003 }
2004 if_inc_counter(ifp, IFCOUNTER_IBYTES, rx_bytes);
2005 if_inc_counter(ifp, IFCOUNTER_IPACKETS, rx_pkts);
2006
2007 /*
2008 * Flush any outstanding LRO work
2009 */
2010 while ((queued = SLIST_FIRST(&rxq->ifr_lc.lro_active)) != NULL) {
2011 SLIST_REMOVE_HEAD(&rxq->ifr_lc.lro_active, next);
2012 tcp_lro_flush(&rxq->ifr_lc, queued);
2013 }
2014 #ifdef INVARIANTS
2015 if ((sctx->isc_flags & IFLIB_HAS_CQ) == 0)
2016 MPASS(cidx == *cidxp);
2017 #endif
2018 if (sctx->isc_flags & IFLIB_HAS_CQ)
2019 *cidxp = cidx;
2020 *genp = gen;
2021 return (iflib_rxd_avail(ctx, rxq, cidx));
2022 }
2023
2024 #define M_CSUM_FLAGS(m) ((m)->m_pkthdr.csum_flags)
2025 #define M_HAS_VLANTAG(m) (m->m_flags & M_VLANTAG)
2026
2027 static __inline void
iflib_txd_db_check(if_ctx_t ctx,iflib_txq_t txq,int ring)2028 iflib_txd_db_check(if_ctx_t ctx, iflib_txq_t txq, int ring)
2029 {
2030 uint32_t dbval, dbval_prev;
2031
2032 if (ring || ++txq->ift_db_pending >= 32) {
2033 #ifdef notyet
2034 iflib_sd_t txsd = &txq->ift_sds[txq->ift_pidx];
2035
2036 /*
2037 * Flush deferred buffers first
2038 */
2039 /* XXX only do this on cards like T3 that can batch packets in a descriptor
2040 * and only do this if pidx != cidx
2041 */
2042 if (__predict_false(txsd->ifsd_m != NULL)) {
2043 struct if_pkt_info pi;
2044
2045 pi.ipi_m = NULL;
2046 pi.ipi_qsidx = txq->ift_id;
2047 pi.ipi_pidx = txq->ift_pidx;
2048 ctx->isc_txd_encap(ctx->ifc_softc, &pi);
2049 txq->ift_pidx = pi.ipi_new_pidx;
2050 }
2051 #endif
2052 dbval_prev = txq->ift_npending ? txq->ift_npending : txq->ift_pidx;
2053 /* the lock will only ever be contended in the !min_latency case */
2054 if (TX_TRY_LOCK(txq) == 0)
2055 return;
2056 dbval = txq->ift_npending ? txq->ift_npending : txq->ift_pidx;
2057 if (dbval == dbval_prev) {
2058 ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, dbval);
2059 txq->ift_db_pending = txq->ift_npending = 0;
2060 }
2061 TX_UNLOCK(txq);
2062 }
2063 }
2064
2065 static void
iflib_txd_deferred_db_check(void * arg)2066 iflib_txd_deferred_db_check(void * arg)
2067 {
2068 iflib_txq_t txq = arg;
2069 if_ctx_t ctx = txq->ift_ctx;
2070 uint32_t dbval;
2071
2072 dbval = txq->ift_npending ? txq->ift_npending : txq->ift_pidx;
2073 ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, dbval);
2074 txq->ift_db_pending = txq->ift_npending = 0;
2075 }
2076
2077 static int
iflib_encap(iflib_txq_t txq,struct mbuf ** m_headp)2078 iflib_encap(iflib_txq_t txq, struct mbuf **m_headp)
2079 {
2080 if_ctx_t ctx = txq->ift_ctx;
2081 if_shared_ctx_t sctx = ctx->ifc_sctx;
2082 bus_dma_segment_t *segs = txq->ift_segs;
2083 struct mbuf *m, *m_head = *m_headp;
2084 int pidx = txq->ift_pidx;
2085 iflib_sd_t txsd = &txq->ift_sds[pidx];
2086 bus_dmamap_t map = txsd->ifsd_map;
2087 struct if_pkt_info pi;
2088 bool remap = TRUE;
2089 int err, nsegs, ndesc;
2090
2091 retry:
2092
2093 err = bus_dmamap_load_mbuf_sg(txq->ift_desc_tag, map,
2094 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
2095
2096 if (__predict_false(err)) {
2097 switch (err) {
2098 case EFBIG:
2099 /* try defrag once */
2100 if (remap == TRUE) {
2101 remap = FALSE;
2102 m = m_defrag(*m_headp, M_NOWAIT);
2103 if (m == NULL) {
2104 txq->ift_mbuf_defrag_failed++;
2105 m_freem(*m_headp);
2106 DBG_COUNTER_INC(tx_frees);
2107 *m_headp = NULL;
2108 err = ENOBUFS;
2109 } else {
2110 *m_headp = m;
2111 goto retry;
2112 }
2113 }
2114 break;
2115 case ENOMEM:
2116 txq->ift_no_tx_dma_setup++;
2117 break;
2118 default:
2119 txq->ift_no_tx_dma_setup++;
2120 m_freem(*m_headp);
2121 DBG_COUNTER_INC(tx_frees);
2122 *m_headp = NULL;
2123 break;
2124 }
2125 DBG_COUNTER_INC(encap_load_mbuf_fail);
2126 return (err);
2127 }
2128
2129 /*
2130 * XXX assumes a 1 to 1 relationship between segments and
2131 * descriptors - this does not hold true on all drivers, e.g.
2132 * cxgb
2133 */
2134 if (nsegs > TXQ_AVAIL(txq)) {
2135 #ifdef INVARIANTS
2136 panic("filled ring in spite of INVARIANTS");
2137 #endif
2138 txq->ift_no_desc_avail++;
2139 bus_dmamap_unload(txq->ift_desc_tag, map);
2140 DBG_COUNTER_INC(encap_txq_avail_fail);
2141 if (txq->ift_task.gt_task.ta_pending == 0)
2142 GROUPTASK_ENQUEUE(&txq->ift_task);
2143 return (ENOBUFS);
2144 }
2145 m_head = *m_headp;
2146 pi.ipi_m = m_head;
2147 pi.ipi_segs = segs;
2148 pi.ipi_nsegs = nsegs;
2149 pi.ipi_pidx = pidx;
2150 pi.ipi_ndescs = 0;
2151 pi.ipi_qsidx = txq->ift_id;
2152
2153 MPASS(pidx >= 0 && pidx < sctx->isc_ntxd);
2154 if ((err = ctx->isc_txd_encap(ctx->ifc_softc, &pi)) == 0) {
2155 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
2156 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2157
2158 DBG_COUNTER_INC(tx_encap);
2159 MPASS(pi.ipi_m != NULL);
2160 MPASS(txsd->ifsd_m == NULL);
2161 #ifdef INVARIANTS
2162 {
2163 int i;
2164 for (i = 0; i < sctx->isc_ntxd; i++)
2165 MPASS(txq->ift_sds[i].ifsd_m != pi.ipi_m);
2166 }
2167 #endif
2168 txsd->ifsd_m = pi.ipi_m;
2169 MPASS(pi.ipi_new_pidx >= 0 && pi.ipi_new_pidx < sctx->isc_ntxd);
2170 if (pi.ipi_new_pidx >= pi.ipi_pidx) {
2171 ndesc = pi.ipi_new_pidx - pi.ipi_pidx;
2172 } else {
2173 ndesc = pi.ipi_new_pidx - pi.ipi_pidx + sctx->isc_ntxd;
2174 txq->ift_gen = 1;
2175 }
2176 MPASS(pi.ipi_new_pidx != pidx);
2177 MPASS(ndesc > 0);
2178 txq->ift_in_use += ndesc;
2179 txq->ift_pidx = pi.ipi_new_pidx;
2180 txq->ift_npending += pi.ipi_ndescs;
2181 } else {
2182 DBG_COUNTER_INC(encap_txd_encap_fail);
2183 printf("encap failed\n");
2184 }
2185 return (err);
2186 }
2187
2188 #define BRBITS 8
2189 #define FIRST_QSET(ctx) 0
2190 #define NQSETS(ctx) ((ctx)->ifc_softc_ctx.isc_nqsets)
2191 #define QIDX(ctx, m) ((((m)->m_pkthdr.flowid >> BRBITS) % NQSETS(ctx)) + FIRST_QSET(ctx))
2192 #define BRIDX(txq, m) ((m)->m_pkthdr.flowid % txq->ift_nbr)
2193 #define DESC_RECLAIMABLE(q) ((int)((q)->ift_processed - (q)->ift_cleaned - (q)->ift_ctx->ifc_softc_ctx.isc_tx_nsegments))
2194 #define RECLAIM_THRESH(ctx) ((ctx)->ifc_sctx->isc_tx_reclaim_thresh)
2195 #define MAX_TX_DESC(ctx) ((ctx)->ifc_softc_ctx.isc_tx_nsegments)
2196
2197
2198
2199 /* if there are more than TXQ_MIN_OCCUPANCY packets pending we consider deferring
2200 * doorbell writes
2201 */
2202 #define TXQ_MIN_OCCUPANCY 8
2203
2204 static inline int
iflib_txq_min_occupancy(iflib_txq_t txq)2205 iflib_txq_min_occupancy(iflib_txq_t txq)
2206 {
2207
2208 return (get_inuse(txq->ift_size, txq->ift_cidx, txq->ift_pidx, txq->ift_gen) < TXQ_MIN_OCCUPANCY + MAX_TX_DESC(txq->ift_ctx));
2209 }
2210
2211 static void
iflib_tx_desc_free(iflib_txq_t txq,int n)2212 iflib_tx_desc_free(iflib_txq_t txq, int n)
2213 {
2214 iflib_sd_t txsd;
2215 uint32_t qsize, cidx, mask, gen;
2216 struct mbuf *m;
2217
2218 cidx = txq->ift_cidx;
2219 gen = txq->ift_gen;
2220 qsize = txq->ift_ctx->ifc_sctx->isc_ntxd;
2221 mask = qsize-1;
2222 txsd = &txq->ift_sds[cidx];
2223
2224 while (n--) {
2225 prefetch(txq->ift_sds[(cidx + 1) & mask].ifsd_m);
2226 prefetch(txq->ift_sds[(cidx + 2) & mask].ifsd_m);
2227
2228 if (txsd->ifsd_m != NULL) {
2229 if (txsd->ifsd_flags & TX_SW_DESC_MAPPED) {
2230 bus_dmamap_unload(txq->ift_desc_tag, txsd->ifsd_map);
2231 txsd->ifsd_flags &= ~TX_SW_DESC_MAPPED;
2232 }
2233 while (txsd->ifsd_m) {
2234 m = txsd->ifsd_m;
2235 /* XXX we don't support any drivers that batch packets yet */
2236 MPASS(m->m_nextpkt == NULL);
2237
2238 txsd->ifsd_m = m->m_nextpkt;
2239 m->m_nextpkt = NULL;
2240 m_freem(m);
2241 DBG_COUNTER_INC(tx_frees);
2242 }
2243 }
2244
2245 ++txsd;
2246 if (++cidx == qsize) {
2247 cidx = 0;
2248 gen = 0;
2249 txsd = txq->ift_sds;
2250 }
2251 }
2252 txq->ift_cidx = cidx;
2253 txq->ift_gen = gen;
2254 }
2255
2256 static __inline int
iflib_completed_tx_reclaim(iflib_txq_t txq,int thresh)2257 iflib_completed_tx_reclaim(iflib_txq_t txq, int thresh)
2258 {
2259 int reclaim;
2260 if_ctx_t ctx = txq->ift_ctx;
2261
2262 KASSERT(thresh >= 0, ("invalid threshold to reclaim"));
2263 MPASS(thresh /*+ MAX_TX_DESC(txq->ift_ctx) */ < txq->ift_size);
2264
2265 /*
2266 * Need a rate-limiting check so that this isn't called every time
2267 */
2268 iflib_tx_credits_update(ctx, txq);
2269 reclaim = DESC_RECLAIMABLE(txq);
2270
2271 if (reclaim <= thresh /* + MAX_TX_DESC(txq->ift_ctx) */)
2272 return (0);
2273
2274 iflib_tx_desc_free(txq, reclaim);
2275 txq->ift_cleaned += reclaim;
2276 txq->ift_in_use -= reclaim;
2277
2278 if (txq->ift_active == FALSE)
2279 txq->ift_active = TRUE;
2280
2281 return (reclaim);
2282 }
2283
2284 #if 0
2285 static void
2286 iflib_tx_timeout(void *arg)
2287 {
2288
2289 /* XXX */
2290 }
2291
2292 static void
2293 iflib_txq_deferred(struct buf_ring_sc *br __unused, void *sc)
2294 {
2295 iflib_txq_t txq = sc;
2296
2297 GROUPTASK_ENQUEUE(&txq->ift_task);
2298 }
2299 #endif
2300
2301
2302 static void
_ring_peek(struct mp_ring * r,struct mbuf ** m,int cidx,int count)2303 _ring_peek(struct mp_ring *r, struct mbuf **m, int cidx, int count)
2304 {
2305 int i;
2306
2307 for (i = 0; i < count; i++)
2308 m[i] = r->items[(cidx + i) & (r->size-1)];
2309 }
2310
2311 static void
_ring_putback(struct mp_ring * r,struct mbuf * m,int i,int cidx)2312 _ring_putback(struct mp_ring *r, struct mbuf *m, int i, int cidx)
2313 {
2314
2315 r->items[(cidx + i) & (r->size-1)] = m;
2316 }
2317
2318 static void
iflib_txq_check_drain(iflib_txq_t txq,int budget)2319 iflib_txq_check_drain(iflib_txq_t txq, int budget)
2320 {
2321
2322 mp_ring_check_drainage(txq->ift_br[0], budget);
2323 }
2324
2325 static uint32_t
iflib_txq_can_drain(struct mp_ring * r)2326 iflib_txq_can_drain(struct mp_ring *r)
2327 {
2328 iflib_txq_t txq = r->cookie;
2329
2330 return (TXQ_AVAIL(txq) >= MAX_TX_DESC(txq->ift_ctx));
2331 }
2332
2333 static uint32_t
iflib_txq_drain(struct mp_ring * r,uint32_t cidx,uint32_t pidx)2334 iflib_txq_drain(struct mp_ring *r, uint32_t cidx, uint32_t pidx)
2335 {
2336 printf("iflib_txq_drain called\n");
2337 iflib_txq_t txq = r->cookie;
2338 if_ctx_t ctx = txq->ift_ctx;
2339 if_t ifp = ctx->ifc_ifp;
2340 struct mbuf **mp = &txq->ift_mp[0];
2341 int i, count, pkt_sent, bytes_sent, mcast_sent, avail;
2342
2343 avail = IDXDIFF(pidx, cidx, r->size);
2344 if (ctx->ifc_flags & IFC_QFLUSH) {
2345 DBG_COUNTER_INC(txq_drain_flushing);
2346 for (i = 0; i < avail; i++) {
2347 m_freem(r->items[(cidx + i) & (r->size-1)]);
2348 r->items[(cidx + i) & (r->size-1)] = NULL;
2349 }
2350 return (avail);
2351 }
2352 iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx));
2353 if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE ) {
2354 txq->ift_qstatus = IFLIB_QUEUE_IDLE;
2355 TX_LOCK(txq);
2356 callout_stop(&txq->ift_timer);
2357 callout_stop(&txq->ift_db_check);
2358 TX_UNLOCK(txq);
2359 DBG_COUNTER_INC(txq_drain_oactive);
2360 return (0);
2361 }
2362 mcast_sent = bytes_sent = pkt_sent = 0;
2363 count = MIN(avail, BATCH_SIZE);
2364 _ring_peek(r, mp, cidx, count);
2365
2366 if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING) ||
2367 !LINK_ACTIVE(ctx)) {
2368 DBG_COUNTER_INC(txq_drain_notready);
2369 goto skip_db;
2370 }
2371
2372 for (i = 0; i < count; i++) {
2373 if(iflib_encap(txq, &mp[i])) {
2374 _ring_putback(r, mp[i], i, cidx);
2375 DBG_COUNTER_INC(txq_drain_encapfail);
2376 goto done;
2377 }
2378 DBG_COUNTER_INC(tx_sent);
2379 pkt_sent++;
2380 bytes_sent += mp[i]->m_pkthdr.len;
2381 if (mp[i]->m_flags & M_MCAST)
2382 mcast_sent++;
2383 iflib_txd_db_check(ctx, txq, 0);
2384 ETHER_BPF_MTAP(ifp, mp[i]);
2385 }
2386 done:
2387
2388 if ((iflib_min_tx_latency || iflib_txq_min_occupancy(txq)) && txq->ift_db_pending)
2389 iflib_txd_db_check(ctx, txq, 1);
2390 else if (txq->ift_db_pending && (callout_pending(&txq->ift_db_check) == 0))
2391 callout_reset_on(&txq->ift_db_check, 1, iflib_txd_deferred_db_check,
2392 txq, txq->ift_db_check.c_cpu);
2393 skip_db:
2394 if_inc_counter(ifp, IFCOUNTER_OBYTES, bytes_sent);
2395 if_inc_counter(ifp, IFCOUNTER_OPACKETS, pkt_sent);
2396 if (mcast_sent)
2397 if_inc_counter(ifp, IFCOUNTER_OMCASTS, mcast_sent);
2398
2399 return (pkt_sent);
2400 }
2401
2402 static void
_task_fn_tx(void * context,int pending)2403 _task_fn_tx(void *context, int pending)
2404 {
2405 iflib_txq_t txq = context;
2406 if_ctx_t ctx = txq->ift_ctx;
2407
2408 if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
2409 return;
2410
2411 mp_ring_check_drainage(txq->ift_br[0], IFLIB_BUDGET);
2412 }
2413
2414 static void
_task_fn_rx(void * context,int pending)2415 _task_fn_rx(void *context, int pending)
2416 {
2417 iflib_rxq_t rxq = context;
2418 if_ctx_t ctx = rxq->ifr_ctx;
2419 bool more;
2420
2421 DBG_COUNTER_INC(task_fn_rxs);
2422 if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
2423 return;
2424
2425 if ((more = iflib_rxeof(rxq, 8 /* XXX */)) == false) {
2426 if (ctx->ifc_flags & IFC_LEGACY)
2427 IFDI_INTR_ENABLE(ctx);
2428 else {
2429 DBG_COUNTER_INC(rx_intr_enables);
2430 IFDI_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id);
2431 }
2432 }
2433 if (more)
2434 GROUPTASK_ENQUEUE(&rxq->ifr_task);
2435 }
2436
2437 static void
_task_fn_admin(void * context,int pending)2438 _task_fn_admin(void *context, int pending)
2439 {
2440 if_ctx_t ctx = context;
2441 if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
2442 iflib_txq_t txq;
2443 int i;
2444
2445 if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
2446 return;
2447
2448 CTX_LOCK(ctx);
2449 for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_nqsets; i++, txq++) {
2450 TX_LOCK(txq);
2451 callout_stop(&txq->ift_timer);
2452 TX_UNLOCK(txq);
2453 }
2454 IFDI_UPDATE_ADMIN_STATUS(ctx);
2455 for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_nqsets; i++, txq++)
2456 callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, txq->ift_timer.c_cpu);
2457 IFDI_LINK_INTR_ENABLE(ctx);
2458 CTX_UNLOCK(ctx);
2459
2460 if (LINK_ACTIVE(ctx) == 0)
2461 return;
2462 for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_nqsets; i++, txq++)
2463 iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET);
2464 }
2465
2466
2467 static void
_task_fn_iov(void * context,int pending)2468 _task_fn_iov(void *context, int pending)
2469 {
2470 if_ctx_t ctx = context;
2471
2472 if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
2473 return;
2474
2475 CTX_LOCK(ctx);
2476 IFDI_VFLR_HANDLE(ctx);
2477 CTX_UNLOCK(ctx);
2478 }
2479
2480
2481 #if 0
2482 void
2483 iflib_intr_rx(void *arg)
2484 {
2485 iflib_rxq_t rxq = arg;
2486
2487 ++rxq->ifr_rx_irq;
2488 _task_fn_rx(arg, 0);
2489 }
2490
2491 void
2492 iflib_intr_tx(void *arg)
2493 {
2494 iflib_txq_t txq= arg;
2495
2496 ++txq->ift_tx_irq;
2497 _task_fn_tx(arg, 0);
2498 }
2499
2500 void
2501 iflib_intr_link(void *arg)
2502 {
2503 if_ctx_t ctx = arg;
2504
2505 ++ctx->ifc_link_irq;
2506 _task_fn_link(arg, 0);
2507 }
2508 #endif
2509
2510 static int
iflib_sysctl_int_delay(SYSCTL_HANDLER_ARGS)2511 iflib_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
2512 {
2513 int err;
2514 if_int_delay_info_t info;
2515 if_ctx_t ctx;
2516
2517 info = (if_int_delay_info_t)arg1;
2518 ctx = info->iidi_ctx;
2519 info->iidi_req = req;
2520 info->iidi_oidp = oidp;
2521 CTX_LOCK(ctx);
2522 err = IFDI_SYSCTL_INT_DELAY(ctx, info);
2523 CTX_UNLOCK(ctx);
2524 return (err);
2525 }
2526
2527 /*********************************************************************
2528 *
2529 * IFNET FUNCTIONS
2530 *
2531 **********************************************************************/
2532
2533 static void
iflib_if_init(void * arg)2534 iflib_if_init(void *arg)
2535 {
2536 if_ctx_t ctx = arg;
2537 CTX_LOCK(ctx);
2538 iflib_stop(ctx);
2539 iflib_init_locked(ctx);
2540 CTX_UNLOCK(ctx);
2541 }
2542
2543 static int
iflib_if_transmit(if_t ifp,struct mbuf * m)2544 iflib_if_transmit(if_t ifp, struct mbuf *m)
2545 {
2546 if_ctx_t ctx = if_getsoftc(ifp);
2547
2548 iflib_txq_t txq;
2549 struct mbuf *marr[16], **mp, *next;
2550 int err, i, count, qidx;
2551
2552 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || !LINK_ACTIVE(ctx)) {
2553 DBG_COUNTER_INC(tx_frees);
2554 m_freem(m);
2555 return (0);
2556 }
2557
2558 qidx = 0;
2559 if ((NQSETS(ctx) > 1) && M_HASHTYPE_GET(m))
2560 qidx = QIDX(ctx, m);
2561 /*
2562 * XXX calculate buf_ring based on flowid (divvy up bits?)
2563 */
2564 txq = &ctx->ifc_txqs[qidx];
2565
2566
2567 if (txq->ift_closed) {
2568 while (m != NULL) {
2569 next = m->m_nextpkt;
2570 m->m_nextpkt = NULL;
2571 m_freem(m);
2572 m = next;
2573 }
2574 return (ENOBUFS);
2575 }
2576
2577 qidx = count = 0;
2578 mp = marr;
2579 next = m;
2580 do {
2581 count++;
2582 next = next->m_nextpkt;
2583 } while (next != NULL);
2584
2585 if (count > 16)
2586 if ((mp = malloc(count*sizeof(struct mbuf *), M_IFLIB, M_NOWAIT)) == NULL) {
2587 /* XXX check nextpkt */
2588 m_freem(m);
2589 /* XXX simplify for now */
2590 DBG_COUNTER_INC(tx_frees);
2591 return (ENOBUFS);
2592 }
2593 for (next = m, i = 0; next != NULL; i++) {
2594 mp[i] = next;
2595 next = next->m_nextpkt;
2596 mp[i]->m_nextpkt = NULL;
2597 }
2598 DBG_COUNTER_INC(tx_seen);
2599 err = mp_ring_enqueue(txq->ift_br[0], (void **)mp, count, IFLIB_BUDGET);
2600 /* drain => err = iflib_txq_transmit(ifp, txq, m); */
2601 if (err) {
2602 txq->ift_closed = TRUE;
2603 for (i = 0; i < count; i++)
2604 m_freem(mp[i]);
2605 mp_ring_check_drainage(txq->ift_br[0], BATCH_SIZE);
2606 }
2607 if (count > 16)
2608 free(mp, M_IFLIB);
2609
2610 return (err);
2611 }
2612
2613 static void
iflib_if_qflush(if_t ifp)2614 iflib_if_qflush(if_t ifp)
2615 {
2616 if_ctx_t ctx = if_getsoftc(ifp);
2617 iflib_txq_t txq = ctx->ifc_txqs;
2618 int i;
2619
2620 CTX_LOCK(ctx);
2621 ctx->ifc_flags |= IFC_QFLUSH;
2622 CTX_UNLOCK(ctx);
2623 for (i = 0; i < NQSETS(ctx); i++, txq++)
2624 while (!mp_ring_is_idle(txq->ift_br[0]))
2625 iflib_txq_check_drain(txq, 0);
2626 CTX_LOCK(ctx);
2627 ctx->ifc_flags &= ~IFC_QFLUSH;
2628 CTX_UNLOCK(ctx);
2629
2630 if_qflush(ifp);
2631 }
2632
2633 static int
iflib_if_ioctl(if_t ifp,u_long command,caddr_t data)2634 iflib_if_ioctl(if_t ifp, u_long command, caddr_t data)
2635 {
2636 if_ctx_t ctx = if_getsoftc(ifp);
2637 struct ifreq *ifr = (struct ifreq *)data;
2638 #if defined(INET) || defined(INET6)
2639 struct ifaddr *ifa = (struct ifaddr *)data;
2640 #endif
2641 bool avoid_reset = FALSE;
2642 int err = 0;
2643
2644 switch (command) {
2645 case SIOCSIFADDR:
2646 #ifdef INET
2647 if (ifa->ifa_addr->sa_family == AF_INET)
2648 avoid_reset = TRUE;
2649 #endif
2650 #ifdef INET6
2651 if (ifa->ifa_addr->sa_family == AF_INET6)
2652 avoid_reset = TRUE;
2653 #endif
2654 /*
2655 ** Calling init results in link renegotiation,
2656 ** so we avoid doing it when possible.
2657 */
2658 if (avoid_reset) {
2659 if_setflagbits(ifp, IFF_UP,0);
2660 if (!(if_getdrvflags(ifp)& IFF_DRV_RUNNING))
2661 iflib_if_init(ctx);
2662 #ifdef INET
2663 if (!(if_getflags(ifp) & IFF_NOARP))
2664 arp_ifinit_drv(ifp, ifa);
2665 #endif
2666 } else
2667 err = ether_ioctl(ifp, command, data);
2668 break;
2669 case SIOCSIFMTU:
2670 CTX_LOCK(ctx);
2671 /* detaching ?*/
2672 if ((err = IFDI_MTU_SET(ctx, ifr->ifr_mtu)) == 0) {
2673 iflib_init_locked(ctx);
2674 err = if_setmtu(ifp, ifr->ifr_mtu);
2675 }
2676 CTX_UNLOCK(ctx);
2677 break;
2678 case SIOCSIFFLAGS:
2679 CTX_LOCK(ctx);
2680 if (if_getflags(ifp) & IFF_UP) {
2681 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
2682 if ((if_getflags(ifp) ^ ctx->ifc_if_flags) &
2683 (IFF_PROMISC | IFF_ALLMULTI)) {
2684 err = IFDI_PROMISC_SET(ctx, if_getflags(ifp));
2685 }
2686 } else
2687 iflib_init_locked(ctx);
2688 } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
2689 iflib_stop(ctx);
2690 }
2691 ctx->ifc_if_flags = if_getflags(ifp);
2692 CTX_UNLOCK(ctx);
2693 break;
2694
2695 break;
2696 case SIOCADDMULTI:
2697 case SIOCDELMULTI:
2698 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
2699 CTX_LOCK(ctx);
2700 IFDI_INTR_DISABLE(ctx);
2701 IFDI_MULTI_SET(ctx);
2702 IFDI_INTR_ENABLE(ctx);
2703 CTX_UNLOCK(ctx);
2704 }
2705 break;
2706 case SIOCSIFMEDIA:
2707 CTX_LOCK(ctx);
2708 IFDI_MEDIA_SET(ctx);
2709 CTX_UNLOCK(ctx);
2710 /* falls thru */
2711 case SIOCGIFMEDIA:
2712 err = ifmedia_ioctl(ifp, ifr, &ctx->ifc_media, command);
2713 break;
2714 case SIOCGI2C:
2715 {
2716 struct ifi2creq i2c;
2717
2718 err = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
2719 if (err != 0)
2720 break;
2721 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
2722 err = EINVAL;
2723 break;
2724 }
2725 if (i2c.len > sizeof(i2c.data)) {
2726 err = EINVAL;
2727 break;
2728 }
2729
2730 if ((err = IFDI_I2C_REQ(ctx, &i2c)) == 0)
2731 err = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
2732 break;
2733 }
2734 case SIOCSIFCAP:
2735 {
2736 int mask, reinit;
2737
2738 reinit = 0;
2739 mask = ifr->ifr_reqcap ^ if_getcapenable(ifp);
2740
2741 #ifdef TCP_OFFLOAD
2742 if (mask & IFCAP_TOE4) {
2743 if_togglecapenable(ifp, IFCAP_TOE4);
2744 reinit = 1;
2745 }
2746 #endif
2747 if (mask & IFCAP_RXCSUM)
2748 if_togglecapenable(ifp, IFCAP_RXCSUM);
2749 if (mask & IFCAP_RXCSUM_IPV6)
2750 if_togglecapenable(ifp, IFCAP_RXCSUM_IPV6);
2751 if (mask & IFCAP_HWCSUM) {
2752 if_togglecapenable(ifp, IFCAP_HWCSUM);
2753 reinit = 1;
2754 }
2755 if (mask & IFCAP_LRO)
2756 if_togglecapenable(ifp, IFCAP_LRO);
2757 if (mask & IFCAP_TSO4) {
2758 if_togglecapenable(ifp, IFCAP_TSO4);
2759 reinit = 1;
2760 }
2761 if (mask & IFCAP_TSO6) {
2762 if_togglecapenable(ifp, IFCAP_TSO6);
2763 reinit = 1;
2764 }
2765 if (mask & IFCAP_VLAN_HWTAGGING) {
2766 if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING);
2767 reinit = 1;
2768 }
2769 if (mask & IFCAP_VLAN_MTU) {
2770 if_togglecapenable(ifp, IFCAP_VLAN_MTU);
2771 reinit = 1;
2772 }
2773 if (mask & IFCAP_VLAN_HWFILTER) {
2774 if_togglecapenable(ifp, IFCAP_VLAN_HWFILTER);
2775 reinit = 1;
2776 }
2777 if (mask & IFCAP_VLAN_HWTSO) {
2778 if_togglecapenable(ifp, IFCAP_VLAN_HWTSO);
2779 reinit = 1;
2780 }
2781 if ((mask & IFCAP_WOL) &&
2782 (if_getcapabilities(ifp) & IFCAP_WOL) != 0) {
2783 if (mask & IFCAP_WOL_MCAST)
2784 if_togglecapenable(ifp, IFCAP_WOL_MCAST);
2785 if (mask & IFCAP_WOL_MAGIC)
2786 if_togglecapenable(ifp, IFCAP_WOL_MAGIC);
2787 }
2788 if (reinit && (if_getdrvflags(ifp) & IFF_DRV_RUNNING)) {
2789 iflib_if_init(ctx);
2790 }
2791 if_vlancap(ifp);
2792 break;
2793 }
2794
2795 default:
2796 err = ether_ioctl(ifp, command, data);
2797 break;
2798 }
2799
2800 return (err);
2801 }
2802
2803 static uint64_t
iflib_if_get_counter(if_t ifp,ift_counter cnt)2804 iflib_if_get_counter(if_t ifp, ift_counter cnt)
2805 {
2806 if_ctx_t ctx = if_getsoftc(ifp);
2807
2808 return (IFDI_GET_COUNTER(ctx, cnt));
2809 }
2810
2811 /*********************************************************************
2812 *
2813 * OTHER FUNCTIONS EXPORTED TO THE STACK
2814 *
2815 **********************************************************************/
2816
2817 static void
iflib_vlan_register(void * arg,if_t ifp,uint16_t vtag)2818 iflib_vlan_register(void *arg, if_t ifp, uint16_t vtag)
2819 {
2820 if_ctx_t ctx = if_getsoftc(ifp);
2821
2822 if ((void *)ctx != arg)
2823 return;
2824
2825 if ((vtag == 0) || (vtag > 4095))
2826 return;
2827
2828 CTX_LOCK(ctx);
2829 IFDI_VLAN_REGISTER(ctx, vtag);
2830 /* Re-init to load the changes */
2831 if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
2832 iflib_init_locked(ctx);
2833 CTX_UNLOCK(ctx);
2834 }
2835
2836 static void
iflib_vlan_unregister(void * arg,if_t ifp,uint16_t vtag)2837 iflib_vlan_unregister(void *arg, if_t ifp, uint16_t vtag)
2838 {
2839 if_ctx_t ctx = if_getsoftc(ifp);
2840
2841 if ((void *)ctx != arg)
2842 return;
2843
2844 if ((vtag == 0) || (vtag > 4095))
2845 return;
2846
2847 CTX_LOCK(ctx);
2848 IFDI_VLAN_UNREGISTER(ctx, vtag);
2849 /* Re-init to load the changes */
2850 if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
2851 iflib_init_locked(ctx);
2852 CTX_UNLOCK(ctx);
2853 }
2854
2855 static void
iflib_led_func(void * arg,int onoff)2856 iflib_led_func(void *arg, int onoff)
2857 {
2858 if_ctx_t ctx = arg;
2859
2860 CTX_LOCK(ctx);
2861 IFDI_LED_FUNC(ctx, onoff);
2862 CTX_UNLOCK(ctx);
2863 }
2864
2865 /*********************************************************************
2866 *
2867 * BUS FUNCTION DEFINITIONS
2868 *
2869 **********************************************************************/
2870
2871 int
iflib_device_probe(device_t dev)2872 iflib_device_probe(device_t dev)
2873 {
2874 pci_vendor_info_t *ent;
2875
2876 uint16_t pci_vendor_id, pci_device_id;
2877 uint16_t pci_subvendor_id, pci_subdevice_id;
2878 uint16_t pci_rev_id;
2879 if_shared_ctx_t sctx;
2880
2881 if ((sctx = DEVICE_REGISTER(dev)) == NULL || sctx->isc_magic != IFLIB_MAGIC)
2882 return (ENOTSUP);
2883
2884 pci_vendor_id = pci_get_vendor(dev);
2885 pci_device_id = pci_get_device(dev);
2886 pci_subvendor_id = pci_get_subvendor(dev);
2887 pci_subdevice_id = pci_get_subdevice(dev);
2888 pci_rev_id = pci_get_revid(dev);
2889
2890 ent = sctx->isc_vendor_info;
2891 while (ent->pvi_vendor_id != 0) {
2892 if ((pci_vendor_id == ent->pvi_vendor_id) &&
2893 (pci_device_id == ent->pvi_device_id) &&
2894
2895 ((pci_subvendor_id == ent->pvi_subvendor_id) ||
2896 (ent->pvi_subvendor_id == 0)) &&
2897
2898 ((pci_subdevice_id == ent->pvi_subdevice_id) ||
2899 (ent->pvi_subdevice_id == 0)) &&
2900 ((pci_rev_id == ent->pvi_rev_id) ||
2901 (ent->pvi_rev_id == 0))) {
2902
2903 device_set_desc_copy(dev, ent->pvi_name);
2904 /* this needs to be changed to zero if the bus probing code
2905 * ever stops re-probing on best match because the sctx
2906 * may have its values over written by register calls
2907 * in subsequent probes
2908 */
2909 return (BUS_PROBE_DEFAULT);
2910 }
2911 ent++;
2912 }
2913 return (ENXIO);
2914 }
2915
2916 int
iflib_device_register(device_t dev,void * sc,if_shared_ctx_t sctx,if_ctx_t * ctxp)2917 iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ctxp)
2918 {
2919 int err, rid, msix, msix_bar;
2920 if_ctx_t ctx;
2921 if_softc_ctx_t scctx;
2922
2923
2924 ctx = malloc(sizeof(* ctx), M_IFLIB, M_WAITOK|M_ZERO);
2925
2926 if (sc == NULL) {
2927 sc = malloc(sctx->isc_driver->size, M_IFLIB, M_WAITOK|M_ZERO);
2928 device_set_softc(dev, ctx);
2929 }
2930
2931 ctx->ifc_sctx = sctx;
2932 ctx->ifc_dev = dev;
2933 ctx->ifc_txrx = *sctx->isc_txrx;
2934 ctx->ifc_softc = sc;
2935
2936 if ((err = iflib_register(ctx)) != 0) {
2937 device_printf(dev, "iflib_register failed %d\n", err);
2938 return (err);
2939 }
2940 if ((err = IFDI_ATTACH_PRE(ctx)) != 0) {
2941 device_printf(dev, "IFDI_ATTACH_PRE failed %d\n", err);
2942 return (err);
2943 }
2944
2945 scctx = &ctx->ifc_softc_ctx;
2946 msix_bar = scctx->isc_msix_bar;
2947
2948 /*
2949 ** Now setup MSI or MSI/X, should
2950 ** return us the number of supported
2951 ** vectors. (Will be 1 for MSI)
2952 */
2953 if (sctx->isc_flags & IFLIB_SKIP_MSIX) {
2954 msix = scctx->isc_vectors;
2955 } else if (scctx->isc_msix_bar != 0)
2956 msix = iflib_msix_init(ctx);
2957 else {
2958 scctx->isc_vectors = 1;
2959 scctx->isc_nqsets = 1;
2960 scctx->isc_intr = IFLIB_INTR_LEGACY;
2961 msix = 0;
2962 }
2963 /* Get memory for the station queues */
2964 if ((err = iflib_queues_alloc(ctx))) {
2965 device_printf(dev, "Unable to allocate queue memory\n");
2966 goto fail;
2967 }
2968
2969 if ((err = iflib_qset_structures_setup(ctx))) {
2970 device_printf(dev, "qset structure setup failed %d\n", err);
2971 goto fail_queues;
2972 }
2973
2974 if (msix > 1 && (err = IFDI_MSIX_INTR_ASSIGN(ctx, msix)) != 0) {
2975 device_printf(dev, "IFDI_MSIX_INTR_ASSIGN failed %d\n", err);
2976 goto fail_intr_free;
2977 }
2978 if (msix <= 1) {
2979 rid = 0;
2980 if (scctx->isc_intr == IFLIB_INTR_MSI) {
2981 MPASS(msix == 1);
2982 rid = 1;
2983 }
2984 if ((err = iflib_legacy_setup(ctx, ctx->isc_legacy_intr, ctx, &rid, "irq0")) != 0) {
2985 device_printf(dev, "iflib_legacy_setup failed %d\n", err);
2986 goto fail_intr_free;
2987 }
2988 }
2989 ether_ifattach(ctx->ifc_ifp, ctx->ifc_mac);
2990 if ((err = IFDI_ATTACH_POST(ctx)) != 0) {
2991 device_printf(dev, "IFDI_ATTACH_POST failed %d\n", err);
2992 goto fail_detach;
2993 }
2994 if ((err = iflib_netmap_attach(ctx))) {
2995 device_printf(ctx->ifc_dev, "netmap attach failed: %d\n", err);
2996 goto fail_detach;
2997 }
2998 *ctxp = ctx;
2999 return (0);
3000 fail_detach:
3001 ether_ifdetach(ctx->ifc_ifp);
3002 fail_intr_free:
3003 if (scctx->isc_intr == IFLIB_INTR_MSIX || scctx->isc_intr == IFLIB_INTR_MSI)
3004 pci_release_msi(ctx->ifc_dev);
3005 fail_queues:
3006 /* XXX free queues */
3007 fail:
3008 IFDI_DETACH(ctx);
3009 return (err);
3010 }
3011
3012 int
iflib_device_attach(device_t dev)3013 iflib_device_attach(device_t dev)
3014 {
3015 if_ctx_t ctx;
3016 if_shared_ctx_t sctx;
3017
3018 if ((sctx = DEVICE_REGISTER(dev)) == NULL || sctx->isc_magic != IFLIB_MAGIC)
3019 return (ENOTSUP);
3020
3021 pci_enable_busmaster(dev);
3022
3023 return (iflib_device_register(dev, NULL, sctx, &ctx));
3024 }
3025
3026 int
iflib_device_deregister(if_ctx_t ctx)3027 iflib_device_deregister(if_ctx_t ctx)
3028 {
3029 if_t ifp = ctx->ifc_ifp;
3030 iflib_txq_t txq;
3031 iflib_rxq_t rxq;
3032 device_t dev = ctx->ifc_dev;
3033 int i, nqsets;
3034 struct taskqgroup *tqg;
3035
3036 /* Make sure VLANS are not using driver */
3037 if (if_vlantrunkinuse(ifp)) {
3038 device_printf(dev,"Vlan in use, detach first\n");
3039 return (EBUSY);
3040 }
3041
3042 CTX_LOCK(ctx);
3043 ctx->ifc_in_detach = 1;
3044 iflib_stop(ctx);
3045 CTX_UNLOCK(ctx);
3046
3047 /* Unregister VLAN events */
3048 if (ctx->ifc_vlan_attach_event != NULL)
3049 EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event);
3050 if (ctx->ifc_vlan_detach_event != NULL)
3051 EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event);
3052
3053 iflib_netmap_detach(ifp);
3054 ether_ifdetach(ifp);
3055 /* ether_ifdetach calls if_qflush - lock must be destroy afterwards*/
3056 CTX_LOCK_DESTROY(ctx);
3057 if (ctx->ifc_led_dev != NULL)
3058 led_destroy(ctx->ifc_led_dev);
3059 /* XXX drain any dependent tasks */
3060 nqsets = ctx->ifc_softc_ctx.isc_nqsets;
3061 tqg = gctx->igc_io_tqg;
3062 for (txq = ctx->ifc_txqs, i = 0, rxq = ctx->ifc_rxqs; i < nqsets; i++, txq++, rxq++) {
3063 callout_drain(&txq->ift_timer);
3064 callout_drain(&txq->ift_db_check);
3065 if (txq->ift_task.gt_uniq != NULL)
3066 taskqgroup_detach(tqg, &txq->ift_task);
3067 if (rxq->ifr_task.gt_uniq != NULL)
3068 taskqgroup_detach(tqg, &rxq->ifr_task);
3069 }
3070 tqg = gctx->igc_config_tqg;
3071 if (ctx->ifc_admin_task.gt_uniq != NULL)
3072 taskqgroup_detach(tqg, &ctx->ifc_admin_task);
3073 if (ctx->ifc_vflr_task.gt_uniq != NULL)
3074 taskqgroup_detach(tqg, &ctx->ifc_vflr_task);
3075
3076 IFDI_DETACH(ctx);
3077 if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_LEGACY) {
3078 pci_release_msi(dev);
3079 }
3080 if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_MSIX) {
3081 iflib_irq_free(ctx, &ctx->ifc_legacy_irq);
3082 }
3083 if (ctx->ifc_msix_mem != NULL) {
3084 bus_release_resource(ctx->ifc_dev, SYS_RES_MEMORY,
3085 ctx->ifc_softc_ctx.isc_msix_bar, ctx->ifc_msix_mem);
3086 ctx->ifc_msix_mem = NULL;
3087 }
3088
3089 bus_generic_detach(dev);
3090 if_free(ifp);
3091
3092 iflib_tx_structures_free(ctx);
3093 iflib_rx_structures_free(ctx);
3094 return (0);
3095 }
3096
3097
3098 int
iflib_device_detach(device_t dev)3099 iflib_device_detach(device_t dev)
3100 {
3101 if_ctx_t ctx = device_get_softc(dev);
3102
3103 return (iflib_device_deregister(ctx));
3104 }
3105
3106 int
iflib_device_suspend(device_t dev)3107 iflib_device_suspend(device_t dev)
3108 {
3109 if_ctx_t ctx = device_get_softc(dev);
3110
3111 CTX_LOCK(ctx);
3112 IFDI_SUSPEND(ctx);
3113 CTX_UNLOCK(ctx);
3114
3115 return bus_generic_suspend(dev);
3116 }
3117 int
iflib_device_shutdown(device_t dev)3118 iflib_device_shutdown(device_t dev)
3119 {
3120 if_ctx_t ctx = device_get_softc(dev);
3121
3122 CTX_LOCK(ctx);
3123 IFDI_SHUTDOWN(ctx);
3124 CTX_UNLOCK(ctx);
3125
3126 return bus_generic_suspend(dev);
3127 }
3128
3129
3130 int
iflib_device_resume(device_t dev)3131 iflib_device_resume(device_t dev)
3132 {
3133 if_ctx_t ctx = device_get_softc(dev);
3134 iflib_txq_t txq = ctx->ifc_txqs;
3135
3136 CTX_LOCK(ctx);
3137 IFDI_RESUME(ctx);
3138 iflib_init_locked(ctx);
3139 CTX_UNLOCK(ctx);
3140 for (int i = 0; i < ctx->ifc_softc_ctx.isc_nqsets; i++, txq++)
3141 iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET);
3142
3143 return (bus_generic_resume(dev));
3144 }
3145
3146 int
iflib_device_iov_init(device_t dev,uint16_t num_vfs,const nvlist_t * params)3147 iflib_device_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params)
3148 {
3149 int error;
3150 if_ctx_t ctx = device_get_softc(dev);
3151
3152 CTX_LOCK(ctx);
3153 error = IFDI_IOV_INIT(ctx, num_vfs, params);
3154 CTX_UNLOCK(ctx);
3155
3156 return (error);
3157 }
3158
3159 void
iflib_device_iov_uninit(device_t dev)3160 iflib_device_iov_uninit(device_t dev)
3161 {
3162 if_ctx_t ctx = device_get_softc(dev);
3163
3164 CTX_LOCK(ctx);
3165 IFDI_IOV_UNINIT(ctx);
3166 CTX_UNLOCK(ctx);
3167 }
3168
3169 int
iflib_device_iov_add_vf(device_t dev,uint16_t vfnum,const nvlist_t * params)3170 iflib_device_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params)
3171 {
3172 int error;
3173 if_ctx_t ctx = device_get_softc(dev);
3174
3175 CTX_LOCK(ctx);
3176 error = IFDI_IOV_VF_ADD(ctx, vfnum, params);
3177 CTX_UNLOCK(ctx);
3178
3179 return (error);
3180 }
3181
3182 /*********************************************************************
3183 *
3184 * MODULE FUNCTION DEFINITIONS
3185 *
3186 **********************************************************************/
3187
3188 /*
3189 * - Start a fast taskqueue thread for each core
3190 * - Start a taskqueue for control operations
3191 */
3192 static int
iflib_module_init(void)3193 iflib_module_init(void)
3194 {
3195
3196 gctx = &global_ctx;
3197 gctx->igc_io_tqg = qgroup_if_io_tqg;
3198 gctx->igc_config_tqg = qgroup_if_config_tqg;
3199
3200 return (0);
3201 }
3202
3203 static int
iflib_module_event_handler(module_t mod,int what,void * arg)3204 iflib_module_event_handler(module_t mod, int what, void *arg)
3205 {
3206 int err;
3207
3208 switch (what) {
3209 case MOD_LOAD:
3210 if ((err = iflib_module_init()) != 0)
3211 return (err);
3212 break;
3213 case MOD_UNLOAD:
3214 return (EBUSY);
3215 default:
3216 return (EOPNOTSUPP);
3217 }
3218
3219 return (0);
3220 }
3221
3222 /*********************************************************************
3223 *
3224 * PUBLIC FUNCTION DEFINITIONS
3225 * ordered as in iflib.h
3226 *
3227 **********************************************************************/
3228
3229
3230 static void
_iflib_assert(if_shared_ctx_t sctx)3231 _iflib_assert(if_shared_ctx_t sctx)
3232 {
3233 MPASS(sctx->isc_tx_maxsize);
3234 MPASS(sctx->isc_tx_maxsegsize);
3235
3236 MPASS(sctx->isc_rx_maxsize);
3237 MPASS(sctx->isc_rx_nsegments);
3238 MPASS(sctx->isc_rx_maxsegsize);
3239
3240 MPASS(sctx->isc_txrx->ift_txd_encap);
3241 MPASS(sctx->isc_txrx->ift_txd_flush);
3242 MPASS(sctx->isc_txrx->ift_txd_credits_update);
3243 MPASS(sctx->isc_txrx->ift_rxd_available);
3244 MPASS(sctx->isc_txrx->ift_rxd_pkt_get);
3245 MPASS(sctx->isc_txrx->ift_rxd_refill);
3246 MPASS(sctx->isc_txrx->ift_rxd_flush);
3247 MPASS(sctx->isc_nrxd);
3248 }
3249
3250 static int
iflib_register(if_ctx_t ctx)3251 iflib_register(if_ctx_t ctx)
3252 {
3253 if_shared_ctx_t sctx = ctx->ifc_sctx;
3254 driver_t *driver = sctx->isc_driver;
3255 device_t dev = ctx->ifc_dev;
3256 if_t ifp;
3257
3258 _iflib_assert(sctx);
3259
3260 CTX_LOCK_INIT(ctx, device_get_nameunit(ctx->ifc_dev));
3261 MPASS(ctx->ifc_flags == 0);
3262
3263 ifp = ctx->ifc_ifp = if_gethandle(IFT_ETHER);
3264 if (ifp == NULL) {
3265 device_printf(dev, "can not allocate ifnet structure\n");
3266 return (ENOMEM);
3267 }
3268
3269 /*
3270 * Initialize our context's device specific methods
3271 */
3272 kobj_init((kobj_t) ctx, (kobj_class_t) driver);
3273 kobj_class_compile((kobj_class_t) driver);
3274 driver->refs++;
3275
3276 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
3277 if_setsoftc(ifp, ctx);
3278 if_setdev(ifp, dev);
3279 if_setinitfn(ifp, iflib_if_init);
3280 if_setioctlfn(ifp, iflib_if_ioctl);
3281 if_settransmitfn(ifp, iflib_if_transmit);
3282 if_setqflushfn(ifp, iflib_if_qflush);
3283 if_setgetcounterfn(ifp, iflib_if_get_counter);
3284 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
3285
3286 if_setcapabilities(ifp, 0);
3287 if_setcapenable(ifp, 0);
3288
3289 ctx->ifc_vlan_attach_event =
3290 EVENTHANDLER_REGISTER(vlan_config, iflib_vlan_register, ctx,
3291 EVENTHANDLER_PRI_FIRST);
3292 ctx->ifc_vlan_detach_event =
3293 EVENTHANDLER_REGISTER(vlan_unconfig, iflib_vlan_unregister, ctx,
3294 EVENTHANDLER_PRI_FIRST);
3295
3296 ifmedia_init(&ctx->ifc_media, IFM_IMASK,
3297 iflib_media_change, iflib_media_status);
3298
3299 return (0);
3300 }
3301
3302
3303 static int
iflib_queues_alloc(if_ctx_t ctx)3304 iflib_queues_alloc(if_ctx_t ctx)
3305 {
3306 if_shared_ctx_t sctx = ctx->ifc_sctx;
3307 device_t dev = ctx->ifc_dev;
3308 int nqsets = ctx->ifc_softc_ctx.isc_nqsets;
3309 iflib_txq_t txq;
3310 iflib_rxq_t rxq;
3311 iflib_qset_t qset;
3312 iflib_fl_t fl = NULL;
3313 int i, j, err, txconf, rxconf;
3314 iflib_dma_info_t ifdip;
3315 uint32_t *qsizes = sctx->isc_qsizes;
3316 uint8_t nqs = sctx->isc_nqs;
3317 int nfree_lists = sctx->isc_nfl ? sctx->isc_nfl : 1;
3318 caddr_t *vaddrs;
3319 uint64_t *paddrs;
3320 struct mp_ring **brscp;
3321 int nbuf_rings = 1; /* XXX determine dynamically */
3322
3323 KASSERT(nqs > 0, ("number of queues must be at least 1"));
3324
3325 if (!(qset =
3326 (iflib_qset_t) malloc(sizeof(struct iflib_qset) *
3327 nqsets, M_IFLIB, M_NOWAIT | M_ZERO))) {
3328 device_printf(dev, "Unable to allocate TX ring memory\n");
3329 err = ENOMEM;
3330 goto fail;
3331 }
3332
3333 /* Allocate the TX ring struct memory */
3334 if (!(txq =
3335 (iflib_txq_t) malloc(sizeof(struct iflib_txq) *
3336 nqsets, M_IFLIB, M_NOWAIT | M_ZERO))) {
3337 device_printf(dev, "Unable to allocate TX ring memory\n");
3338 err = ENOMEM;
3339 goto fail;
3340 }
3341
3342 /* Now allocate the RX */
3343 if (!(rxq =
3344 (iflib_rxq_t) malloc(sizeof(struct iflib_rxq) *
3345 nqsets, M_IFLIB, M_NOWAIT | M_ZERO))) {
3346 device_printf(dev, "Unable to allocate RX ring memory\n");
3347 err = ENOMEM;
3348 goto rx_fail;
3349 }
3350 if (!(brscp = malloc(sizeof(void *) * nbuf_rings * nqsets, M_IFLIB, M_NOWAIT | M_ZERO))) {
3351 device_printf(dev, "Unable to buf_ring_sc * memory\n");
3352 err = ENOMEM;
3353 goto rx_fail;
3354 }
3355
3356 ctx->ifc_qsets = qset;
3357 ctx->ifc_txqs = txq;
3358 ctx->ifc_rxqs = rxq;
3359
3360 /*
3361 * XXX handle allocation failure
3362 */
3363 for (qset = ctx->ifc_qsets, rxconf = txconf = i = 0; i < nqsets;
3364 i++, txconf++, rxconf++, qset++, txq++, rxq++) {
3365 /* Set up some basics */
3366
3367 if ((ifdip = malloc(sizeof(struct iflib_dma_info) * nqs, M_IFLIB, M_WAITOK|M_ZERO)) == NULL) {
3368 device_printf(dev, "failed to allocate iflib_dma_info\n");
3369 err = ENOMEM;
3370 goto fail;
3371 }
3372 qset->ifq_ifdi = ifdip;
3373 qset->ifq_nhwqs = nqs;
3374 for (j = 0; j < nqs; j++, ifdip++) {
3375 if (iflib_dma_alloc(ctx, qsizes[j], ifdip, BUS_DMA_NOWAIT)) {
3376 device_printf(dev, "Unable to allocate Descriptor memory\n");
3377 err = ENOMEM;
3378 goto err_tx_desc;
3379 }
3380 bzero((void *)ifdip->idi_vaddr, qsizes[j]);
3381 }
3382 txq->ift_ctx = ctx;
3383 txq->ift_id = i;
3384 /* XXX fix this */
3385 txq->ift_timer.c_cpu = i % mp_ncpus;
3386 txq->ift_db_check.c_cpu = i % mp_ncpus;
3387 txq->ift_nbr = nbuf_rings;
3388 txq->ift_ifdi = &qset->ifq_ifdi[0];
3389
3390 if (iflib_txsd_alloc(txq)) {
3391 device_printf(dev,
3392 "Critical Failure setting up transmit buffers\n");
3393 err = ENOMEM;
3394 goto err_tx_desc;
3395 }
3396
3397 /* Initialize the TX lock */
3398 snprintf(txq->ift_mtx_name, MTX_NAME_LEN, "%s:tx(%d)",
3399 device_get_nameunit(dev), txq->ift_id);
3400 mtx_init(&txq->ift_mtx, txq->ift_mtx_name, NULL, MTX_DEF);
3401 callout_init_mtx(&txq->ift_timer, &txq->ift_mtx, 0);
3402 callout_init_mtx(&txq->ift_db_check, &txq->ift_mtx, 0);
3403
3404 /* Allocate a buf ring */
3405 txq->ift_br = brscp + i*nbuf_rings;
3406 for (j = 0; j < nbuf_rings; j++) {
3407 err = mp_ring_alloc(&txq->ift_br[j], 2048, txq, iflib_txq_drain,
3408 iflib_txq_can_drain, M_IFLIB, M_WAITOK);
3409 if (err) {
3410 /* XXX free any allocated rings */
3411 device_printf(dev, "Unable to allocate buf_ring\n");
3412 goto fail;
3413 }
3414 }
3415 /*
3416 * Next the RX queues...
3417 */
3418 rxq->ifr_ctx = ctx;
3419 rxq->ifr_id = i;
3420 rxq->ifr_ifdi = &qset->ifq_ifdi[1];
3421 rxq->ifr_nfl = nfree_lists;
3422 if (!(fl =
3423 (iflib_fl_t) malloc(sizeof(struct iflib_fl) * nfree_lists, M_IFLIB, M_NOWAIT | M_ZERO))) {
3424 device_printf(dev, "Unable to allocate free list memory\n");
3425 err = ENOMEM;
3426 goto fail;
3427 }
3428 rxq->ifr_fl = fl;
3429 for (j = 0; j < nfree_lists; j++) {
3430 rxq->ifr_fl[j].ifl_rxq = rxq;
3431 rxq->ifr_fl[j].ifl_id = j;
3432 }
3433 /* Allocate receive buffers for the ring*/
3434 if (iflib_rxsd_alloc(rxq)) {
3435 device_printf(dev,
3436 "Critical Failure setting up receive buffers\n");
3437 err = ENOMEM;
3438 goto err_rx_desc;
3439 }
3440
3441 /* Initialize the RX lock */
3442 snprintf(rxq->ifr_mtx_name, MTX_NAME_LEN, "%s:rx(%d)",
3443 device_get_nameunit(dev), rxq->ifr_id);
3444 mtx_init(&rxq->ifr_mtx, rxq->ifr_mtx_name, NULL, MTX_DEF);
3445 }
3446
3447 vaddrs = malloc(sizeof(caddr_t)*nqsets*nqs, M_IFLIB, M_WAITOK);
3448 paddrs = malloc(sizeof(uint64_t)*nqsets*nqs, M_IFLIB, M_WAITOK);
3449 for (i = 0; i < nqsets; i++) {
3450 iflib_dma_info_t di = ctx->ifc_qsets[i].ifq_ifdi;
3451
3452 for (j = 0; j < nqs; j++, di++) {
3453 vaddrs[i*nqs + j] = di->idi_vaddr;
3454 paddrs[i*nqs + j] = di->idi_paddr;
3455 }
3456 }
3457 if ((err = IFDI_QUEUES_ALLOC(ctx, vaddrs, paddrs, nqs)) != 0) {
3458 device_printf(ctx->ifc_dev, "device queue allocation failed\n");
3459 iflib_tx_structures_free(ctx);
3460 free(vaddrs, M_IFLIB);
3461 free(paddrs, M_IFLIB);
3462 goto err_rx_desc;
3463 }
3464 free(vaddrs, M_IFLIB);
3465 free(paddrs, M_IFLIB);
3466
3467 return (0);
3468 err_rx_desc:
3469 err_tx_desc:
3470 if (ctx->ifc_rxqs != NULL)
3471 free(ctx->ifc_rxqs, M_IFLIB);
3472 ctx->ifc_rxqs = NULL;
3473 rx_fail:
3474 if (ctx->ifc_txqs != NULL)
3475 free(ctx->ifc_txqs, M_IFLIB);
3476 ctx->ifc_txqs = NULL;
3477 fail:
3478 return (err);
3479 }
3480
3481 static int
iflib_tx_structures_setup(if_ctx_t ctx)3482 iflib_tx_structures_setup(if_ctx_t ctx)
3483 {
3484 iflib_txq_t txq = ctx->ifc_txqs;
3485 int i;
3486
3487 for (i = 0; i < ctx->ifc_softc_ctx.isc_nqsets; i++, txq++)
3488 iflib_txq_setup(txq);
3489
3490 return (0);
3491 }
3492
3493 static void
iflib_tx_structures_free(if_ctx_t ctx)3494 iflib_tx_structures_free(if_ctx_t ctx)
3495 {
3496 iflib_txq_t txq = ctx->ifc_txqs;
3497 iflib_qset_t qset = ctx->ifc_qsets;
3498 int i, j;
3499
3500 for (i = 0; i < ctx->ifc_softc_ctx.isc_nqsets; i++, txq++, qset++) {
3501 iflib_txq_destroy(txq);
3502 for (j = 0; j < qset->ifq_nhwqs; j++)
3503 iflib_dma_free(&qset->ifq_ifdi[j]);
3504 }
3505 free(ctx->ifc_txqs, M_IFLIB);
3506 free(ctx->ifc_qsets, M_IFLIB);
3507 ctx->ifc_txqs = NULL;
3508 ctx->ifc_qsets = NULL;
3509 IFDI_QUEUES_FREE(ctx);
3510 }
3511
3512 /*********************************************************************
3513 *
3514 * Initialize all receive rings.
3515 *
3516 **********************************************************************/
3517 static int
iflib_rx_structures_setup(if_ctx_t ctx)3518 iflib_rx_structures_setup(if_ctx_t ctx)
3519 {
3520 iflib_rxq_t rxq = ctx->ifc_rxqs;
3521 int i, q, err;
3522
3523 for (q = 0; q < ctx->ifc_softc_ctx.isc_nqsets; q++, rxq++) {
3524 tcp_lro_free(&rxq->ifr_lc);
3525 if (ctx->ifc_ifp->if_capenable & IFCAP_LRO) {
3526 if ((err = tcp_lro_init(&rxq->ifr_lc)) != 0) {
3527 device_printf(ctx->ifc_dev, "LRO Initialization failed!\n");
3528 goto fail;
3529 }
3530 rxq->ifr_lro_enabled = TRUE;
3531 rxq->ifr_lc.ifp = ctx->ifc_ifp;
3532 }
3533
3534 IFDI_RXQ_SETUP(ctx, rxq->ifr_id);
3535 }
3536 return (0);
3537 fail:
3538 /*
3539 * Free RX software descriptors allocated so far, we will only handle
3540 * the rings that completed, the failing case will have
3541 * cleaned up for itself. 'q' failed, so its the terminus.
3542 */
3543 rxq = ctx->ifc_rxqs;
3544 for (i = 0; i < q; ++i, rxq++) {
3545 iflib_rx_sds_free(rxq);
3546 rxq->ifr_gen = rxq->ifr_cidx = rxq->ifr_pidx = 0;
3547 }
3548 return (err);
3549 }
3550
3551 /*********************************************************************
3552 *
3553 * Free all receive rings.
3554 *
3555 **********************************************************************/
3556 static void
iflib_rx_structures_free(if_ctx_t ctx)3557 iflib_rx_structures_free(if_ctx_t ctx)
3558 {
3559 iflib_rxq_t rxq = ctx->ifc_rxqs;
3560
3561 for (int i = 0; i < ctx->ifc_softc_ctx.isc_nqsets; i++, rxq++) {
3562 iflib_rx_sds_free(rxq);
3563 }
3564 }
3565
3566 static int
iflib_qset_structures_setup(if_ctx_t ctx)3567 iflib_qset_structures_setup(if_ctx_t ctx)
3568 {
3569 int err;
3570
3571 if ((err = iflib_tx_structures_setup(ctx)) != 0)
3572 return (err);
3573
3574 if ((err = iflib_rx_structures_setup(ctx)) != 0) {
3575 device_printf(ctx->ifc_dev, "iflib_rx_structures_setup failed: %d\n", err);
3576 iflib_tx_structures_free(ctx);
3577 iflib_rx_structures_free(ctx);
3578 }
3579 return (err);
3580 }
3581
3582 int
iflib_irq_alloc(if_ctx_t ctx,if_irq_t irq,int rid,driver_filter_t filter,void * filter_arg,driver_intr_t handler,void * arg,char * name)3583 iflib_irq_alloc(if_ctx_t ctx, if_irq_t irq, int rid,
3584 driver_filter_t filter, void *filter_arg, driver_intr_t handler, void *arg, char *name)
3585 {
3586
3587 return (_iflib_irq_alloc(ctx, irq, rid, filter, handler, arg, name));
3588 }
3589
3590 int
iflib_irq_alloc_generic(if_ctx_t ctx,if_irq_t irq,int rid,iflib_intr_type_t type,driver_filter_t * filter,void * filter_arg,int qid,char * name)3591 iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid,
3592 iflib_intr_type_t type, driver_filter_t *filter,
3593 void *filter_arg, int qid, char *name)
3594 {
3595 struct grouptask *gtask;
3596 struct taskqgroup *tqg;
3597 iflib_filter_info_t info;
3598 cpuset_t cpus;
3599 task_fn_t *fn;
3600 int tqrid;
3601 void *q;
3602 int err, i, cpuid;
3603
3604 info = &ctx->ifc_filter_info;
3605
3606 switch (type) {
3607 /* XXX merge tx/rx for netmap? */
3608 case IFLIB_INTR_TX:
3609 q = &ctx->ifc_txqs[qid];
3610 info = &ctx->ifc_txqs[qid].ift_filter_info;
3611 gtask = &ctx->ifc_txqs[qid].ift_task;
3612 tqg = gctx->igc_io_tqg;
3613 tqrid = irq->ii_rid;
3614 fn = _task_fn_tx;
3615 break;
3616 case IFLIB_INTR_RX:
3617 q = &ctx->ifc_rxqs[qid];
3618 info = &ctx->ifc_rxqs[qid].ifr_filter_info;
3619 gtask = &ctx->ifc_rxqs[qid].ifr_task;
3620 tqg = gctx->igc_io_tqg;
3621 tqrid = irq->ii_rid;
3622 fn = _task_fn_rx;
3623 break;
3624 case IFLIB_INTR_ADMIN:
3625 q = ctx;
3626 info = &ctx->ifc_filter_info;
3627 gtask = &ctx->ifc_admin_task;
3628 tqg = gctx->igc_config_tqg;
3629 tqrid = -1;
3630 fn = _task_fn_admin;
3631 break;
3632 default:
3633 panic("unknown net intr type");
3634 }
3635 GROUPTASK_INIT(gtask, 0, fn, q);
3636
3637 info->ifi_filter = filter;
3638 info->ifi_filter_arg = filter_arg;
3639 info->ifi_task = gtask;
3640
3641 /* XXX query cpu that rid belongs to */
3642
3643 err = _iflib_irq_alloc(ctx, irq, rid, iflib_fast_intr, NULL, info, name);
3644 if (err != 0)
3645 return (err);
3646 if (tqrid != -1 && CPU_COUNT(&ctx->ifc_cpus) > qid) {
3647 CPU_COPY(&ctx->ifc_cpus, &cpus);
3648 for (i = 0; i < qid; i++) {
3649 cpuid = CPU_FFS(&cpus);
3650 CPU_CLR(cpuid, &cpus);
3651 }
3652 taskqgroup_attach_cpu(tqg, gtask, q, CPU_FFS(&cpus), irq->ii_rid, name);
3653 } else
3654 taskqgroup_attach(tqg, gtask, q, tqrid, name);
3655
3656
3657 return (0);
3658 }
3659
3660 void
iflib_softirq_alloc_generic(if_ctx_t ctx,int rid,iflib_intr_type_t type,void * arg,int qid,char * name)3661 iflib_softirq_alloc_generic(if_ctx_t ctx, int rid, iflib_intr_type_t type, void *arg, int qid, char *name)
3662 {
3663 struct grouptask *gtask;
3664 struct taskqgroup *tqg;
3665 task_fn_t *fn;
3666 void *q;
3667
3668 switch (type) {
3669 case IFLIB_INTR_TX:
3670 q = &ctx->ifc_txqs[qid];
3671 gtask = &ctx->ifc_txqs[qid].ift_task;
3672 tqg = gctx->igc_io_tqg;
3673 fn = _task_fn_tx;
3674 break;
3675 case IFLIB_INTR_RX:
3676 q = &ctx->ifc_rxqs[qid];
3677 gtask = &ctx->ifc_rxqs[qid].ifr_task;
3678 tqg = gctx->igc_io_tqg;
3679 fn = _task_fn_rx;
3680 break;
3681 case IFLIB_INTR_ADMIN:
3682 q = ctx;
3683 gtask = &ctx->ifc_admin_task;
3684 tqg = gctx->igc_config_tqg;
3685 rid = -1;
3686 fn = _task_fn_admin;
3687 break;
3688 case IFLIB_INTR_IOV:
3689 q = ctx;
3690 gtask = &ctx->ifc_vflr_task;
3691 tqg = gctx->igc_config_tqg;
3692 rid = -1;
3693 fn = _task_fn_iov;
3694 break;
3695 default:
3696 panic("unknown net intr type");
3697 }
3698 GROUPTASK_INIT(gtask, 0, fn, q);
3699 taskqgroup_attach(tqg, gtask, q, rid, name);
3700 }
3701
3702 void
iflib_irq_free(if_ctx_t ctx,if_irq_t irq)3703 iflib_irq_free(if_ctx_t ctx, if_irq_t irq)
3704 {
3705 if (irq->ii_tag)
3706 bus_teardown_intr(ctx->ifc_dev, irq->ii_res, irq->ii_tag);
3707
3708 if (irq->ii_res)
3709 bus_release_resource(ctx->ifc_dev, SYS_RES_IRQ, irq->ii_rid, irq->ii_res);
3710 }
3711
3712 static int
iflib_legacy_setup(if_ctx_t ctx,driver_filter_t filter,void * filter_arg,int * rid,char * name)3713 iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filter_arg, int *rid, char *name)
3714 {
3715 iflib_txq_t txq = ctx->ifc_txqs;
3716 iflib_rxq_t rxq = ctx->ifc_rxqs;
3717 if_irq_t irq = &ctx->ifc_legacy_irq;
3718 iflib_filter_info_t info;
3719 struct grouptask *gtask;
3720 struct taskqgroup *tqg;
3721 task_fn_t *fn;
3722 int tqrid;
3723 void *q;
3724 int err;
3725
3726 q = &ctx->ifc_rxqs[0];
3727 info = &rxq[0].ifr_filter_info;
3728 gtask = &rxq[0].ifr_task;
3729 tqg = gctx->igc_io_tqg;
3730 tqrid = irq->ii_rid = *rid;
3731 fn = _task_fn_rx;
3732
3733 ctx->ifc_flags |= IFC_LEGACY;
3734 info->ifi_filter = filter;
3735 info->ifi_filter_arg = filter_arg;
3736 info->ifi_task = gtask;
3737
3738 /* We allocate a single interrupt resource */
3739 if ((err = _iflib_irq_alloc(ctx, irq, tqrid, iflib_fast_intr, NULL, info, name)) != 0)
3740 return (err);
3741 GROUPTASK_INIT(gtask, 0, fn, q);
3742 taskqgroup_attach(tqg, gtask, q, tqrid, name);
3743
3744 GROUPTASK_INIT(&txq->ift_task, 0, _task_fn_tx, txq);
3745 taskqgroup_attach(gctx->igc_io_tqg, &txq->ift_task, txq, tqrid, "tx");
3746 GROUPTASK_INIT(&ctx->ifc_admin_task, 0, _task_fn_admin, ctx);
3747 taskqgroup_attach(gctx->igc_config_tqg, &ctx->ifc_admin_task, ctx, -1, "admin/link");
3748
3749 return (0);
3750 }
3751
3752 void
iflib_led_create(if_ctx_t ctx)3753 iflib_led_create(if_ctx_t ctx)
3754 {
3755
3756 ctx->ifc_led_dev = led_create(iflib_led_func, ctx,
3757 device_get_nameunit(ctx->ifc_dev));
3758 }
3759
3760 void
iflib_tx_intr_deferred(if_ctx_t ctx,int txqid)3761 iflib_tx_intr_deferred(if_ctx_t ctx, int txqid)
3762 {
3763
3764 GROUPTASK_ENQUEUE(&ctx->ifc_txqs[txqid].ift_task);
3765 }
3766
3767 void
iflib_rx_intr_deferred(if_ctx_t ctx,int rxqid)3768 iflib_rx_intr_deferred(if_ctx_t ctx, int rxqid)
3769 {
3770
3771 GROUPTASK_ENQUEUE(&ctx->ifc_rxqs[rxqid].ifr_task);
3772 }
3773
3774 void
iflib_admin_intr_deferred(if_ctx_t ctx)3775 iflib_admin_intr_deferred(if_ctx_t ctx)
3776 {
3777
3778 GROUPTASK_ENQUEUE(&ctx->ifc_admin_task);
3779 }
3780
3781 void
iflib_iov_intr_deferred(if_ctx_t ctx)3782 iflib_iov_intr_deferred(if_ctx_t ctx)
3783 {
3784
3785 GROUPTASK_ENQUEUE(&ctx->ifc_vflr_task);
3786 }
3787
3788 void
iflib_io_tqg_attach(struct grouptask * gt,void * uniq,int cpu,char * name)3789 iflib_io_tqg_attach(struct grouptask *gt, void *uniq, int cpu, char *name)
3790 {
3791
3792 taskqgroup_attach_cpu(gctx->igc_io_tqg, gt, uniq, cpu, -1, name);
3793 }
3794
3795 void
iflib_config_gtask_init(if_ctx_t ctx,struct grouptask * gtask,task_fn_t * fn,char * name)3796 iflib_config_gtask_init(if_ctx_t ctx, struct grouptask *gtask, task_fn_t *fn,
3797 char *name)
3798 {
3799
3800 GROUPTASK_INIT(gtask, 0, fn, ctx);
3801 taskqgroup_attach(gctx->igc_config_tqg, gtask, gtask, -1, name);
3802 }
3803
3804 void
iflib_link_state_change(if_ctx_t ctx,int link_state)3805 iflib_link_state_change(if_ctx_t ctx, int link_state)
3806 {
3807 if_t ifp = ctx->ifc_ifp;
3808 iflib_txq_t txq = ctx->ifc_txqs;
3809
3810 #if 0
3811 if_setbaudrate(ifp, baudrate);
3812 #endif
3813 /* If link down, disable watchdog */
3814 if ((ctx->ifc_link_state == LINK_STATE_UP) && (link_state == LINK_STATE_DOWN)) {
3815 for (int i = 0; i < ctx->ifc_softc_ctx.isc_nqsets; i++, txq++)
3816 txq->ift_qstatus = IFLIB_QUEUE_IDLE;
3817 }
3818 ctx->ifc_link_state = link_state;
3819 if_link_state_change(ifp, link_state);
3820 }
3821
3822 static int
iflib_tx_credits_update(if_ctx_t ctx,iflib_txq_t txq)3823 iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq)
3824 {
3825 int credits;
3826
3827 if (ctx->isc_txd_credits_update == NULL)
3828 return (0);
3829
3830 if ((credits = ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, txq->ift_cidx_processed)) == 0)
3831 return (0);
3832
3833 txq->ift_processed += credits;
3834 txq->ift_cidx_processed += credits;
3835
3836 if (txq->ift_cidx_processed >= txq->ift_size)
3837 txq->ift_cidx_processed -= txq->ift_size;
3838 return (credits);
3839 }
3840
3841 static int
iflib_rxd_avail(if_ctx_t ctx,iflib_rxq_t rxq,int cidx)3842 iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, int cidx)
3843 {
3844 int avail;
3845
3846 avail = ctx->isc_rxd_available(ctx->ifc_softc, rxq->ifr_id, cidx);
3847 #if 0
3848 rxq->ifr_pidx += avail;
3849 if (rxq->ifr_pidx >= rxq->ifr_size) {
3850 rxq->ifr_pidx -= rxq->ifr_size;
3851 rxq->ifr_gen = 1;
3852 }
3853
3854 return (get_inuse(rxq->ifr_size, rxq->ifr_cidx, rxq->ifr_pidx, rxq->ifr_gen));
3855 #endif
3856 return (avail);
3857 }
3858
3859 void
iflib_add_int_delay_sysctl(if_ctx_t ctx,const char * name,const char * description,if_int_delay_info_t info,int offset,int value)3860 iflib_add_int_delay_sysctl(if_ctx_t ctx, const char *name,
3861 const char *description, if_int_delay_info_t info,
3862 int offset, int value)
3863 {
3864 info->iidi_ctx = ctx;
3865 info->iidi_offset = offset;
3866 info->iidi_value = value;
3867 SYSCTL_ADD_PROC(device_get_sysctl_ctx(ctx->ifc_dev),
3868 SYSCTL_CHILDREN(device_get_sysctl_tree(ctx->ifc_dev)),
3869 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
3870 info, 0, iflib_sysctl_int_delay, "I", description);
3871 }
3872
3873 struct mtx *
iflib_ctx_lock_get(if_ctx_t ctx)3874 iflib_ctx_lock_get(if_ctx_t ctx)
3875 {
3876
3877 return (&ctx->ifc_mtx);
3878 }
3879
3880 static int
iflib_msix_init(if_ctx_t ctx)3881 iflib_msix_init(if_ctx_t ctx)
3882 {
3883 device_t dev = ctx->ifc_dev;
3884 if_shared_ctx_t sctx = ctx->ifc_sctx;
3885 if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
3886 int vectors, queues, queuemsgs, msgs;
3887 int err, admincnt, bar;
3888
3889 bar = ctx->ifc_softc_ctx.isc_msix_bar;
3890 admincnt = sctx->isc_admin_intrcnt;
3891 /* Override by tuneable */
3892 if (enable_msix == 0)
3893 goto msi;
3894
3895 /*
3896 ** When used in a virtualized environment
3897 ** PCI BUSMASTER capability may not be set
3898 ** so explicity set it here and rewrite
3899 ** the ENABLE in the MSIX control register
3900 ** at this point to cause the host to
3901 ** successfully initialize us.
3902 */
3903 {
3904 uint16_t pci_cmd_word;
3905 int msix_ctrl, rid;
3906
3907 rid = 0;
3908 pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
3909 pci_cmd_word |= PCIM_CMD_BUSMASTEREN;
3910 pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2);
3911 pci_find_cap(dev, PCIY_MSIX, &rid);
3912 rid += PCIR_MSIX_CTRL;
3913 msix_ctrl = pci_read_config(dev, rid, 2);
3914 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
3915 pci_write_config(dev, rid, msix_ctrl, 2);
3916 }
3917
3918 /* First try MSI/X */
3919 ctx->ifc_msix_mem = bus_alloc_resource_any(dev,
3920 SYS_RES_MEMORY, &bar, RF_ACTIVE);
3921 if (ctx->ifc_msix_mem == NULL) {
3922 /* May not be enabled */
3923 device_printf(dev, "Unable to map MSIX table \n");
3924 goto msi;
3925 }
3926
3927 if ((msgs = pci_msix_count(dev)) == 0) { /* system has msix disabled */
3928 device_printf(dev, "System has MSIX disabled \n");
3929 bus_release_resource(dev, SYS_RES_MEMORY,
3930 bar, ctx->ifc_msix_mem);
3931 ctx->ifc_msix_mem = NULL;
3932 goto msi;
3933 }
3934 #if IFLIB_DEBUG
3935 /* use only 1 qset in debug mode */
3936 queuemsgs = min(msgs - admincnt, 1);
3937 #else
3938 queuemsgs = msgs - admincnt;
3939 #endif
3940 if (bus_get_cpus(dev, INTR_CPUS, &ctx->ifc_cpus) == 0) {
3941 #ifdef RSS
3942 queues = imin(queuemsgs, rss_getnumbuckets());
3943 #else
3944 queues = queuemsgs;
3945 #endif
3946 queues = imin(CPU_COUNT(&ctx->ifc_cpus), queues);
3947 device_printf(dev, "pxm cpus: %d queue msgs: %d admincnt: %d\n",
3948 CPU_COUNT(&ctx->ifc_cpus), queuemsgs, admincnt);
3949 } else {
3950 device_printf(dev, "Unable to fetch CPU list\n");
3951 /* Figure out a reasonable auto config value */
3952 queues = min(queuemsgs, mp_ncpus);
3953 device_printf(dev, "using %d queues\n", queues);
3954 }
3955 #ifdef RSS
3956 /* If we're doing RSS, clamp at the number of RSS buckets */
3957 if (queues > rss_getnumbuckets())
3958 queues = rss_getnumbuckets();
3959 #endif
3960
3961 vectors = queues + admincnt;
3962 if ((err = pci_alloc_msix(dev, &vectors)) == 0) {
3963 device_printf(dev,
3964 "Using MSIX interrupts with %d vectors\n", vectors);
3965 scctx->isc_vectors = vectors;
3966 scctx->isc_nqsets = queues;
3967 scctx->isc_intr = IFLIB_INTR_MSIX;
3968 return (vectors);
3969 } else {
3970 device_printf(dev, "failed to allocate %d msix vectors, err: %d - using MSI\n", vectors, err);
3971 }
3972 msi:
3973 vectors = pci_msi_count(dev);
3974 scctx->isc_nqsets = 1;
3975 scctx->isc_vectors = vectors;
3976 if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0) {
3977 device_printf(dev,"Using an MSI interrupt\n");
3978 scctx->isc_intr = IFLIB_INTR_MSI;
3979 } else {
3980 device_printf(dev,"Using a Legacy interrupt\n");
3981 scctx->isc_intr = IFLIB_INTR_LEGACY;
3982 }
3983
3984 return (vectors);
3985 }
3986