1 /*-
2 * Copyright (c) 2016, Vincenzo Maffione
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 /* Driver for ptnet paravirtualized network device. */
28
29 #include <sys/cdefs.h>
30
31 #include <sys/types.h>
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/kernel.h>
35 #include <sys/sockio.h>
36 #include <sys/mbuf.h>
37 #include <sys/malloc.h>
38 #include <sys/module.h>
39 #include <sys/socket.h>
40 #include <sys/sysctl.h>
41 #include <sys/lock.h>
42 #include <sys/mutex.h>
43 #include <sys/taskqueue.h>
44 #include <sys/smp.h>
45 #include <sys/time.h>
46 #include <machine/smp.h>
47
48 #include <vm/uma.h>
49 #include <vm/vm.h>
50 #include <vm/pmap.h>
51
52 #include <net/ethernet.h>
53 #include <net/if.h>
54 #include <net/if_var.h>
55 #include <net/if_arp.h>
56 #include <net/if_dl.h>
57 #include <net/if_types.h>
58 #include <net/if_media.h>
59 #include <net/if_vlan_var.h>
60 #include <net/bpf.h>
61
62 #include <netinet/in_systm.h>
63 #include <netinet/in.h>
64 #include <netinet/ip.h>
65 #include <netinet/ip6.h>
66 #include <netinet6/ip6_var.h>
67 #include <netinet/udp.h>
68 #include <netinet/tcp.h>
69
70 #include <machine/bus.h>
71 #include <machine/resource.h>
72 #include <sys/bus.h>
73 #include <sys/rman.h>
74
75 #include <dev/pci/pcivar.h>
76 #include <dev/pci/pcireg.h>
77
78 #include "opt_inet.h"
79 #include "opt_inet6.h"
80
81 #include <sys/selinfo.h>
82 #include <net/netmap.h>
83 #include <dev/netmap/netmap_kern.h>
84 #include <net/netmap_virt.h>
85 #include <dev/netmap/netmap_mem2.h>
86 #include <dev/virtio/network/virtio_net.h>
87
88 #ifdef WITH_PTNETMAP
89
90 #ifndef INET
91 #error "INET not defined, cannot support offloadings"
92 #endif
93
94 static uint64_t ptnet_get_counter(if_t, ift_counter);
95
96 //#define PTNETMAP_STATS
97 //#define DEBUG
98 #ifdef DEBUG
99 #define DBG(x) x
100 #else /* !DEBUG */
101 #define DBG(x)
102 #endif /* !DEBUG */
103
104 extern int ptnet_vnet_hdr; /* Tunable parameter */
105
106 struct ptnet_softc;
107
108 struct ptnet_queue_stats {
109 uint64_t packets; /* if_[io]packets */
110 uint64_t bytes; /* if_[io]bytes */
111 uint64_t errors; /* if_[io]errors */
112 uint64_t iqdrops; /* if_iqdrops */
113 uint64_t mcasts; /* if_[io]mcasts */
114 #ifdef PTNETMAP_STATS
115 uint64_t intrs;
116 uint64_t kicks;
117 #endif /* PTNETMAP_STATS */
118 };
119
120 struct ptnet_queue {
121 struct ptnet_softc *sc;
122 struct resource *irq;
123 void *cookie;
124 int kring_id;
125 struct nm_csb_atok *atok;
126 struct nm_csb_ktoa *ktoa;
127 unsigned int kick;
128 struct mtx lock;
129 struct buf_ring *bufring; /* for TX queues */
130 struct ptnet_queue_stats stats;
131 #ifdef PTNETMAP_STATS
132 struct ptnet_queue_stats last_stats;
133 #endif /* PTNETMAP_STATS */
134 struct taskqueue *taskq;
135 struct task task;
136 char lock_name[16];
137 };
138
139 #define PTNET_Q_LOCK(_pq) mtx_lock(&(_pq)->lock)
140 #define PTNET_Q_TRYLOCK(_pq) mtx_trylock(&(_pq)->lock)
141 #define PTNET_Q_UNLOCK(_pq) mtx_unlock(&(_pq)->lock)
142
143 struct ptnet_softc {
144 device_t dev;
145 if_t ifp;
146 struct ifmedia media;
147 struct mtx lock;
148 char lock_name[16];
149 char hwaddr[ETHER_ADDR_LEN];
150
151 /* Mirror of PTFEAT register. */
152 uint32_t ptfeatures;
153 unsigned int vnet_hdr_len;
154
155 /* PCI BARs support. */
156 struct resource *iomem;
157 struct resource *msix_mem;
158
159 unsigned int num_rings;
160 unsigned int num_tx_rings;
161 struct ptnet_queue *queues;
162 struct ptnet_queue *rxqueues;
163 struct nm_csb_atok *csb_gh;
164 struct nm_csb_ktoa *csb_hg;
165
166 unsigned int min_tx_space;
167
168 struct netmap_pt_guest_adapter *ptna;
169
170 struct callout tick;
171 #ifdef PTNETMAP_STATS
172 struct timeval last_ts;
173 #endif /* PTNETMAP_STATS */
174 };
175
176 #define PTNET_CORE_LOCK(_sc) mtx_lock(&(_sc)->lock)
177 #define PTNET_CORE_UNLOCK(_sc) mtx_unlock(&(_sc)->lock)
178
179 static int ptnet_probe(device_t);
180 static int ptnet_attach(device_t);
181 static int ptnet_detach(device_t);
182 static int ptnet_suspend(device_t);
183 static int ptnet_resume(device_t);
184 static int ptnet_shutdown(device_t);
185
186 static void ptnet_init(void *opaque);
187 static int ptnet_ioctl(if_t ifp, u_long cmd, caddr_t data);
188 static int ptnet_init_locked(struct ptnet_softc *sc);
189 static int ptnet_stop(struct ptnet_softc *sc);
190 static int ptnet_transmit(if_t ifp, struct mbuf *m);
191 static int ptnet_drain_transmit_queue(struct ptnet_queue *pq,
192 unsigned int budget,
193 bool may_resched);
194 static void ptnet_qflush(if_t ifp);
195 static void ptnet_tx_task(void *context, int pending);
196
197 static int ptnet_media_change(if_t ifp);
198 static void ptnet_media_status(if_t ifp, struct ifmediareq *ifmr);
199 #ifdef PTNETMAP_STATS
200 static void ptnet_tick(void *opaque);
201 #endif
202
203 static int ptnet_irqs_init(struct ptnet_softc *sc);
204 static void ptnet_irqs_fini(struct ptnet_softc *sc);
205
206 static uint32_t ptnet_nm_ptctl(struct ptnet_softc *sc, uint32_t cmd);
207 static int ptnet_nm_config(struct netmap_adapter *na,
208 struct nm_config_info *info);
209 static void ptnet_update_vnet_hdr(struct ptnet_softc *sc);
210 static int ptnet_nm_register(struct netmap_adapter *na, int onoff);
211 static int ptnet_nm_txsync(struct netmap_kring *kring, int flags);
212 static int ptnet_nm_rxsync(struct netmap_kring *kring, int flags);
213 static void ptnet_nm_intr(struct netmap_adapter *na, int onoff);
214
215 static void ptnet_tx_intr(void *opaque);
216 static void ptnet_rx_intr(void *opaque);
217
218 static unsigned ptnet_rx_discard(struct netmap_kring *kring,
219 unsigned int head);
220 static int ptnet_rx_eof(struct ptnet_queue *pq, unsigned int budget,
221 bool may_resched);
222 static void ptnet_rx_task(void *context, int pending);
223
224 #ifdef DEVICE_POLLING
225 static poll_handler_t ptnet_poll;
226 #endif
227
228 static device_method_t ptnet_methods[] = {
229 DEVMETHOD(device_probe, ptnet_probe),
230 DEVMETHOD(device_attach, ptnet_attach),
231 DEVMETHOD(device_detach, ptnet_detach),
232 DEVMETHOD(device_suspend, ptnet_suspend),
233 DEVMETHOD(device_resume, ptnet_resume),
234 DEVMETHOD(device_shutdown, ptnet_shutdown),
235 DEVMETHOD_END
236 };
237
238 static driver_t ptnet_driver = {
239 "ptnet",
240 ptnet_methods,
241 sizeof(struct ptnet_softc)
242 };
243
244 /* We use (SI_ORDER_MIDDLE+2) here, see DEV_MODULE_ORDERED() invocation. */
245 static devclass_t ptnet_devclass;
246 DRIVER_MODULE_ORDERED(ptnet, pci, ptnet_driver, ptnet_devclass,
247 NULL, NULL, SI_ORDER_MIDDLE + 2);
248
249 static int
ptnet_probe(device_t dev)250 ptnet_probe(device_t dev)
251 {
252 if (pci_get_vendor(dev) != PTNETMAP_PCI_VENDOR_ID ||
253 pci_get_device(dev) != PTNETMAP_PCI_NETIF_ID) {
254 return (ENXIO);
255 }
256
257 device_set_desc(dev, "ptnet network adapter");
258
259 return (BUS_PROBE_DEFAULT);
260 }
261
ptnet_kick(struct ptnet_queue * pq)262 static inline void ptnet_kick(struct ptnet_queue *pq)
263 {
264 #ifdef PTNETMAP_STATS
265 pq->stats.kicks ++;
266 #endif /* PTNETMAP_STATS */
267 bus_write_4(pq->sc->iomem, pq->kick, 0);
268 }
269
270 #define PTNET_BUF_RING_SIZE 4096
271 #define PTNET_RX_BUDGET 512
272 #define PTNET_RX_BATCH 1
273 #define PTNET_TX_BUDGET 512
274 #define PTNET_TX_BATCH 64
275 #define PTNET_HDR_SIZE sizeof(struct virtio_net_hdr_mrg_rxbuf)
276 #define PTNET_MAX_PKT_SIZE 65536
277
278 #define PTNET_CSUM_OFFLOAD (CSUM_TCP | CSUM_UDP)
279 #define PTNET_CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6 | CSUM_UDP_IPV6)
280 #define PTNET_ALL_OFFLOAD (CSUM_TSO | PTNET_CSUM_OFFLOAD |\
281 PTNET_CSUM_OFFLOAD_IPV6)
282
283 static int
ptnet_attach(device_t dev)284 ptnet_attach(device_t dev)
285 {
286 uint32_t ptfeatures = 0;
287 unsigned int num_rx_rings, num_tx_rings;
288 struct netmap_adapter na_arg;
289 unsigned int nifp_offset;
290 struct ptnet_softc *sc;
291 if_t ifp;
292 uint32_t macreg;
293 int err, rid;
294 int i;
295
296 sc = device_get_softc(dev);
297 sc->dev = dev;
298
299 /* Setup PCI resources. */
300 pci_enable_busmaster(dev);
301
302 rid = PCIR_BAR(PTNETMAP_IO_PCI_BAR);
303 sc->iomem = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
304 RF_ACTIVE);
305 if (sc->iomem == NULL) {
306 device_printf(dev, "Failed to map I/O BAR\n");
307 return (ENXIO);
308 }
309
310 /* Negotiate features with the hypervisor. */
311 if (ptnet_vnet_hdr) {
312 ptfeatures |= PTNETMAP_F_VNET_HDR;
313 }
314 bus_write_4(sc->iomem, PTNET_IO_PTFEAT, ptfeatures); /* wanted */
315 ptfeatures = bus_read_4(sc->iomem, PTNET_IO_PTFEAT); /* acked */
316 sc->ptfeatures = ptfeatures;
317
318 num_tx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_RINGS);
319 num_rx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_RINGS);
320 sc->num_rings = num_tx_rings + num_rx_rings;
321 sc->num_tx_rings = num_tx_rings;
322
323 if (sc->num_rings * sizeof(struct nm_csb_atok) > PAGE_SIZE) {
324 device_printf(dev, "CSB cannot handle that many rings (%u)\n",
325 sc->num_rings);
326 err = ENOMEM;
327 goto err_path;
328 }
329
330 /* Allocate CSB and carry out CSB allocation protocol. */
331 sc->csb_gh = contigmalloc(2*PAGE_SIZE, M_DEVBUF, M_NOWAIT | M_ZERO,
332 (size_t)0, -1UL, PAGE_SIZE, 0);
333 if (sc->csb_gh == NULL) {
334 device_printf(dev, "Failed to allocate CSB\n");
335 err = ENOMEM;
336 goto err_path;
337 }
338 sc->csb_hg = (struct nm_csb_ktoa *)(((char *)sc->csb_gh) + PAGE_SIZE);
339
340 {
341 /*
342 * We use uint64_t rather than vm_paddr_t since we
343 * need 64 bit addresses even on 32 bit platforms.
344 */
345 uint64_t paddr = vtophys(sc->csb_gh);
346
347 /* CSB allocation protocol: write to BAH first, then
348 * to BAL (for both GH and HG sections). */
349 bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAH,
350 (paddr >> 32) & 0xffffffff);
351 bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAL,
352 paddr & 0xffffffff);
353 paddr = vtophys(sc->csb_hg);
354 bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAH,
355 (paddr >> 32) & 0xffffffff);
356 bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAL,
357 paddr & 0xffffffff);
358 }
359
360 /* Allocate and initialize per-queue data structures. */
361 sc->queues = malloc(sizeof(struct ptnet_queue) * sc->num_rings,
362 M_DEVBUF, M_NOWAIT | M_ZERO);
363 if (sc->queues == NULL) {
364 err = ENOMEM;
365 goto err_path;
366 }
367 sc->rxqueues = sc->queues + num_tx_rings;
368
369 for (i = 0; i < sc->num_rings; i++) {
370 struct ptnet_queue *pq = sc->queues + i;
371
372 pq->sc = sc;
373 pq->kring_id = i;
374 pq->kick = PTNET_IO_KICK_BASE + 4 * i;
375 pq->atok = sc->csb_gh + i;
376 pq->ktoa = sc->csb_hg + i;
377 snprintf(pq->lock_name, sizeof(pq->lock_name), "%s-%d",
378 device_get_nameunit(dev), i);
379 mtx_init(&pq->lock, pq->lock_name, NULL, MTX_DEF);
380 if (i >= num_tx_rings) {
381 /* RX queue: fix kring_id. */
382 pq->kring_id -= num_tx_rings;
383 } else {
384 /* TX queue: allocate buf_ring. */
385 pq->bufring = buf_ring_alloc(PTNET_BUF_RING_SIZE,
386 M_DEVBUF, M_NOWAIT, &pq->lock);
387 if (pq->bufring == NULL) {
388 err = ENOMEM;
389 goto err_path;
390 }
391 }
392 }
393
394 sc->min_tx_space = 64; /* Safe initial value. */
395
396 err = ptnet_irqs_init(sc);
397 if (err) {
398 goto err_path;
399 }
400
401 /* Setup Ethernet interface. */
402 sc->ifp = ifp = if_alloc(IFT_ETHER);
403 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
404 ifp->if_baudrate = IF_Gbps(10);
405 ifp->if_softc = sc;
406 ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;
407 ifp->if_init = ptnet_init;
408 ifp->if_ioctl = ptnet_ioctl;
409 ifp->if_get_counter = ptnet_get_counter;
410 ifp->if_transmit = ptnet_transmit;
411 ifp->if_qflush = ptnet_qflush;
412
413 ifmedia_init(&sc->media, IFM_IMASK, ptnet_media_change,
414 ptnet_media_status);
415 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX, 0, NULL);
416 ifmedia_set(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX);
417
418 macreg = bus_read_4(sc->iomem, PTNET_IO_MAC_HI);
419 sc->hwaddr[0] = (macreg >> 8) & 0xff;
420 sc->hwaddr[1] = macreg & 0xff;
421 macreg = bus_read_4(sc->iomem, PTNET_IO_MAC_LO);
422 sc->hwaddr[2] = (macreg >> 24) & 0xff;
423 sc->hwaddr[3] = (macreg >> 16) & 0xff;
424 sc->hwaddr[4] = (macreg >> 8) & 0xff;
425 sc->hwaddr[5] = macreg & 0xff;
426
427 ether_ifattach(ifp, sc->hwaddr);
428
429 ifp->if_hdrlen = sizeof(struct ether_vlan_header);
430 ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU;
431
432 if (sc->ptfeatures & PTNETMAP_F_VNET_HDR) {
433 /* Similarly to what the vtnet driver does, we can emulate
434 * VLAN offloadings by inserting and removing the 802.1Q
435 * header during transmit and receive. We are then able
436 * to do checksum offloading of VLAN frames. */
437 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6
438 | IFCAP_VLAN_HWCSUM
439 | IFCAP_TSO | IFCAP_LRO
440 | IFCAP_VLAN_HWTSO
441 | IFCAP_VLAN_HWTAGGING;
442 }
443
444 ifp->if_capenable = ifp->if_capabilities;
445 #ifdef DEVICE_POLLING
446 /* Don't enable polling by default. */
447 ifp->if_capabilities |= IFCAP_POLLING;
448 #endif
449 snprintf(sc->lock_name, sizeof(sc->lock_name),
450 "%s", device_get_nameunit(dev));
451 mtx_init(&sc->lock, sc->lock_name, "ptnet core lock", MTX_DEF);
452 callout_init_mtx(&sc->tick, &sc->lock, 0);
453
454 /* Prepare a netmap_adapter struct instance to do netmap_attach(). */
455 nifp_offset = bus_read_4(sc->iomem, PTNET_IO_NIFP_OFS);
456 memset(&na_arg, 0, sizeof(na_arg));
457 na_arg.ifp = ifp;
458 na_arg.num_tx_desc = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_SLOTS);
459 na_arg.num_rx_desc = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_SLOTS);
460 na_arg.num_tx_rings = num_tx_rings;
461 na_arg.num_rx_rings = num_rx_rings;
462 na_arg.nm_config = ptnet_nm_config;
463 na_arg.nm_krings_create = ptnet_nm_krings_create;
464 na_arg.nm_krings_delete = ptnet_nm_krings_delete;
465 na_arg.nm_dtor = ptnet_nm_dtor;
466 na_arg.nm_intr = ptnet_nm_intr;
467 na_arg.nm_register = ptnet_nm_register;
468 na_arg.nm_txsync = ptnet_nm_txsync;
469 na_arg.nm_rxsync = ptnet_nm_rxsync;
470
471 netmap_pt_guest_attach(&na_arg, nifp_offset,
472 bus_read_4(sc->iomem, PTNET_IO_HOSTMEMID));
473
474 /* Now a netmap adapter for this ifp has been allocated, and it
475 * can be accessed through NA(ifp). We also have to initialize the CSB
476 * pointer. */
477 sc->ptna = (struct netmap_pt_guest_adapter *)NA(ifp);
478
479 /* If virtio-net header was negotiated, set the virt_hdr_len field in
480 * the netmap adapter, to inform users that this netmap adapter requires
481 * the application to deal with the headers. */
482 ptnet_update_vnet_hdr(sc);
483
484 device_printf(dev, "%s() completed\n", __func__);
485
486 return (0);
487
488 err_path:
489 ptnet_detach(dev);
490 return err;
491 }
492
493 /* Stop host sync-kloop if it was running. */
494 static void
ptnet_device_shutdown(struct ptnet_softc * sc)495 ptnet_device_shutdown(struct ptnet_softc *sc)
496 {
497 ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_DELETE);
498 bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAH, 0);
499 bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAL, 0);
500 bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAH, 0);
501 bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAL, 0);
502 }
503
504 static int
ptnet_detach(device_t dev)505 ptnet_detach(device_t dev)
506 {
507 struct ptnet_softc *sc = device_get_softc(dev);
508 int i;
509
510 ptnet_device_shutdown(sc);
511
512 #ifdef DEVICE_POLLING
513 if (sc->ifp->if_capenable & IFCAP_POLLING) {
514 ether_poll_deregister(sc->ifp);
515 }
516 #endif
517 callout_drain(&sc->tick);
518
519 if (sc->queues) {
520 /* Drain taskqueues before calling if_detach. */
521 for (i = 0; i < sc->num_rings; i++) {
522 struct ptnet_queue *pq = sc->queues + i;
523
524 if (pq->taskq) {
525 taskqueue_drain(pq->taskq, &pq->task);
526 }
527 }
528 }
529
530 if (sc->ifp) {
531 ether_ifdetach(sc->ifp);
532
533 /* Uninitialize netmap adapters for this device. */
534 netmap_detach(sc->ifp);
535
536 ifmedia_removeall(&sc->media);
537 if_free(sc->ifp);
538 sc->ifp = NULL;
539 }
540
541 ptnet_irqs_fini(sc);
542
543 if (sc->csb_gh) {
544 contigfree(sc->csb_gh, 2*PAGE_SIZE, M_DEVBUF);
545 sc->csb_gh = NULL;
546 sc->csb_hg = NULL;
547 }
548
549 if (sc->queues) {
550 for (i = 0; i < sc->num_rings; i++) {
551 struct ptnet_queue *pq = sc->queues + i;
552
553 if (mtx_initialized(&pq->lock)) {
554 mtx_destroy(&pq->lock);
555 }
556 if (pq->bufring != NULL) {
557 buf_ring_free(pq->bufring, M_DEVBUF);
558 }
559 }
560 free(sc->queues, M_DEVBUF);
561 sc->queues = NULL;
562 }
563
564 if (sc->iomem) {
565 bus_release_resource(dev, SYS_RES_IOPORT,
566 PCIR_BAR(PTNETMAP_IO_PCI_BAR), sc->iomem);
567 sc->iomem = NULL;
568 }
569
570 mtx_destroy(&sc->lock);
571
572 device_printf(dev, "%s() completed\n", __func__);
573
574 return (0);
575 }
576
577 static int
ptnet_suspend(device_t dev)578 ptnet_suspend(device_t dev)
579 {
580 struct ptnet_softc *sc = device_get_softc(dev);
581
582 (void)sc;
583
584 return (0);
585 }
586
587 static int
ptnet_resume(device_t dev)588 ptnet_resume(device_t dev)
589 {
590 struct ptnet_softc *sc = device_get_softc(dev);
591
592 (void)sc;
593
594 return (0);
595 }
596
597 static int
ptnet_shutdown(device_t dev)598 ptnet_shutdown(device_t dev)
599 {
600 struct ptnet_softc *sc = device_get_softc(dev);
601
602 ptnet_device_shutdown(sc);
603
604 return (0);
605 }
606
607 static int
ptnet_irqs_init(struct ptnet_softc * sc)608 ptnet_irqs_init(struct ptnet_softc *sc)
609 {
610 int rid = PCIR_BAR(PTNETMAP_MSIX_PCI_BAR);
611 int nvecs = sc->num_rings;
612 device_t dev = sc->dev;
613 int err = ENOSPC;
614 int cpu_cur;
615 int i;
616
617 if (pci_find_cap(dev, PCIY_MSIX, NULL) != 0) {
618 device_printf(dev, "Could not find MSI-X capability\n");
619 return (ENXIO);
620 }
621
622 sc->msix_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
623 &rid, RF_ACTIVE);
624 if (sc->msix_mem == NULL) {
625 device_printf(dev, "Failed to allocate MSIX PCI BAR\n");
626 return (ENXIO);
627 }
628
629 if (pci_msix_count(dev) < nvecs) {
630 device_printf(dev, "Not enough MSI-X vectors\n");
631 goto err_path;
632 }
633
634 err = pci_alloc_msix(dev, &nvecs);
635 if (err) {
636 device_printf(dev, "Failed to allocate MSI-X vectors\n");
637 goto err_path;
638 }
639
640 for (i = 0; i < nvecs; i++) {
641 struct ptnet_queue *pq = sc->queues + i;
642
643 rid = i + 1;
644 pq->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
645 RF_ACTIVE);
646 if (pq->irq == NULL) {
647 device_printf(dev, "Failed to allocate interrupt "
648 "for queue #%d\n", i);
649 err = ENOSPC;
650 goto err_path;
651 }
652 }
653
654 cpu_cur = CPU_FIRST();
655 for (i = 0; i < nvecs; i++) {
656 struct ptnet_queue *pq = sc->queues + i;
657 void (*handler)(void *) = ptnet_tx_intr;
658
659 if (i >= sc->num_tx_rings) {
660 handler = ptnet_rx_intr;
661 }
662 err = bus_setup_intr(dev, pq->irq, INTR_TYPE_NET | INTR_MPSAFE,
663 NULL /* intr_filter */, handler,
664 pq, &pq->cookie);
665 if (err) {
666 device_printf(dev, "Failed to register intr handler "
667 "for queue #%d\n", i);
668 goto err_path;
669 }
670
671 bus_describe_intr(dev, pq->irq, pq->cookie, "q%d", i);
672 #if 0
673 bus_bind_intr(sc->dev, pq->irq, cpu_cur);
674 #endif
675 cpu_cur = CPU_NEXT(cpu_cur);
676 }
677
678 device_printf(dev, "Allocated %d MSI-X vectors\n", nvecs);
679
680 cpu_cur = CPU_FIRST();
681 for (i = 0; i < nvecs; i++) {
682 struct ptnet_queue *pq = sc->queues + i;
683
684 if (i < sc->num_tx_rings)
685 TASK_INIT(&pq->task, 0, ptnet_tx_task, pq);
686 else
687 NET_TASK_INIT(&pq->task, 0, ptnet_rx_task, pq);
688
689 pq->taskq = taskqueue_create_fast("ptnet_queue", M_NOWAIT,
690 taskqueue_thread_enqueue, &pq->taskq);
691 taskqueue_start_threads(&pq->taskq, 1, PI_NET, "%s-pq-%d",
692 device_get_nameunit(sc->dev), cpu_cur);
693 cpu_cur = CPU_NEXT(cpu_cur);
694 }
695
696 return 0;
697 err_path:
698 ptnet_irqs_fini(sc);
699 return err;
700 }
701
702 static void
ptnet_irqs_fini(struct ptnet_softc * sc)703 ptnet_irqs_fini(struct ptnet_softc *sc)
704 {
705 device_t dev = sc->dev;
706 int i;
707
708 for (i = 0; i < sc->num_rings; i++) {
709 struct ptnet_queue *pq = sc->queues + i;
710
711 if (pq->taskq) {
712 taskqueue_free(pq->taskq);
713 pq->taskq = NULL;
714 }
715
716 if (pq->cookie) {
717 bus_teardown_intr(dev, pq->irq, pq->cookie);
718 pq->cookie = NULL;
719 }
720
721 if (pq->irq) {
722 bus_release_resource(dev, SYS_RES_IRQ, i + 1, pq->irq);
723 pq->irq = NULL;
724 }
725 }
726
727 if (sc->msix_mem) {
728 pci_release_msi(dev);
729
730 bus_release_resource(dev, SYS_RES_MEMORY,
731 PCIR_BAR(PTNETMAP_MSIX_PCI_BAR),
732 sc->msix_mem);
733 sc->msix_mem = NULL;
734 }
735 }
736
737 static void
ptnet_init(void * opaque)738 ptnet_init(void *opaque)
739 {
740 struct ptnet_softc *sc = opaque;
741
742 PTNET_CORE_LOCK(sc);
743 ptnet_init_locked(sc);
744 PTNET_CORE_UNLOCK(sc);
745 }
746
747 static int
ptnet_ioctl(if_t ifp,u_long cmd,caddr_t data)748 ptnet_ioctl(if_t ifp, u_long cmd, caddr_t data)
749 {
750 struct ptnet_softc *sc = if_getsoftc(ifp);
751 device_t dev = sc->dev;
752 struct ifreq *ifr = (struct ifreq *)data;
753 int mask __unused, err = 0;
754
755 switch (cmd) {
756 case SIOCSIFFLAGS:
757 device_printf(dev, "SIOCSIFFLAGS %x\n", ifp->if_flags);
758 PTNET_CORE_LOCK(sc);
759 if (ifp->if_flags & IFF_UP) {
760 /* Network stack wants the iff to be up. */
761 err = ptnet_init_locked(sc);
762 } else {
763 /* Network stack wants the iff to be down. */
764 err = ptnet_stop(sc);
765 }
766 /* We don't need to do nothing to support IFF_PROMISC,
767 * since that is managed by the backend port. */
768 PTNET_CORE_UNLOCK(sc);
769 break;
770
771 case SIOCSIFCAP:
772 device_printf(dev, "SIOCSIFCAP %x %x\n",
773 ifr->ifr_reqcap, ifp->if_capenable);
774 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
775 #ifdef DEVICE_POLLING
776 if (mask & IFCAP_POLLING) {
777 struct ptnet_queue *pq;
778 int i;
779
780 if (ifr->ifr_reqcap & IFCAP_POLLING) {
781 err = ether_poll_register(ptnet_poll, ifp);
782 if (err) {
783 break;
784 }
785 /* Stop queues and sync with taskqueues. */
786 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
787 for (i = 0; i < sc->num_rings; i++) {
788 pq = sc-> queues + i;
789 /* Make sure the worker sees the
790 * IFF_DRV_RUNNING down. */
791 PTNET_Q_LOCK(pq);
792 pq->atok->appl_need_kick = 0;
793 PTNET_Q_UNLOCK(pq);
794 /* Wait for rescheduling to finish. */
795 if (pq->taskq) {
796 taskqueue_drain(pq->taskq,
797 &pq->task);
798 }
799 }
800 ifp->if_drv_flags |= IFF_DRV_RUNNING;
801 } else {
802 err = ether_poll_deregister(ifp);
803 for (i = 0; i < sc->num_rings; i++) {
804 pq = sc-> queues + i;
805 PTNET_Q_LOCK(pq);
806 pq->atok->appl_need_kick = 1;
807 PTNET_Q_UNLOCK(pq);
808 }
809 }
810 }
811 #endif /* DEVICE_POLLING */
812 ifp->if_capenable = ifr->ifr_reqcap;
813 break;
814
815 case SIOCSIFMTU:
816 /* We support any reasonable MTU. */
817 if (ifr->ifr_mtu < ETHERMIN ||
818 ifr->ifr_mtu > PTNET_MAX_PKT_SIZE) {
819 err = EINVAL;
820 } else {
821 PTNET_CORE_LOCK(sc);
822 ifp->if_mtu = ifr->ifr_mtu;
823 PTNET_CORE_UNLOCK(sc);
824 }
825 break;
826
827 case SIOCSIFMEDIA:
828 case SIOCGIFMEDIA:
829 err = ifmedia_ioctl(ifp, ifr, &sc->media, cmd);
830 break;
831
832 default:
833 err = ether_ioctl(ifp, cmd, data);
834 break;
835 }
836
837 return err;
838 }
839
840 static int
ptnet_init_locked(struct ptnet_softc * sc)841 ptnet_init_locked(struct ptnet_softc *sc)
842 {
843 if_t ifp = sc->ifp;
844 struct netmap_adapter *na_dr = &sc->ptna->dr.up;
845 struct netmap_adapter *na_nm = &sc->ptna->hwup.up;
846 unsigned int nm_buf_size;
847 int ret;
848
849 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
850 return 0; /* nothing to do */
851 }
852
853 device_printf(sc->dev, "%s\n", __func__);
854
855 /* Translate offload capabilities according to if_capenable. */
856 ifp->if_hwassist = 0;
857 if (ifp->if_capenable & IFCAP_TXCSUM)
858 ifp->if_hwassist |= PTNET_CSUM_OFFLOAD;
859 if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
860 ifp->if_hwassist |= PTNET_CSUM_OFFLOAD_IPV6;
861 if (ifp->if_capenable & IFCAP_TSO4)
862 ifp->if_hwassist |= CSUM_IP_TSO;
863 if (ifp->if_capenable & IFCAP_TSO6)
864 ifp->if_hwassist |= CSUM_IP6_TSO;
865
866 /*
867 * Prepare the interface for netmap mode access.
868 */
869 netmap_update_config(na_dr);
870
871 ret = netmap_mem_finalize(na_dr->nm_mem, na_dr);
872 if (ret) {
873 device_printf(sc->dev, "netmap_mem_finalize() failed\n");
874 return ret;
875 }
876
877 if (sc->ptna->backend_users == 0) {
878 ret = ptnet_nm_krings_create(na_nm);
879 if (ret) {
880 device_printf(sc->dev, "ptnet_nm_krings_create() "
881 "failed\n");
882 goto err_mem_finalize;
883 }
884
885 ret = netmap_mem_rings_create(na_dr);
886 if (ret) {
887 device_printf(sc->dev, "netmap_mem_rings_create() "
888 "failed\n");
889 goto err_rings_create;
890 }
891
892 ret = netmap_mem_get_lut(na_dr->nm_mem, &na_dr->na_lut);
893 if (ret) {
894 device_printf(sc->dev, "netmap_mem_get_lut() "
895 "failed\n");
896 goto err_get_lut;
897 }
898 }
899
900 ret = ptnet_nm_register(na_dr, 1 /* on */);
901 if (ret) {
902 goto err_register;
903 }
904
905 nm_buf_size = NETMAP_BUF_SIZE(na_dr);
906
907 KASSERT(nm_buf_size > 0, ("Invalid netmap buffer size"));
908 sc->min_tx_space = PTNET_MAX_PKT_SIZE / nm_buf_size + 2;
909 device_printf(sc->dev, "%s: min_tx_space = %u\n", __func__,
910 sc->min_tx_space);
911 #ifdef PTNETMAP_STATS
912 callout_reset(&sc->tick, hz, ptnet_tick, sc);
913 #endif
914
915 ifp->if_drv_flags |= IFF_DRV_RUNNING;
916
917 return 0;
918
919 err_register:
920 memset(&na_dr->na_lut, 0, sizeof(na_dr->na_lut));
921 err_get_lut:
922 netmap_mem_rings_delete(na_dr);
923 err_rings_create:
924 ptnet_nm_krings_delete(na_nm);
925 err_mem_finalize:
926 netmap_mem_deref(na_dr->nm_mem, na_dr);
927
928 return ret;
929 }
930
931 /* To be called under core lock. */
932 static int
ptnet_stop(struct ptnet_softc * sc)933 ptnet_stop(struct ptnet_softc *sc)
934 {
935 if_t ifp = sc->ifp;
936 struct netmap_adapter *na_dr = &sc->ptna->dr.up;
937 struct netmap_adapter *na_nm = &sc->ptna->hwup.up;
938 int i;
939
940 device_printf(sc->dev, "%s\n", __func__);
941
942 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
943 return 0; /* nothing to do */
944 }
945
946 /* Clear the driver-ready flag, and synchronize with all the queues,
947 * so that after this loop we are sure nobody is working anymore with
948 * the device. This scheme is taken from the vtnet driver. */
949 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
950 callout_stop(&sc->tick);
951 for (i = 0; i < sc->num_rings; i++) {
952 PTNET_Q_LOCK(sc->queues + i);
953 PTNET_Q_UNLOCK(sc->queues + i);
954 }
955
956 ptnet_nm_register(na_dr, 0 /* off */);
957
958 if (sc->ptna->backend_users == 0) {
959 netmap_mem_rings_delete(na_dr);
960 ptnet_nm_krings_delete(na_nm);
961 }
962 netmap_mem_deref(na_dr->nm_mem, na_dr);
963
964 return 0;
965 }
966
967 static void
ptnet_qflush(if_t ifp)968 ptnet_qflush(if_t ifp)
969 {
970 struct ptnet_softc *sc = if_getsoftc(ifp);
971 int i;
972
973 /* Flush all the bufrings and do the interface flush. */
974 for (i = 0; i < sc->num_rings; i++) {
975 struct ptnet_queue *pq = sc->queues + i;
976 struct mbuf *m;
977
978 PTNET_Q_LOCK(pq);
979 if (pq->bufring) {
980 while ((m = buf_ring_dequeue_sc(pq->bufring))) {
981 m_freem(m);
982 }
983 }
984 PTNET_Q_UNLOCK(pq);
985 }
986
987 if_qflush(ifp);
988 }
989
990 static int
ptnet_media_change(if_t ifp)991 ptnet_media_change(if_t ifp)
992 {
993 struct ptnet_softc *sc = if_getsoftc(ifp);
994 struct ifmedia *ifm = &sc->media;
995
996 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) {
997 return EINVAL;
998 }
999
1000 return 0;
1001 }
1002
1003 static uint64_t
ptnet_get_counter(if_t ifp,ift_counter cnt)1004 ptnet_get_counter(if_t ifp, ift_counter cnt)
1005 {
1006 struct ptnet_softc *sc = if_getsoftc(ifp);
1007 struct ptnet_queue_stats stats[2];
1008 int i;
1009
1010 /* Accumulate statistics over the queues. */
1011 memset(stats, 0, sizeof(stats));
1012 for (i = 0; i < sc->num_rings; i++) {
1013 struct ptnet_queue *pq = sc->queues + i;
1014 int idx = (i < sc->num_tx_rings) ? 0 : 1;
1015
1016 stats[idx].packets += pq->stats.packets;
1017 stats[idx].bytes += pq->stats.bytes;
1018 stats[idx].errors += pq->stats.errors;
1019 stats[idx].iqdrops += pq->stats.iqdrops;
1020 stats[idx].mcasts += pq->stats.mcasts;
1021 }
1022
1023 switch (cnt) {
1024 case IFCOUNTER_IPACKETS:
1025 return (stats[1].packets);
1026 case IFCOUNTER_IQDROPS:
1027 return (stats[1].iqdrops);
1028 case IFCOUNTER_IERRORS:
1029 return (stats[1].errors);
1030 case IFCOUNTER_OPACKETS:
1031 return (stats[0].packets);
1032 case IFCOUNTER_OBYTES:
1033 return (stats[0].bytes);
1034 case IFCOUNTER_OMCASTS:
1035 return (stats[0].mcasts);
1036 default:
1037 return (if_get_counter_default(ifp, cnt));
1038 }
1039 }
1040
1041
1042 #ifdef PTNETMAP_STATS
1043 /* Called under core lock. */
1044 static void
ptnet_tick(void * opaque)1045 ptnet_tick(void *opaque)
1046 {
1047 struct ptnet_softc *sc = opaque;
1048 int i;
1049
1050 for (i = 0; i < sc->num_rings; i++) {
1051 struct ptnet_queue *pq = sc->queues + i;
1052 struct ptnet_queue_stats cur = pq->stats;
1053 struct timeval now;
1054 unsigned int delta;
1055
1056 microtime(&now);
1057 delta = now.tv_usec - sc->last_ts.tv_usec +
1058 (now.tv_sec - sc->last_ts.tv_sec) * 1000000;
1059 delta /= 1000; /* in milliseconds */
1060
1061 if (delta == 0)
1062 continue;
1063
1064 device_printf(sc->dev, "#%d[%u ms]:pkts %lu, kicks %lu, "
1065 "intr %lu\n", i, delta,
1066 (cur.packets - pq->last_stats.packets),
1067 (cur.kicks - pq->last_stats.kicks),
1068 (cur.intrs - pq->last_stats.intrs));
1069 pq->last_stats = cur;
1070 }
1071 microtime(&sc->last_ts);
1072 callout_schedule(&sc->tick, hz);
1073 }
1074 #endif /* PTNETMAP_STATS */
1075
1076 static void
ptnet_media_status(if_t ifp,struct ifmediareq * ifmr)1077 ptnet_media_status(if_t ifp, struct ifmediareq *ifmr)
1078 {
1079 /* We are always active, as the backend netmap port is
1080 * always open in netmap mode. */
1081 ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE;
1082 ifmr->ifm_active = IFM_ETHER | IFM_10G_T | IFM_FDX;
1083 }
1084
1085 static uint32_t
ptnet_nm_ptctl(struct ptnet_softc * sc,uint32_t cmd)1086 ptnet_nm_ptctl(struct ptnet_softc *sc, uint32_t cmd)
1087 {
1088 /*
1089 * Write a command and read back error status,
1090 * with zero meaning success.
1091 */
1092 bus_write_4(sc->iomem, PTNET_IO_PTCTL, cmd);
1093 return bus_read_4(sc->iomem, PTNET_IO_PTCTL);
1094 }
1095
1096 static int
ptnet_nm_config(struct netmap_adapter * na,struct nm_config_info * info)1097 ptnet_nm_config(struct netmap_adapter *na, struct nm_config_info *info)
1098 {
1099 struct ptnet_softc *sc = if_getsoftc(na->ifp);
1100
1101 info->num_tx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_RINGS);
1102 info->num_rx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_RINGS);
1103 info->num_tx_descs = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_SLOTS);
1104 info->num_rx_descs = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_SLOTS);
1105 info->rx_buf_maxsize = NETMAP_BUF_SIZE(na);
1106
1107 device_printf(sc->dev, "txr %u, rxr %u, txd %u, rxd %u, rxbufsz %u\n",
1108 info->num_tx_rings, info->num_rx_rings,
1109 info->num_tx_descs, info->num_rx_descs,
1110 info->rx_buf_maxsize);
1111
1112 return 0;
1113 }
1114
1115 static void
ptnet_sync_from_csb(struct ptnet_softc * sc,struct netmap_adapter * na)1116 ptnet_sync_from_csb(struct ptnet_softc *sc, struct netmap_adapter *na)
1117 {
1118 int i;
1119
1120 /* Sync krings from the host, reading from
1121 * CSB. */
1122 for (i = 0; i < sc->num_rings; i++) {
1123 struct nm_csb_atok *atok = sc->queues[i].atok;
1124 struct nm_csb_ktoa *ktoa = sc->queues[i].ktoa;
1125 struct netmap_kring *kring;
1126
1127 if (i < na->num_tx_rings) {
1128 kring = na->tx_rings[i];
1129 } else {
1130 kring = na->rx_rings[i - na->num_tx_rings];
1131 }
1132 kring->rhead = kring->ring->head = atok->head;
1133 kring->rcur = kring->ring->cur = atok->cur;
1134 kring->nr_hwcur = ktoa->hwcur;
1135 kring->nr_hwtail = kring->rtail =
1136 kring->ring->tail = ktoa->hwtail;
1137
1138 nm_prdis("%d,%d: csb {hc %u h %u c %u ht %u}", t, i,
1139 ktoa->hwcur, atok->head, atok->cur,
1140 ktoa->hwtail);
1141 nm_prdis("%d,%d: kring {hc %u rh %u rc %u h %u c %u ht %u rt %u t %u}",
1142 t, i, kring->nr_hwcur, kring->rhead, kring->rcur,
1143 kring->ring->head, kring->ring->cur, kring->nr_hwtail,
1144 kring->rtail, kring->ring->tail);
1145 }
1146 }
1147
1148 static void
ptnet_update_vnet_hdr(struct ptnet_softc * sc)1149 ptnet_update_vnet_hdr(struct ptnet_softc *sc)
1150 {
1151 unsigned int wanted_hdr_len = ptnet_vnet_hdr ? PTNET_HDR_SIZE : 0;
1152
1153 bus_write_4(sc->iomem, PTNET_IO_VNET_HDR_LEN, wanted_hdr_len);
1154 sc->vnet_hdr_len = bus_read_4(sc->iomem, PTNET_IO_VNET_HDR_LEN);
1155 sc->ptna->hwup.up.virt_hdr_len = sc->vnet_hdr_len;
1156 }
1157
1158 static int
ptnet_nm_register(struct netmap_adapter * na,int onoff)1159 ptnet_nm_register(struct netmap_adapter *na, int onoff)
1160 {
1161 /* device-specific */
1162 if_t ifp = na->ifp;
1163 struct ptnet_softc *sc = if_getsoftc(ifp);
1164 int native = (na == &sc->ptna->hwup.up);
1165 struct ptnet_queue *pq;
1166 int ret = 0;
1167 int i;
1168
1169 if (!onoff) {
1170 sc->ptna->backend_users--;
1171 }
1172
1173 /* If this is the last netmap client, guest interrupt enable flags may
1174 * be in arbitrary state. Since these flags are going to be used also
1175 * by the netdevice driver, we have to make sure to start with
1176 * notifications enabled. Also, schedule NAPI to flush pending packets
1177 * in the RX rings, since we will not receive further interrupts
1178 * until these will be processed. */
1179 if (native && !onoff && na->active_fds == 0) {
1180 nm_prinf("Exit netmap mode, re-enable interrupts");
1181 for (i = 0; i < sc->num_rings; i++) {
1182 pq = sc->queues + i;
1183 pq->atok->appl_need_kick = 1;
1184 }
1185 }
1186
1187 if (onoff) {
1188 if (sc->ptna->backend_users == 0) {
1189 /* Initialize notification enable fields in the CSB. */
1190 for (i = 0; i < sc->num_rings; i++) {
1191 pq = sc->queues + i;
1192 pq->ktoa->kern_need_kick = 1;
1193 pq->atok->appl_need_kick =
1194 (!(ifp->if_capenable & IFCAP_POLLING)
1195 && i >= sc->num_tx_rings);
1196 }
1197
1198 /* Set the virtio-net header length. */
1199 ptnet_update_vnet_hdr(sc);
1200
1201 /* Make sure the host adapter passed through is ready
1202 * for txsync/rxsync. */
1203 ret = ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_CREATE);
1204 if (ret) {
1205 return ret;
1206 }
1207
1208 /* Align the guest krings and rings to the state stored
1209 * in the CSB. */
1210 ptnet_sync_from_csb(sc, na);
1211 }
1212
1213 /* If not native, don't call nm_set_native_flags, since we don't want
1214 * to replace if_transmit method, nor set NAF_NETMAP_ON */
1215 if (native) {
1216 netmap_krings_mode_commit(na, onoff);
1217 nm_set_native_flags(na);
1218 }
1219
1220 } else {
1221 if (native) {
1222 nm_clear_native_flags(na);
1223 netmap_krings_mode_commit(na, onoff);
1224 }
1225
1226 if (sc->ptna->backend_users == 0) {
1227 ret = ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_DELETE);
1228 }
1229 }
1230
1231 if (onoff) {
1232 sc->ptna->backend_users++;
1233 }
1234
1235 return ret;
1236 }
1237
1238 static int
ptnet_nm_txsync(struct netmap_kring * kring,int flags)1239 ptnet_nm_txsync(struct netmap_kring *kring, int flags)
1240 {
1241 struct ptnet_softc *sc = if_getsoftc(kring->na->ifp);
1242 struct ptnet_queue *pq = sc->queues + kring->ring_id;
1243 bool notify;
1244
1245 notify = netmap_pt_guest_txsync(pq->atok, pq->ktoa, kring, flags);
1246 if (notify) {
1247 ptnet_kick(pq);
1248 }
1249
1250 return 0;
1251 }
1252
1253 static int
ptnet_nm_rxsync(struct netmap_kring * kring,int flags)1254 ptnet_nm_rxsync(struct netmap_kring *kring, int flags)
1255 {
1256 struct ptnet_softc *sc = if_getsoftc(kring->na->ifp);
1257 struct ptnet_queue *pq = sc->rxqueues + kring->ring_id;
1258 bool notify;
1259
1260 notify = netmap_pt_guest_rxsync(pq->atok, pq->ktoa, kring, flags);
1261 if (notify) {
1262 ptnet_kick(pq);
1263 }
1264
1265 return 0;
1266 }
1267
1268 static void
ptnet_nm_intr(struct netmap_adapter * na,int onoff)1269 ptnet_nm_intr(struct netmap_adapter *na, int onoff)
1270 {
1271 struct ptnet_softc *sc = if_getsoftc(na->ifp);
1272 int i;
1273
1274 for (i = 0; i < sc->num_rings; i++) {
1275 struct ptnet_queue *pq = sc->queues + i;
1276 pq->atok->appl_need_kick = onoff;
1277 }
1278 }
1279
1280 static void
ptnet_tx_intr(void * opaque)1281 ptnet_tx_intr(void *opaque)
1282 {
1283 struct ptnet_queue *pq = opaque;
1284 struct ptnet_softc *sc = pq->sc;
1285
1286 DBG(device_printf(sc->dev, "Tx interrupt #%d\n", pq->kring_id));
1287 #ifdef PTNETMAP_STATS
1288 pq->stats.intrs ++;
1289 #endif /* PTNETMAP_STATS */
1290
1291 if (netmap_tx_irq(sc->ifp, pq->kring_id) != NM_IRQ_PASS) {
1292 return;
1293 }
1294
1295 /* Schedule the tasqueue to flush process transmissions requests.
1296 * However, vtnet, if_em and if_igb just call ptnet_transmit() here,
1297 * at least when using MSI-X interrupts. The if_em driver, instead
1298 * schedule taskqueue when using legacy interrupts. */
1299 taskqueue_enqueue(pq->taskq, &pq->task);
1300 }
1301
1302 static void
ptnet_rx_intr(void * opaque)1303 ptnet_rx_intr(void *opaque)
1304 {
1305 struct ptnet_queue *pq = opaque;
1306 struct ptnet_softc *sc = pq->sc;
1307 unsigned int unused;
1308
1309 DBG(device_printf(sc->dev, "Rx interrupt #%d\n", pq->kring_id));
1310 #ifdef PTNETMAP_STATS
1311 pq->stats.intrs ++;
1312 #endif /* PTNETMAP_STATS */
1313
1314 if (netmap_rx_irq(sc->ifp, pq->kring_id, &unused) != NM_IRQ_PASS) {
1315 return;
1316 }
1317
1318 /* Like vtnet, if_igb and if_em drivers when using MSI-X interrupts,
1319 * receive-side processing is executed directly in the interrupt
1320 * service routine. Alternatively, we may schedule the taskqueue. */
1321 ptnet_rx_eof(pq, PTNET_RX_BUDGET, true);
1322 }
1323
1324 static void
ptnet_vlan_tag_remove(struct mbuf * m)1325 ptnet_vlan_tag_remove(struct mbuf *m)
1326 {
1327 struct ether_vlan_header *evh;
1328
1329 evh = mtod(m, struct ether_vlan_header *);
1330 m->m_pkthdr.ether_vtag = ntohs(evh->evl_tag);
1331 m->m_flags |= M_VLANTAG;
1332
1333 /* Strip the 802.1Q header. */
1334 bcopy((char *) evh, (char *) evh + ETHER_VLAN_ENCAP_LEN,
1335 ETHER_HDR_LEN - ETHER_TYPE_LEN);
1336 m_adj(m, ETHER_VLAN_ENCAP_LEN);
1337 }
1338
1339 static void
ptnet_ring_update(struct ptnet_queue * pq,struct netmap_kring * kring,unsigned int head,unsigned int sync_flags)1340 ptnet_ring_update(struct ptnet_queue *pq, struct netmap_kring *kring,
1341 unsigned int head, unsigned int sync_flags)
1342 {
1343 struct netmap_ring *ring = kring->ring;
1344 struct nm_csb_atok *atok = pq->atok;
1345 struct nm_csb_ktoa *ktoa = pq->ktoa;
1346
1347 /* Some packets have been pushed to the netmap ring. We have
1348 * to tell the host to process the new packets, updating cur
1349 * and head in the CSB. */
1350 ring->head = ring->cur = head;
1351
1352 /* Mimic nm_txsync_prologue/nm_rxsync_prologue. */
1353 kring->rcur = kring->rhead = head;
1354
1355 nm_sync_kloop_appl_write(atok, kring->rcur, kring->rhead);
1356
1357 /* Kick the host if needed. */
1358 if (NM_ACCESS_ONCE(ktoa->kern_need_kick)) {
1359 atok->sync_flags = sync_flags;
1360 ptnet_kick(pq);
1361 }
1362 }
1363
1364 #define PTNET_TX_NOSPACE(_h, _k, _min) \
1365 ((((_h) < (_k)->rtail) ? 0 : (_k)->nkr_num_slots) + \
1366 (_k)->rtail - (_h)) < (_min)
1367
1368 /* This function may be called by the network stack, or by
1369 * by the taskqueue thread. */
1370 static int
ptnet_drain_transmit_queue(struct ptnet_queue * pq,unsigned int budget,bool may_resched)1371 ptnet_drain_transmit_queue(struct ptnet_queue *pq, unsigned int budget,
1372 bool may_resched)
1373 {
1374 struct ptnet_softc *sc = pq->sc;
1375 bool have_vnet_hdr = sc->vnet_hdr_len;
1376 struct netmap_adapter *na = &sc->ptna->dr.up;
1377 if_t ifp = sc->ifp;
1378 unsigned int batch_count = 0;
1379 struct nm_csb_atok *atok;
1380 struct nm_csb_ktoa *ktoa;
1381 struct netmap_kring *kring;
1382 struct netmap_ring *ring;
1383 struct netmap_slot *slot;
1384 unsigned int count = 0;
1385 unsigned int minspace;
1386 unsigned int head;
1387 unsigned int lim;
1388 struct mbuf *mhead;
1389 struct mbuf *mf;
1390 int nmbuf_bytes;
1391 uint8_t *nmbuf;
1392
1393 if (!PTNET_Q_TRYLOCK(pq)) {
1394 /* We failed to acquire the lock, schedule the taskqueue. */
1395 nm_prlim(1, "Deferring TX work");
1396 if (may_resched) {
1397 taskqueue_enqueue(pq->taskq, &pq->task);
1398 }
1399
1400 return 0;
1401 }
1402
1403 if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) {
1404 PTNET_Q_UNLOCK(pq);
1405 nm_prlim(1, "Interface is down");
1406 return ENETDOWN;
1407 }
1408
1409 atok = pq->atok;
1410 ktoa = pq->ktoa;
1411 kring = na->tx_rings[pq->kring_id];
1412 ring = kring->ring;
1413 lim = kring->nkr_num_slots - 1;
1414 head = ring->head;
1415 minspace = sc->min_tx_space;
1416
1417 while (count < budget) {
1418 if (PTNET_TX_NOSPACE(head, kring, minspace)) {
1419 /* We ran out of slot, let's see if the host has
1420 * freed up some, by reading hwcur and hwtail from
1421 * the CSB. */
1422 ptnet_sync_tail(ktoa, kring);
1423
1424 if (PTNET_TX_NOSPACE(head, kring, minspace)) {
1425 /* Still no slots available. Reactivate the
1426 * interrupts so that we can be notified
1427 * when some free slots are made available by
1428 * the host. */
1429 atok->appl_need_kick = 1;
1430
1431 /* Double check. We need a full barrier to
1432 * prevent the store to atok->appl_need_kick
1433 * to be reordered with the load from
1434 * ktoa->hwcur and ktoa->hwtail (store-load
1435 * barrier). */
1436 nm_stld_barrier();
1437 ptnet_sync_tail(ktoa, kring);
1438 if (likely(PTNET_TX_NOSPACE(head, kring,
1439 minspace))) {
1440 break;
1441 }
1442
1443 nm_prlim(1, "Found more slots by doublecheck");
1444 /* More slots were freed before reactivating
1445 * the interrupts. */
1446 atok->appl_need_kick = 0;
1447 }
1448 }
1449
1450 mhead = drbr_peek(ifp, pq->bufring);
1451 if (!mhead) {
1452 break;
1453 }
1454
1455 /* Initialize transmission state variables. */
1456 slot = ring->slot + head;
1457 nmbuf = NMB(na, slot);
1458 nmbuf_bytes = 0;
1459
1460 /* If needed, prepare the virtio-net header at the beginning
1461 * of the first slot. */
1462 if (have_vnet_hdr) {
1463 struct virtio_net_hdr *vh =
1464 (struct virtio_net_hdr *)nmbuf;
1465
1466 /* For performance, we could replace this memset() with
1467 * two 8-bytes-wide writes. */
1468 memset(nmbuf, 0, PTNET_HDR_SIZE);
1469 if (mhead->m_pkthdr.csum_flags & PTNET_ALL_OFFLOAD) {
1470 mhead = virtio_net_tx_offload(ifp, mhead, false,
1471 vh);
1472 if (unlikely(!mhead)) {
1473 /* Packet dropped because errors
1474 * occurred while preparing the vnet
1475 * header. Let's go ahead with the next
1476 * packet. */
1477 pq->stats.errors ++;
1478 drbr_advance(ifp, pq->bufring);
1479 continue;
1480 }
1481 }
1482 nm_prdis(1, "%s: [csum_flags %lX] vnet hdr: flags %x "
1483 "csum_start %u csum_ofs %u hdr_len = %u "
1484 "gso_size %u gso_type %x", __func__,
1485 mhead->m_pkthdr.csum_flags, vh->flags,
1486 vh->csum_start, vh->csum_offset, vh->hdr_len,
1487 vh->gso_size, vh->gso_type);
1488
1489 nmbuf += PTNET_HDR_SIZE;
1490 nmbuf_bytes += PTNET_HDR_SIZE;
1491 }
1492
1493 for (mf = mhead; mf; mf = mf->m_next) {
1494 uint8_t *mdata = mf->m_data;
1495 int mlen = mf->m_len;
1496
1497 for (;;) {
1498 int copy = NETMAP_BUF_SIZE(na) - nmbuf_bytes;
1499
1500 if (mlen < copy) {
1501 copy = mlen;
1502 }
1503 memcpy(nmbuf, mdata, copy);
1504
1505 mdata += copy;
1506 mlen -= copy;
1507 nmbuf += copy;
1508 nmbuf_bytes += copy;
1509
1510 if (!mlen) {
1511 break;
1512 }
1513
1514 slot->len = nmbuf_bytes;
1515 slot->flags = NS_MOREFRAG;
1516
1517 head = nm_next(head, lim);
1518 KASSERT(head != ring->tail,
1519 ("Unexpectedly run out of TX space"));
1520 slot = ring->slot + head;
1521 nmbuf = NMB(na, slot);
1522 nmbuf_bytes = 0;
1523 }
1524 }
1525
1526 /* Complete last slot and update head. */
1527 slot->len = nmbuf_bytes;
1528 slot->flags = 0;
1529 head = nm_next(head, lim);
1530
1531 /* Consume the packet just processed. */
1532 drbr_advance(ifp, pq->bufring);
1533
1534 /* Copy the packet to listeners. */
1535 ETHER_BPF_MTAP(ifp, mhead);
1536
1537 pq->stats.packets ++;
1538 pq->stats.bytes += mhead->m_pkthdr.len;
1539 if (mhead->m_flags & M_MCAST) {
1540 pq->stats.mcasts ++;
1541 }
1542
1543 m_freem(mhead);
1544
1545 count ++;
1546 if (++batch_count == PTNET_TX_BATCH) {
1547 ptnet_ring_update(pq, kring, head, NAF_FORCE_RECLAIM);
1548 batch_count = 0;
1549 }
1550 }
1551
1552 if (batch_count) {
1553 ptnet_ring_update(pq, kring, head, NAF_FORCE_RECLAIM);
1554 }
1555
1556 if (count >= budget && may_resched) {
1557 DBG(nm_prlim(1, "out of budget: resched, %d mbufs pending\n",
1558 drbr_inuse(ifp, pq->bufring)));
1559 taskqueue_enqueue(pq->taskq, &pq->task);
1560 }
1561
1562 PTNET_Q_UNLOCK(pq);
1563
1564 return count;
1565 }
1566
1567 static int
ptnet_transmit(if_t ifp,struct mbuf * m)1568 ptnet_transmit(if_t ifp, struct mbuf *m)
1569 {
1570 struct ptnet_softc *sc = if_getsoftc(ifp);
1571 struct ptnet_queue *pq;
1572 unsigned int queue_idx;
1573 int err;
1574
1575 DBG(device_printf(sc->dev, "transmit %p\n", m));
1576
1577 /* Insert 802.1Q header if needed. */
1578 if (m->m_flags & M_VLANTAG) {
1579 m = ether_vlanencap(m, m->m_pkthdr.ether_vtag);
1580 if (m == NULL) {
1581 return ENOBUFS;
1582 }
1583 m->m_flags &= ~M_VLANTAG;
1584 }
1585
1586 /* Get the flow-id if available. */
1587 queue_idx = (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) ?
1588 m->m_pkthdr.flowid : curcpu;
1589
1590 if (unlikely(queue_idx >= sc->num_tx_rings)) {
1591 queue_idx %= sc->num_tx_rings;
1592 }
1593
1594 pq = sc->queues + queue_idx;
1595
1596 err = drbr_enqueue(ifp, pq->bufring, m);
1597 if (err) {
1598 /* ENOBUFS when the bufring is full */
1599 nm_prlim(1, "%s: drbr_enqueue() failed %d\n",
1600 __func__, err);
1601 pq->stats.errors ++;
1602 return err;
1603 }
1604
1605 if (ifp->if_capenable & IFCAP_POLLING) {
1606 /* If polling is on, the transmit queues will be
1607 * drained by the poller. */
1608 return 0;
1609 }
1610
1611 err = ptnet_drain_transmit_queue(pq, PTNET_TX_BUDGET, true);
1612
1613 return (err < 0) ? err : 0;
1614 }
1615
1616 static unsigned int
ptnet_rx_discard(struct netmap_kring * kring,unsigned int head)1617 ptnet_rx_discard(struct netmap_kring *kring, unsigned int head)
1618 {
1619 struct netmap_ring *ring = kring->ring;
1620 struct netmap_slot *slot = ring->slot + head;
1621
1622 for (;;) {
1623 head = nm_next(head, kring->nkr_num_slots - 1);
1624 if (!(slot->flags & NS_MOREFRAG) || head == ring->tail) {
1625 break;
1626 }
1627 slot = ring->slot + head;
1628 }
1629
1630 return head;
1631 }
1632
1633 static inline struct mbuf *
ptnet_rx_slot(struct mbuf * mtail,uint8_t * nmbuf,unsigned int nmbuf_len)1634 ptnet_rx_slot(struct mbuf *mtail, uint8_t *nmbuf, unsigned int nmbuf_len)
1635 {
1636 uint8_t *mdata = mtod(mtail, uint8_t *) + mtail->m_len;
1637
1638 do {
1639 unsigned int copy;
1640
1641 if (mtail->m_len == MCLBYTES) {
1642 struct mbuf *mf;
1643
1644 mf = m_getcl(M_NOWAIT, MT_DATA, 0);
1645 if (unlikely(!mf)) {
1646 return NULL;
1647 }
1648
1649 mtail->m_next = mf;
1650 mtail = mf;
1651 mdata = mtod(mtail, uint8_t *);
1652 mtail->m_len = 0;
1653 }
1654
1655 copy = MCLBYTES - mtail->m_len;
1656 if (nmbuf_len < copy) {
1657 copy = nmbuf_len;
1658 }
1659
1660 memcpy(mdata, nmbuf, copy);
1661
1662 nmbuf += copy;
1663 nmbuf_len -= copy;
1664 mdata += copy;
1665 mtail->m_len += copy;
1666 } while (nmbuf_len);
1667
1668 return mtail;
1669 }
1670
1671 static int
ptnet_rx_eof(struct ptnet_queue * pq,unsigned int budget,bool may_resched)1672 ptnet_rx_eof(struct ptnet_queue *pq, unsigned int budget, bool may_resched)
1673 {
1674 struct ptnet_softc *sc = pq->sc;
1675 bool have_vnet_hdr = sc->vnet_hdr_len;
1676 struct nm_csb_atok *atok = pq->atok;
1677 struct nm_csb_ktoa *ktoa = pq->ktoa;
1678 struct netmap_adapter *na = &sc->ptna->dr.up;
1679 struct netmap_kring *kring = na->rx_rings[pq->kring_id];
1680 struct netmap_ring *ring = kring->ring;
1681 unsigned int const lim = kring->nkr_num_slots - 1;
1682 unsigned int batch_count = 0;
1683 if_t ifp = sc->ifp;
1684 unsigned int count = 0;
1685 uint32_t head;
1686
1687 PTNET_Q_LOCK(pq);
1688
1689 if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) {
1690 goto unlock;
1691 }
1692
1693 kring->nr_kflags &= ~NKR_PENDINTR;
1694
1695 head = ring->head;
1696 while (count < budget) {
1697 uint32_t prev_head = head;
1698 struct mbuf *mhead, *mtail;
1699 struct virtio_net_hdr *vh;
1700 struct netmap_slot *slot;
1701 unsigned int nmbuf_len;
1702 uint8_t *nmbuf;
1703 int deliver = 1; /* the mbuf to the network stack. */
1704 host_sync:
1705 if (head == ring->tail) {
1706 /* We ran out of slot, let's see if the host has
1707 * added some, by reading hwcur and hwtail from
1708 * the CSB. */
1709 ptnet_sync_tail(ktoa, kring);
1710
1711 if (head == ring->tail) {
1712 /* Still no slots available. Reactivate
1713 * interrupts as they were disabled by the
1714 * host thread right before issuing the
1715 * last interrupt. */
1716 atok->appl_need_kick = 1;
1717
1718 /* Double check for more completed RX slots.
1719 * We need a full barrier to prevent the store
1720 * to atok->appl_need_kick to be reordered with
1721 * the load from ktoa->hwcur and ktoa->hwtail
1722 * (store-load barrier). */
1723 nm_stld_barrier();
1724 ptnet_sync_tail(ktoa, kring);
1725 if (likely(head == ring->tail)) {
1726 break;
1727 }
1728 atok->appl_need_kick = 0;
1729 }
1730 }
1731
1732 /* Initialize ring state variables, possibly grabbing the
1733 * virtio-net header. */
1734 slot = ring->slot + head;
1735 nmbuf = NMB(na, slot);
1736 nmbuf_len = slot->len;
1737
1738 vh = (struct virtio_net_hdr *)nmbuf;
1739 if (have_vnet_hdr) {
1740 if (unlikely(nmbuf_len < PTNET_HDR_SIZE)) {
1741 /* There is no good reason why host should
1742 * put the header in multiple netmap slots.
1743 * If this is the case, discard. */
1744 nm_prlim(1, "Fragmented vnet-hdr: dropping");
1745 head = ptnet_rx_discard(kring, head);
1746 pq->stats.iqdrops ++;
1747 deliver = 0;
1748 goto skip;
1749 }
1750 nm_prdis(1, "%s: vnet hdr: flags %x csum_start %u "
1751 "csum_ofs %u hdr_len = %u gso_size %u "
1752 "gso_type %x", __func__, vh->flags,
1753 vh->csum_start, vh->csum_offset, vh->hdr_len,
1754 vh->gso_size, vh->gso_type);
1755 nmbuf += PTNET_HDR_SIZE;
1756 nmbuf_len -= PTNET_HDR_SIZE;
1757 }
1758
1759 /* Allocate the head of a new mbuf chain.
1760 * We use m_getcl() to allocate an mbuf with standard cluster
1761 * size (MCLBYTES). In the future we could use m_getjcl()
1762 * to choose different sizes. */
1763 mhead = mtail = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
1764 if (unlikely(mhead == NULL)) {
1765 device_printf(sc->dev, "%s: failed to allocate mbuf "
1766 "head\n", __func__);
1767 pq->stats.errors ++;
1768 break;
1769 }
1770
1771 /* Initialize the mbuf state variables. */
1772 mhead->m_pkthdr.len = nmbuf_len;
1773 mtail->m_len = 0;
1774
1775 /* Scan all the netmap slots containing the current packet. */
1776 for (;;) {
1777 DBG(device_printf(sc->dev, "%s: h %u t %u rcv frag "
1778 "len %u, flags %u\n", __func__,
1779 head, ring->tail, slot->len,
1780 slot->flags));
1781
1782 mtail = ptnet_rx_slot(mtail, nmbuf, nmbuf_len);
1783 if (unlikely(!mtail)) {
1784 /* Ouch. We ran out of memory while processing
1785 * a packet. We have to restore the previous
1786 * head position, free the mbuf chain, and
1787 * schedule the taskqueue to give the packet
1788 * another chance. */
1789 device_printf(sc->dev, "%s: failed to allocate"
1790 " mbuf frag, reset head %u --> %u\n",
1791 __func__, head, prev_head);
1792 head = prev_head;
1793 m_freem(mhead);
1794 pq->stats.errors ++;
1795 if (may_resched) {
1796 taskqueue_enqueue(pq->taskq,
1797 &pq->task);
1798 }
1799 goto escape;
1800 }
1801
1802 /* We have to increment head irrespective of the
1803 * NS_MOREFRAG being set or not. */
1804 head = nm_next(head, lim);
1805
1806 if (!(slot->flags & NS_MOREFRAG)) {
1807 break;
1808 }
1809
1810 if (unlikely(head == ring->tail)) {
1811 /* The very last slot prepared by the host has
1812 * the NS_MOREFRAG set. Drop it and continue
1813 * the outer cycle (to do the double-check). */
1814 nm_prlim(1, "Incomplete packet: dropping");
1815 m_freem(mhead);
1816 pq->stats.iqdrops ++;
1817 goto host_sync;
1818 }
1819
1820 slot = ring->slot + head;
1821 nmbuf = NMB(na, slot);
1822 nmbuf_len = slot->len;
1823 mhead->m_pkthdr.len += nmbuf_len;
1824 }
1825
1826 mhead->m_pkthdr.rcvif = ifp;
1827 mhead->m_pkthdr.csum_flags = 0;
1828
1829 /* Store the queue idx in the packet header. */
1830 mhead->m_pkthdr.flowid = pq->kring_id;
1831 M_HASHTYPE_SET(mhead, M_HASHTYPE_OPAQUE);
1832
1833 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1834 struct ether_header *eh;
1835
1836 eh = mtod(mhead, struct ether_header *);
1837 if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
1838 ptnet_vlan_tag_remove(mhead);
1839 /*
1840 * With the 802.1Q header removed, update the
1841 * checksum starting location accordingly.
1842 */
1843 if (vh->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
1844 vh->csum_start -= ETHER_VLAN_ENCAP_LEN;
1845 }
1846 }
1847
1848 if (unlikely(have_vnet_hdr && virtio_net_rx_csum(mhead, vh))) {
1849 m_freem(mhead);
1850 nm_prlim(1, "Csum offload error: dropping");
1851 pq->stats.iqdrops ++;
1852 deliver = 0;
1853 }
1854
1855 skip:
1856 count ++;
1857 if (++batch_count >= PTNET_RX_BATCH) {
1858 /* Some packets have been (or will be) pushed to the network
1859 * stack. We need to update the CSB to tell the host about
1860 * the new ring->cur and ring->head (RX buffer refill). */
1861 ptnet_ring_update(pq, kring, head, NAF_FORCE_READ);
1862 batch_count = 0;
1863 }
1864
1865 if (likely(deliver)) {
1866 pq->stats.packets ++;
1867 pq->stats.bytes += mhead->m_pkthdr.len;
1868
1869 PTNET_Q_UNLOCK(pq);
1870 (*ifp->if_input)(ifp, mhead);
1871 PTNET_Q_LOCK(pq);
1872 /* The ring->head index (and related indices) are
1873 * updated under pq lock by ptnet_ring_update().
1874 * Since we dropped the lock to call if_input(), we
1875 * must reload ring->head and restart processing the
1876 * ring from there. */
1877 head = ring->head;
1878
1879 if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) {
1880 /* The interface has gone down while we didn't
1881 * have the lock. Stop any processing and exit. */
1882 goto unlock;
1883 }
1884 }
1885 }
1886 escape:
1887 if (batch_count) {
1888 ptnet_ring_update(pq, kring, head, NAF_FORCE_READ);
1889
1890 }
1891
1892 if (count >= budget && may_resched) {
1893 /* If we ran out of budget or the double-check found new
1894 * slots to process, schedule the taskqueue. */
1895 DBG(nm_prlim(1, "out of budget: resched h %u t %u\n",
1896 head, ring->tail));
1897 taskqueue_enqueue(pq->taskq, &pq->task);
1898 }
1899 unlock:
1900 PTNET_Q_UNLOCK(pq);
1901
1902 return count;
1903 }
1904
1905 static void
ptnet_rx_task(void * context,int pending)1906 ptnet_rx_task(void *context, int pending)
1907 {
1908 struct ptnet_queue *pq = context;
1909
1910 DBG(nm_prlim(1, "%s: pq #%u\n", __func__, pq->kring_id));
1911 ptnet_rx_eof(pq, PTNET_RX_BUDGET, true);
1912 }
1913
1914 static void
ptnet_tx_task(void * context,int pending)1915 ptnet_tx_task(void *context, int pending)
1916 {
1917 struct ptnet_queue *pq = context;
1918
1919 DBG(nm_prlim(1, "%s: pq #%u\n", __func__, pq->kring_id));
1920 ptnet_drain_transmit_queue(pq, PTNET_TX_BUDGET, true);
1921 }
1922
1923 #ifdef DEVICE_POLLING
1924 /* We don't need to handle differently POLL_AND_CHECK_STATUS and
1925 * POLL_ONLY, since we don't have an Interrupt Status Register. */
1926 static int
ptnet_poll(if_t ifp,enum poll_cmd cmd,int budget)1927 ptnet_poll(if_t ifp, enum poll_cmd cmd, int budget)
1928 {
1929 struct ptnet_softc *sc = if_getsoftc(ifp);
1930 unsigned int queue_budget;
1931 unsigned int count = 0;
1932 bool borrow = false;
1933 int i;
1934
1935 KASSERT(sc->num_rings > 0, ("Found no queues in while polling ptnet"));
1936 queue_budget = MAX(budget / sc->num_rings, 1);
1937 nm_prlim(1, "Per-queue budget is %d", queue_budget);
1938
1939 while (budget) {
1940 unsigned int rcnt = 0;
1941
1942 for (i = 0; i < sc->num_rings; i++) {
1943 struct ptnet_queue *pq = sc->queues + i;
1944
1945 if (borrow) {
1946 queue_budget = MIN(queue_budget, budget);
1947 if (queue_budget == 0) {
1948 break;
1949 }
1950 }
1951
1952 if (i < sc->num_tx_rings) {
1953 rcnt += ptnet_drain_transmit_queue(pq,
1954 queue_budget, false);
1955 } else {
1956 rcnt += ptnet_rx_eof(pq, queue_budget,
1957 false);
1958 }
1959 }
1960
1961 if (!rcnt) {
1962 /* A scan of the queues gave no result, we can
1963 * stop here. */
1964 break;
1965 }
1966
1967 if (rcnt > budget) {
1968 /* This may happen when initial budget < sc->num_rings,
1969 * since one packet budget is given to each queue
1970 * anyway. Just pretend we didn't eat "so much". */
1971 rcnt = budget;
1972 }
1973 count += rcnt;
1974 budget -= rcnt;
1975 borrow = true;
1976 }
1977
1978
1979 return count;
1980 }
1981 #endif /* DEVICE_POLLING */
1982 #endif /* WITH_PTNETMAP */
1983