1 /*-
2 * Copyright (c) 2016, Vincenzo Maffione
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * $FreeBSD: stable/12/sys/dev/netmap/if_ptnet.c 372829 2022-12-31 12:30:28Z git2svn $
27 */
28
29 /* Driver for ptnet paravirtualized network device. */
30
31 #include <sys/cdefs.h>
32
33 #include <sys/types.h>
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
37 #include <sys/sockio.h>
38 #include <sys/mbuf.h>
39 #include <sys/malloc.h>
40 #include <sys/module.h>
41 #include <sys/socket.h>
42 #include <sys/sysctl.h>
43 #include <sys/lock.h>
44 #include <sys/mutex.h>
45 #include <sys/taskqueue.h>
46 #include <sys/smp.h>
47 #include <sys/time.h>
48 #include <machine/smp.h>
49
50 #include <vm/uma.h>
51 #include <vm/vm.h>
52 #include <vm/pmap.h>
53
54 #include <net/ethernet.h>
55 #include <net/if.h>
56 #include <net/if_var.h>
57 #include <net/if_arp.h>
58 #include <net/if_dl.h>
59 #include <net/if_types.h>
60 #include <net/if_media.h>
61 #include <net/if_vlan_var.h>
62 #include <net/bpf.h>
63
64 #include <netinet/in_systm.h>
65 #include <netinet/in.h>
66 #include <netinet/ip.h>
67 #include <netinet/ip6.h>
68 #include <netinet6/ip6_var.h>
69 #include <netinet/udp.h>
70 #include <netinet/tcp.h>
71
72 #include <machine/bus.h>
73 #include <machine/resource.h>
74 #include <sys/bus.h>
75 #include <sys/rman.h>
76
77 #include <dev/pci/pcivar.h>
78 #include <dev/pci/pcireg.h>
79
80 #include "opt_inet.h"
81 #include "opt_inet6.h"
82
83 #include <sys/selinfo.h>
84 #include <net/netmap.h>
85 #include <dev/netmap/netmap_kern.h>
86 #include <net/netmap_virt.h>
87 #include <dev/netmap/netmap_mem2.h>
88 #include <dev/virtio/network/virtio_net.h>
89
90 #ifdef WITH_PTNETMAP
91
92 #ifndef INET
93 #error "INET not defined, cannot support offloadings"
94 #endif
95
96 static uint64_t ptnet_get_counter(if_t, ift_counter);
97
98 //#define PTNETMAP_STATS
99 //#define DEBUG
100 #ifdef DEBUG
101 #define DBG(x) x
102 #else /* !DEBUG */
103 #define DBG(x)
104 #endif /* !DEBUG */
105
106 extern int ptnet_vnet_hdr; /* Tunable parameter */
107
108 struct ptnet_softc;
109
110 struct ptnet_queue_stats {
111 uint64_t packets; /* if_[io]packets */
112 uint64_t bytes; /* if_[io]bytes */
113 uint64_t errors; /* if_[io]errors */
114 uint64_t iqdrops; /* if_iqdrops */
115 uint64_t mcasts; /* if_[io]mcasts */
116 #ifdef PTNETMAP_STATS
117 uint64_t intrs;
118 uint64_t kicks;
119 #endif /* PTNETMAP_STATS */
120 };
121
122 struct ptnet_queue {
123 struct ptnet_softc *sc;
124 struct resource *irq;
125 void *cookie;
126 int kring_id;
127 struct nm_csb_atok *atok;
128 struct nm_csb_ktoa *ktoa;
129 unsigned int kick;
130 struct mtx lock;
131 struct buf_ring *bufring; /* for TX queues */
132 struct ptnet_queue_stats stats;
133 #ifdef PTNETMAP_STATS
134 struct ptnet_queue_stats last_stats;
135 #endif /* PTNETMAP_STATS */
136 struct taskqueue *taskq;
137 struct task task;
138 char lock_name[16];
139 };
140
141 #define PTNET_Q_LOCK(_pq) mtx_lock(&(_pq)->lock)
142 #define PTNET_Q_TRYLOCK(_pq) mtx_trylock(&(_pq)->lock)
143 #define PTNET_Q_UNLOCK(_pq) mtx_unlock(&(_pq)->lock)
144
145 struct ptnet_softc {
146 device_t dev;
147 if_t ifp;
148 struct ifmedia media;
149 struct mtx lock;
150 char lock_name[16];
151 char hwaddr[ETHER_ADDR_LEN];
152
153 /* Mirror of PTFEAT register. */
154 uint32_t ptfeatures;
155 unsigned int vnet_hdr_len;
156
157 /* PCI BARs support. */
158 struct resource *iomem;
159 struct resource *msix_mem;
160
161 unsigned int num_rings;
162 unsigned int num_tx_rings;
163 struct ptnet_queue *queues;
164 struct ptnet_queue *rxqueues;
165 struct nm_csb_atok *csb_gh;
166 struct nm_csb_ktoa *csb_hg;
167
168 unsigned int min_tx_space;
169
170 struct netmap_pt_guest_adapter *ptna;
171
172 struct callout tick;
173 #ifdef PTNETMAP_STATS
174 struct timeval last_ts;
175 #endif /* PTNETMAP_STATS */
176 };
177
178 #define PTNET_CORE_LOCK(_sc) mtx_lock(&(_sc)->lock)
179 #define PTNET_CORE_UNLOCK(_sc) mtx_unlock(&(_sc)->lock)
180
181 static int ptnet_probe(device_t);
182 static int ptnet_attach(device_t);
183 static int ptnet_detach(device_t);
184 static int ptnet_suspend(device_t);
185 static int ptnet_resume(device_t);
186 static int ptnet_shutdown(device_t);
187
188 static void ptnet_init(void *opaque);
189 static int ptnet_ioctl(if_t ifp, u_long cmd, caddr_t data);
190 static int ptnet_init_locked(struct ptnet_softc *sc);
191 static int ptnet_stop(struct ptnet_softc *sc);
192 static int ptnet_transmit(if_t ifp, struct mbuf *m);
193 static int ptnet_drain_transmit_queue(struct ptnet_queue *pq,
194 unsigned int budget,
195 bool may_resched);
196 static void ptnet_qflush(if_t ifp);
197 static void ptnet_tx_task(void *context, int pending);
198
199 static int ptnet_media_change(if_t ifp);
200 static void ptnet_media_status(if_t ifp, struct ifmediareq *ifmr);
201 #ifdef PTNETMAP_STATS
202 static void ptnet_tick(void *opaque);
203 #endif
204
205 static int ptnet_irqs_init(struct ptnet_softc *sc);
206 static void ptnet_irqs_fini(struct ptnet_softc *sc);
207
208 static uint32_t ptnet_nm_ptctl(struct ptnet_softc *sc, uint32_t cmd);
209 static int ptnet_nm_config(struct netmap_adapter *na,
210 struct nm_config_info *info);
211 static void ptnet_update_vnet_hdr(struct ptnet_softc *sc);
212 static int ptnet_nm_register(struct netmap_adapter *na, int onoff);
213 static int ptnet_nm_txsync(struct netmap_kring *kring, int flags);
214 static int ptnet_nm_rxsync(struct netmap_kring *kring, int flags);
215 static void ptnet_nm_intr(struct netmap_adapter *na, int onoff);
216
217 static void ptnet_tx_intr(void *opaque);
218 static void ptnet_rx_intr(void *opaque);
219
220 static unsigned ptnet_rx_discard(struct netmap_kring *kring,
221 unsigned int head);
222 static int ptnet_rx_eof(struct ptnet_queue *pq, unsigned int budget,
223 bool may_resched);
224 static void ptnet_rx_task(void *context, int pending);
225
226 #ifdef DEVICE_POLLING
227 static poll_handler_t ptnet_poll;
228 #endif
229
230 static device_method_t ptnet_methods[] = {
231 DEVMETHOD(device_probe, ptnet_probe),
232 DEVMETHOD(device_attach, ptnet_attach),
233 DEVMETHOD(device_detach, ptnet_detach),
234 DEVMETHOD(device_suspend, ptnet_suspend),
235 DEVMETHOD(device_resume, ptnet_resume),
236 DEVMETHOD(device_shutdown, ptnet_shutdown),
237 DEVMETHOD_END
238 };
239
240 static driver_t ptnet_driver = {
241 "ptnet",
242 ptnet_methods,
243 sizeof(struct ptnet_softc)
244 };
245
246 /* We use (SI_ORDER_MIDDLE+2) here, see DEV_MODULE_ORDERED() invocation. */
247 static devclass_t ptnet_devclass;
248 DRIVER_MODULE_ORDERED(ptnet, pci, ptnet_driver, ptnet_devclass,
249 NULL, NULL, SI_ORDER_MIDDLE + 2);
250
251 static int
ptnet_probe(device_t dev)252 ptnet_probe(device_t dev)
253 {
254 if (pci_get_vendor(dev) != PTNETMAP_PCI_VENDOR_ID ||
255 pci_get_device(dev) != PTNETMAP_PCI_NETIF_ID) {
256 return (ENXIO);
257 }
258
259 device_set_desc(dev, "ptnet network adapter");
260
261 return (BUS_PROBE_DEFAULT);
262 }
263
ptnet_kick(struct ptnet_queue * pq)264 static inline void ptnet_kick(struct ptnet_queue *pq)
265 {
266 #ifdef PTNETMAP_STATS
267 pq->stats.kicks ++;
268 #endif /* PTNETMAP_STATS */
269 bus_write_4(pq->sc->iomem, pq->kick, 0);
270 }
271
272 #define PTNET_BUF_RING_SIZE 4096
273 #define PTNET_RX_BUDGET 512
274 #define PTNET_RX_BATCH 1
275 #define PTNET_TX_BUDGET 512
276 #define PTNET_TX_BATCH 64
277 #define PTNET_HDR_SIZE sizeof(struct virtio_net_hdr_mrg_rxbuf)
278 #define PTNET_MAX_PKT_SIZE 65536
279
280 #define PTNET_CSUM_OFFLOAD (CSUM_TCP | CSUM_UDP)
281 #define PTNET_CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6 | CSUM_UDP_IPV6)
282 #define PTNET_ALL_OFFLOAD (CSUM_TSO | PTNET_CSUM_OFFLOAD |\
283 PTNET_CSUM_OFFLOAD_IPV6)
284
285 static int
ptnet_attach(device_t dev)286 ptnet_attach(device_t dev)
287 {
288 uint32_t ptfeatures = 0;
289 unsigned int num_rx_rings, num_tx_rings;
290 struct netmap_adapter na_arg;
291 unsigned int nifp_offset;
292 struct ptnet_softc *sc;
293 if_t ifp;
294 uint32_t macreg;
295 int err, rid;
296 int i;
297
298 sc = device_get_softc(dev);
299 sc->dev = dev;
300
301 /* Setup PCI resources. */
302 pci_enable_busmaster(dev);
303
304 rid = PCIR_BAR(PTNETMAP_IO_PCI_BAR);
305 sc->iomem = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
306 RF_ACTIVE);
307 if (sc->iomem == NULL) {
308 device_printf(dev, "Failed to map I/O BAR\n");
309 return (ENXIO);
310 }
311
312 /* Negotiate features with the hypervisor. */
313 if (ptnet_vnet_hdr) {
314 ptfeatures |= PTNETMAP_F_VNET_HDR;
315 }
316 bus_write_4(sc->iomem, PTNET_IO_PTFEAT, ptfeatures); /* wanted */
317 ptfeatures = bus_read_4(sc->iomem, PTNET_IO_PTFEAT); /* acked */
318 sc->ptfeatures = ptfeatures;
319
320 num_tx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_RINGS);
321 num_rx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_RINGS);
322 sc->num_rings = num_tx_rings + num_rx_rings;
323 sc->num_tx_rings = num_tx_rings;
324
325 if (sc->num_rings * sizeof(struct nm_csb_atok) > PAGE_SIZE) {
326 device_printf(dev, "CSB cannot handle that many rings (%u)\n",
327 sc->num_rings);
328 err = ENOMEM;
329 goto err_path;
330 }
331
332 /* Allocate CSB and carry out CSB allocation protocol. */
333 sc->csb_gh = contigmalloc(2*PAGE_SIZE, M_DEVBUF, M_NOWAIT | M_ZERO,
334 (size_t)0, -1UL, PAGE_SIZE, 0);
335 if (sc->csb_gh == NULL) {
336 device_printf(dev, "Failed to allocate CSB\n");
337 err = ENOMEM;
338 goto err_path;
339 }
340 sc->csb_hg = (struct nm_csb_ktoa *)(((char *)sc->csb_gh) + PAGE_SIZE);
341
342 {
343 /*
344 * We use uint64_t rather than vm_paddr_t since we
345 * need 64 bit addresses even on 32 bit platforms.
346 */
347 uint64_t paddr = vtophys(sc->csb_gh);
348
349 /* CSB allocation protocol: write to BAH first, then
350 * to BAL (for both GH and HG sections). */
351 bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAH,
352 (paddr >> 32) & 0xffffffff);
353 bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAL,
354 paddr & 0xffffffff);
355 paddr = vtophys(sc->csb_hg);
356 bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAH,
357 (paddr >> 32) & 0xffffffff);
358 bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAL,
359 paddr & 0xffffffff);
360 }
361
362 /* Allocate and initialize per-queue data structures. */
363 sc->queues = malloc(sizeof(struct ptnet_queue) * sc->num_rings,
364 M_DEVBUF, M_NOWAIT | M_ZERO);
365 if (sc->queues == NULL) {
366 err = ENOMEM;
367 goto err_path;
368 }
369 sc->rxqueues = sc->queues + num_tx_rings;
370
371 for (i = 0; i < sc->num_rings; i++) {
372 struct ptnet_queue *pq = sc->queues + i;
373
374 pq->sc = sc;
375 pq->kring_id = i;
376 pq->kick = PTNET_IO_KICK_BASE + 4 * i;
377 pq->atok = sc->csb_gh + i;
378 pq->ktoa = sc->csb_hg + i;
379 snprintf(pq->lock_name, sizeof(pq->lock_name), "%s-%d",
380 device_get_nameunit(dev), i);
381 mtx_init(&pq->lock, pq->lock_name, NULL, MTX_DEF);
382 if (i >= num_tx_rings) {
383 /* RX queue: fix kring_id. */
384 pq->kring_id -= num_tx_rings;
385 } else {
386 /* TX queue: allocate buf_ring. */
387 pq->bufring = buf_ring_alloc(PTNET_BUF_RING_SIZE,
388 M_DEVBUF, M_NOWAIT, &pq->lock);
389 if (pq->bufring == NULL) {
390 err = ENOMEM;
391 goto err_path;
392 }
393 }
394 }
395
396 sc->min_tx_space = 64; /* Safe initial value. */
397
398 err = ptnet_irqs_init(sc);
399 if (err) {
400 goto err_path;
401 }
402
403 /* Setup Ethernet interface. */
404 sc->ifp = ifp = if_alloc(IFT_ETHER);
405 if (ifp == NULL) {
406 device_printf(dev, "Failed to allocate ifnet\n");
407 err = ENOMEM;
408 goto err_path;
409 }
410
411 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
412 ifp->if_baudrate = IF_Gbps(10);
413 ifp->if_softc = sc;
414 ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;
415 ifp->if_init = ptnet_init;
416 ifp->if_ioctl = ptnet_ioctl;
417 ifp->if_get_counter = ptnet_get_counter;
418 ifp->if_transmit = ptnet_transmit;
419 ifp->if_qflush = ptnet_qflush;
420
421 ifmedia_init(&sc->media, IFM_IMASK, ptnet_media_change,
422 ptnet_media_status);
423 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX, 0, NULL);
424 ifmedia_set(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX);
425
426 macreg = bus_read_4(sc->iomem, PTNET_IO_MAC_HI);
427 sc->hwaddr[0] = (macreg >> 8) & 0xff;
428 sc->hwaddr[1] = macreg & 0xff;
429 macreg = bus_read_4(sc->iomem, PTNET_IO_MAC_LO);
430 sc->hwaddr[2] = (macreg >> 24) & 0xff;
431 sc->hwaddr[3] = (macreg >> 16) & 0xff;
432 sc->hwaddr[4] = (macreg >> 8) & 0xff;
433 sc->hwaddr[5] = macreg & 0xff;
434
435 ether_ifattach(ifp, sc->hwaddr);
436
437 ifp->if_hdrlen = sizeof(struct ether_vlan_header);
438 ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU;
439
440 if (sc->ptfeatures & PTNETMAP_F_VNET_HDR) {
441 /* Similarly to what the vtnet driver does, we can emulate
442 * VLAN offloadings by inserting and removing the 802.1Q
443 * header during transmit and receive. We are then able
444 * to do checksum offloading of VLAN frames. */
445 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6
446 | IFCAP_VLAN_HWCSUM
447 | IFCAP_TSO | IFCAP_LRO
448 | IFCAP_VLAN_HWTSO
449 | IFCAP_VLAN_HWTAGGING;
450 }
451
452 ifp->if_capenable = ifp->if_capabilities;
453 #ifdef DEVICE_POLLING
454 /* Don't enable polling by default. */
455 ifp->if_capabilities |= IFCAP_POLLING;
456 #endif
457 snprintf(sc->lock_name, sizeof(sc->lock_name),
458 "%s", device_get_nameunit(dev));
459 mtx_init(&sc->lock, sc->lock_name, "ptnet core lock", MTX_DEF);
460 callout_init_mtx(&sc->tick, &sc->lock, 0);
461
462 /* Prepare a netmap_adapter struct instance to do netmap_attach(). */
463 nifp_offset = bus_read_4(sc->iomem, PTNET_IO_NIFP_OFS);
464 memset(&na_arg, 0, sizeof(na_arg));
465 na_arg.ifp = ifp;
466 na_arg.num_tx_desc = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_SLOTS);
467 na_arg.num_rx_desc = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_SLOTS);
468 na_arg.num_tx_rings = num_tx_rings;
469 na_arg.num_rx_rings = num_rx_rings;
470 na_arg.nm_config = ptnet_nm_config;
471 na_arg.nm_krings_create = ptnet_nm_krings_create;
472 na_arg.nm_krings_delete = ptnet_nm_krings_delete;
473 na_arg.nm_dtor = ptnet_nm_dtor;
474 na_arg.nm_intr = ptnet_nm_intr;
475 na_arg.nm_register = ptnet_nm_register;
476 na_arg.nm_txsync = ptnet_nm_txsync;
477 na_arg.nm_rxsync = ptnet_nm_rxsync;
478
479 netmap_pt_guest_attach(&na_arg, nifp_offset,
480 bus_read_4(sc->iomem, PTNET_IO_HOSTMEMID));
481
482 /* Now a netmap adapter for this ifp has been allocated, and it
483 * can be accessed through NA(ifp). We also have to initialize the CSB
484 * pointer. */
485 sc->ptna = (struct netmap_pt_guest_adapter *)NA(ifp);
486
487 /* If virtio-net header was negotiated, set the virt_hdr_len field in
488 * the netmap adapter, to inform users that this netmap adapter requires
489 * the application to deal with the headers. */
490 ptnet_update_vnet_hdr(sc);
491
492 device_printf(dev, "%s() completed\n", __func__);
493
494 return (0);
495
496 err_path:
497 ptnet_detach(dev);
498 return err;
499 }
500
501 /* Stop host sync-kloop if it was running. */
502 static void
ptnet_device_shutdown(struct ptnet_softc * sc)503 ptnet_device_shutdown(struct ptnet_softc *sc)
504 {
505 ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_DELETE);
506 bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAH, 0);
507 bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAL, 0);
508 bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAH, 0);
509 bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAL, 0);
510 }
511
512 static int
ptnet_detach(device_t dev)513 ptnet_detach(device_t dev)
514 {
515 struct ptnet_softc *sc = device_get_softc(dev);
516 int i;
517
518 ptnet_device_shutdown(sc);
519
520 #ifdef DEVICE_POLLING
521 if (sc->ifp->if_capenable & IFCAP_POLLING) {
522 ether_poll_deregister(sc->ifp);
523 }
524 #endif
525 callout_drain(&sc->tick);
526
527 if (sc->queues) {
528 /* Drain taskqueues before calling if_detach. */
529 for (i = 0; i < sc->num_rings; i++) {
530 struct ptnet_queue *pq = sc->queues + i;
531
532 if (pq->taskq) {
533 taskqueue_drain(pq->taskq, &pq->task);
534 }
535 }
536 }
537
538 if (sc->ifp) {
539 ether_ifdetach(sc->ifp);
540
541 /* Uninitialize netmap adapters for this device. */
542 netmap_detach(sc->ifp);
543
544 ifmedia_removeall(&sc->media);
545 if_free(sc->ifp);
546 sc->ifp = NULL;
547 }
548
549 ptnet_irqs_fini(sc);
550
551 if (sc->csb_gh) {
552 contigfree(sc->csb_gh, 2*PAGE_SIZE, M_DEVBUF);
553 sc->csb_gh = NULL;
554 sc->csb_hg = NULL;
555 }
556
557 if (sc->queues) {
558 for (i = 0; i < sc->num_rings; i++) {
559 struct ptnet_queue *pq = sc->queues + i;
560
561 if (mtx_initialized(&pq->lock)) {
562 mtx_destroy(&pq->lock);
563 }
564 if (pq->bufring != NULL) {
565 buf_ring_free(pq->bufring, M_DEVBUF);
566 }
567 }
568 free(sc->queues, M_DEVBUF);
569 sc->queues = NULL;
570 }
571
572 if (sc->iomem) {
573 bus_release_resource(dev, SYS_RES_IOPORT,
574 PCIR_BAR(PTNETMAP_IO_PCI_BAR), sc->iomem);
575 sc->iomem = NULL;
576 }
577
578 mtx_destroy(&sc->lock);
579
580 device_printf(dev, "%s() completed\n", __func__);
581
582 return (0);
583 }
584
585 static int
ptnet_suspend(device_t dev)586 ptnet_suspend(device_t dev)
587 {
588 struct ptnet_softc *sc = device_get_softc(dev);
589
590 (void)sc;
591
592 return (0);
593 }
594
595 static int
ptnet_resume(device_t dev)596 ptnet_resume(device_t dev)
597 {
598 struct ptnet_softc *sc = device_get_softc(dev);
599
600 (void)sc;
601
602 return (0);
603 }
604
605 static int
ptnet_shutdown(device_t dev)606 ptnet_shutdown(device_t dev)
607 {
608 struct ptnet_softc *sc = device_get_softc(dev);
609
610 ptnet_device_shutdown(sc);
611
612 return (0);
613 }
614
615 static int
ptnet_irqs_init(struct ptnet_softc * sc)616 ptnet_irqs_init(struct ptnet_softc *sc)
617 {
618 int rid = PCIR_BAR(PTNETMAP_MSIX_PCI_BAR);
619 int nvecs = sc->num_rings;
620 device_t dev = sc->dev;
621 int err = ENOSPC;
622 int cpu_cur;
623 int i;
624
625 if (pci_find_cap(dev, PCIY_MSIX, NULL) != 0) {
626 device_printf(dev, "Could not find MSI-X capability\n");
627 return (ENXIO);
628 }
629
630 sc->msix_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
631 &rid, RF_ACTIVE);
632 if (sc->msix_mem == NULL) {
633 device_printf(dev, "Failed to allocate MSIX PCI BAR\n");
634 return (ENXIO);
635 }
636
637 if (pci_msix_count(dev) < nvecs) {
638 device_printf(dev, "Not enough MSI-X vectors\n");
639 goto err_path;
640 }
641
642 err = pci_alloc_msix(dev, &nvecs);
643 if (err) {
644 device_printf(dev, "Failed to allocate MSI-X vectors\n");
645 goto err_path;
646 }
647
648 for (i = 0; i < nvecs; i++) {
649 struct ptnet_queue *pq = sc->queues + i;
650
651 rid = i + 1;
652 pq->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
653 RF_ACTIVE);
654 if (pq->irq == NULL) {
655 device_printf(dev, "Failed to allocate interrupt "
656 "for queue #%d\n", i);
657 err = ENOSPC;
658 goto err_path;
659 }
660 }
661
662 cpu_cur = CPU_FIRST();
663 for (i = 0; i < nvecs; i++) {
664 struct ptnet_queue *pq = sc->queues + i;
665 void (*handler)(void *) = ptnet_tx_intr;
666
667 if (i >= sc->num_tx_rings) {
668 handler = ptnet_rx_intr;
669 }
670 err = bus_setup_intr(dev, pq->irq, INTR_TYPE_NET | INTR_MPSAFE,
671 NULL /* intr_filter */, handler,
672 pq, &pq->cookie);
673 if (err) {
674 device_printf(dev, "Failed to register intr handler "
675 "for queue #%d\n", i);
676 goto err_path;
677 }
678
679 bus_describe_intr(dev, pq->irq, pq->cookie, "q%d", i);
680 #if 0
681 bus_bind_intr(sc->dev, pq->irq, cpu_cur);
682 #endif
683 cpu_cur = CPU_NEXT(cpu_cur);
684 }
685
686 device_printf(dev, "Allocated %d MSI-X vectors\n", nvecs);
687
688 cpu_cur = CPU_FIRST();
689 for (i = 0; i < nvecs; i++) {
690 struct ptnet_queue *pq = sc->queues + i;
691 static void (*handler)(void *context, int pending);
692
693 handler = (i < sc->num_tx_rings) ? ptnet_tx_task : ptnet_rx_task;
694
695 TASK_INIT(&pq->task, 0, handler, pq);
696 pq->taskq = taskqueue_create_fast("ptnet_queue", M_NOWAIT,
697 taskqueue_thread_enqueue, &pq->taskq);
698 taskqueue_start_threads(&pq->taskq, 1, PI_NET, "%s-pq-%d",
699 device_get_nameunit(sc->dev), cpu_cur);
700 cpu_cur = CPU_NEXT(cpu_cur);
701 }
702
703 return 0;
704 err_path:
705 ptnet_irqs_fini(sc);
706 return err;
707 }
708
709 static void
ptnet_irqs_fini(struct ptnet_softc * sc)710 ptnet_irqs_fini(struct ptnet_softc *sc)
711 {
712 device_t dev = sc->dev;
713 int i;
714
715 for (i = 0; i < sc->num_rings; i++) {
716 struct ptnet_queue *pq = sc->queues + i;
717
718 if (pq->taskq) {
719 taskqueue_free(pq->taskq);
720 pq->taskq = NULL;
721 }
722
723 if (pq->cookie) {
724 bus_teardown_intr(dev, pq->irq, pq->cookie);
725 pq->cookie = NULL;
726 }
727
728 if (pq->irq) {
729 bus_release_resource(dev, SYS_RES_IRQ, i + 1, pq->irq);
730 pq->irq = NULL;
731 }
732 }
733
734 if (sc->msix_mem) {
735 pci_release_msi(dev);
736
737 bus_release_resource(dev, SYS_RES_MEMORY,
738 PCIR_BAR(PTNETMAP_MSIX_PCI_BAR),
739 sc->msix_mem);
740 sc->msix_mem = NULL;
741 }
742 }
743
744 static void
ptnet_init(void * opaque)745 ptnet_init(void *opaque)
746 {
747 struct ptnet_softc *sc = opaque;
748
749 PTNET_CORE_LOCK(sc);
750 ptnet_init_locked(sc);
751 PTNET_CORE_UNLOCK(sc);
752 }
753
754 static int
ptnet_ioctl(if_t ifp,u_long cmd,caddr_t data)755 ptnet_ioctl(if_t ifp, u_long cmd, caddr_t data)
756 {
757 struct ptnet_softc *sc = if_getsoftc(ifp);
758 device_t dev = sc->dev;
759 struct ifreq *ifr = (struct ifreq *)data;
760 int mask __unused, err = 0;
761
762 switch (cmd) {
763 case SIOCSIFFLAGS:
764 device_printf(dev, "SIOCSIFFLAGS %x\n", ifp->if_flags);
765 PTNET_CORE_LOCK(sc);
766 if (ifp->if_flags & IFF_UP) {
767 /* Network stack wants the iff to be up. */
768 err = ptnet_init_locked(sc);
769 } else {
770 /* Network stack wants the iff to be down. */
771 err = ptnet_stop(sc);
772 }
773 /* We don't need to do nothing to support IFF_PROMISC,
774 * since that is managed by the backend port. */
775 PTNET_CORE_UNLOCK(sc);
776 break;
777
778 case SIOCSIFCAP:
779 device_printf(dev, "SIOCSIFCAP %x %x\n",
780 ifr->ifr_reqcap, ifp->if_capenable);
781 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
782 #ifdef DEVICE_POLLING
783 if (mask & IFCAP_POLLING) {
784 struct ptnet_queue *pq;
785 int i;
786
787 if (ifr->ifr_reqcap & IFCAP_POLLING) {
788 err = ether_poll_register(ptnet_poll, ifp);
789 if (err) {
790 break;
791 }
792 /* Stop queues and sync with taskqueues. */
793 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
794 for (i = 0; i < sc->num_rings; i++) {
795 pq = sc-> queues + i;
796 /* Make sure the worker sees the
797 * IFF_DRV_RUNNING down. */
798 PTNET_Q_LOCK(pq);
799 pq->atok->appl_need_kick = 0;
800 PTNET_Q_UNLOCK(pq);
801 /* Wait for rescheduling to finish. */
802 if (pq->taskq) {
803 taskqueue_drain(pq->taskq,
804 &pq->task);
805 }
806 }
807 ifp->if_drv_flags |= IFF_DRV_RUNNING;
808 } else {
809 err = ether_poll_deregister(ifp);
810 for (i = 0; i < sc->num_rings; i++) {
811 pq = sc-> queues + i;
812 PTNET_Q_LOCK(pq);
813 pq->atok->appl_need_kick = 1;
814 PTNET_Q_UNLOCK(pq);
815 }
816 }
817 }
818 #endif /* DEVICE_POLLING */
819 ifp->if_capenable = ifr->ifr_reqcap;
820 break;
821
822 case SIOCSIFMTU:
823 /* We support any reasonable MTU. */
824 if (ifr->ifr_mtu < ETHERMIN ||
825 ifr->ifr_mtu > PTNET_MAX_PKT_SIZE) {
826 err = EINVAL;
827 } else {
828 PTNET_CORE_LOCK(sc);
829 ifp->if_mtu = ifr->ifr_mtu;
830 PTNET_CORE_UNLOCK(sc);
831 }
832 break;
833
834 case SIOCSIFMEDIA:
835 case SIOCGIFMEDIA:
836 err = ifmedia_ioctl(ifp, ifr, &sc->media, cmd);
837 break;
838
839 default:
840 err = ether_ioctl(ifp, cmd, data);
841 break;
842 }
843
844 return err;
845 }
846
847 static int
ptnet_init_locked(struct ptnet_softc * sc)848 ptnet_init_locked(struct ptnet_softc *sc)
849 {
850 if_t ifp = sc->ifp;
851 struct netmap_adapter *na_dr = &sc->ptna->dr.up;
852 struct netmap_adapter *na_nm = &sc->ptna->hwup.up;
853 unsigned int nm_buf_size;
854 int ret;
855
856 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
857 return 0; /* nothing to do */
858 }
859
860 device_printf(sc->dev, "%s\n", __func__);
861
862 /* Translate offload capabilities according to if_capenable. */
863 ifp->if_hwassist = 0;
864 if (ifp->if_capenable & IFCAP_TXCSUM)
865 ifp->if_hwassist |= PTNET_CSUM_OFFLOAD;
866 if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
867 ifp->if_hwassist |= PTNET_CSUM_OFFLOAD_IPV6;
868 if (ifp->if_capenable & IFCAP_TSO4)
869 ifp->if_hwassist |= CSUM_IP_TSO;
870 if (ifp->if_capenable & IFCAP_TSO6)
871 ifp->if_hwassist |= CSUM_IP6_TSO;
872
873 /*
874 * Prepare the interface for netmap mode access.
875 */
876 netmap_update_config(na_dr);
877
878 ret = netmap_mem_finalize(na_dr->nm_mem, na_dr);
879 if (ret) {
880 device_printf(sc->dev, "netmap_mem_finalize() failed\n");
881 return ret;
882 }
883
884 if (sc->ptna->backend_users == 0) {
885 ret = ptnet_nm_krings_create(na_nm);
886 if (ret) {
887 device_printf(sc->dev, "ptnet_nm_krings_create() "
888 "failed\n");
889 goto err_mem_finalize;
890 }
891
892 ret = netmap_mem_rings_create(na_dr);
893 if (ret) {
894 device_printf(sc->dev, "netmap_mem_rings_create() "
895 "failed\n");
896 goto err_rings_create;
897 }
898
899 ret = netmap_mem_get_lut(na_dr->nm_mem, &na_dr->na_lut);
900 if (ret) {
901 device_printf(sc->dev, "netmap_mem_get_lut() "
902 "failed\n");
903 goto err_get_lut;
904 }
905 }
906
907 ret = ptnet_nm_register(na_dr, 1 /* on */);
908 if (ret) {
909 goto err_register;
910 }
911
912 nm_buf_size = NETMAP_BUF_SIZE(na_dr);
913
914 KASSERT(nm_buf_size > 0, ("Invalid netmap buffer size"));
915 sc->min_tx_space = PTNET_MAX_PKT_SIZE / nm_buf_size + 2;
916 device_printf(sc->dev, "%s: min_tx_space = %u\n", __func__,
917 sc->min_tx_space);
918 #ifdef PTNETMAP_STATS
919 callout_reset(&sc->tick, hz, ptnet_tick, sc);
920 #endif
921
922 ifp->if_drv_flags |= IFF_DRV_RUNNING;
923
924 return 0;
925
926 err_register:
927 memset(&na_dr->na_lut, 0, sizeof(na_dr->na_lut));
928 err_get_lut:
929 netmap_mem_rings_delete(na_dr);
930 err_rings_create:
931 ptnet_nm_krings_delete(na_nm);
932 err_mem_finalize:
933 netmap_mem_deref(na_dr->nm_mem, na_dr);
934
935 return ret;
936 }
937
938 /* To be called under core lock. */
939 static int
ptnet_stop(struct ptnet_softc * sc)940 ptnet_stop(struct ptnet_softc *sc)
941 {
942 if_t ifp = sc->ifp;
943 struct netmap_adapter *na_dr = &sc->ptna->dr.up;
944 struct netmap_adapter *na_nm = &sc->ptna->hwup.up;
945 int i;
946
947 device_printf(sc->dev, "%s\n", __func__);
948
949 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
950 return 0; /* nothing to do */
951 }
952
953 /* Clear the driver-ready flag, and synchronize with all the queues,
954 * so that after this loop we are sure nobody is working anymore with
955 * the device. This scheme is taken from the vtnet driver. */
956 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
957 callout_stop(&sc->tick);
958 for (i = 0; i < sc->num_rings; i++) {
959 PTNET_Q_LOCK(sc->queues + i);
960 PTNET_Q_UNLOCK(sc->queues + i);
961 }
962
963 ptnet_nm_register(na_dr, 0 /* off */);
964
965 if (sc->ptna->backend_users == 0) {
966 netmap_mem_rings_delete(na_dr);
967 ptnet_nm_krings_delete(na_nm);
968 }
969 netmap_mem_deref(na_dr->nm_mem, na_dr);
970
971 return 0;
972 }
973
974 static void
ptnet_qflush(if_t ifp)975 ptnet_qflush(if_t ifp)
976 {
977 struct ptnet_softc *sc = if_getsoftc(ifp);
978 int i;
979
980 /* Flush all the bufrings and do the interface flush. */
981 for (i = 0; i < sc->num_rings; i++) {
982 struct ptnet_queue *pq = sc->queues + i;
983 struct mbuf *m;
984
985 PTNET_Q_LOCK(pq);
986 if (pq->bufring) {
987 while ((m = buf_ring_dequeue_sc(pq->bufring))) {
988 m_freem(m);
989 }
990 }
991 PTNET_Q_UNLOCK(pq);
992 }
993
994 if_qflush(ifp);
995 }
996
997 static int
ptnet_media_change(if_t ifp)998 ptnet_media_change(if_t ifp)
999 {
1000 struct ptnet_softc *sc = if_getsoftc(ifp);
1001 struct ifmedia *ifm = &sc->media;
1002
1003 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) {
1004 return EINVAL;
1005 }
1006
1007 return 0;
1008 }
1009
1010 static uint64_t
ptnet_get_counter(if_t ifp,ift_counter cnt)1011 ptnet_get_counter(if_t ifp, ift_counter cnt)
1012 {
1013 struct ptnet_softc *sc = if_getsoftc(ifp);
1014 struct ptnet_queue_stats stats[2];
1015 int i;
1016
1017 /* Accumulate statistics over the queues. */
1018 memset(stats, 0, sizeof(stats));
1019 for (i = 0; i < sc->num_rings; i++) {
1020 struct ptnet_queue *pq = sc->queues + i;
1021 int idx = (i < sc->num_tx_rings) ? 0 : 1;
1022
1023 stats[idx].packets += pq->stats.packets;
1024 stats[idx].bytes += pq->stats.bytes;
1025 stats[idx].errors += pq->stats.errors;
1026 stats[idx].iqdrops += pq->stats.iqdrops;
1027 stats[idx].mcasts += pq->stats.mcasts;
1028 }
1029
1030 switch (cnt) {
1031 case IFCOUNTER_IPACKETS:
1032 return (stats[1].packets);
1033 case IFCOUNTER_IQDROPS:
1034 return (stats[1].iqdrops);
1035 case IFCOUNTER_IERRORS:
1036 return (stats[1].errors);
1037 case IFCOUNTER_OPACKETS:
1038 return (stats[0].packets);
1039 case IFCOUNTER_OBYTES:
1040 return (stats[0].bytes);
1041 case IFCOUNTER_OMCASTS:
1042 return (stats[0].mcasts);
1043 default:
1044 return (if_get_counter_default(ifp, cnt));
1045 }
1046 }
1047
1048
1049 #ifdef PTNETMAP_STATS
1050 /* Called under core lock. */
1051 static void
ptnet_tick(void * opaque)1052 ptnet_tick(void *opaque)
1053 {
1054 struct ptnet_softc *sc = opaque;
1055 int i;
1056
1057 for (i = 0; i < sc->num_rings; i++) {
1058 struct ptnet_queue *pq = sc->queues + i;
1059 struct ptnet_queue_stats cur = pq->stats;
1060 struct timeval now;
1061 unsigned int delta;
1062
1063 microtime(&now);
1064 delta = now.tv_usec - sc->last_ts.tv_usec +
1065 (now.tv_sec - sc->last_ts.tv_sec) * 1000000;
1066 delta /= 1000; /* in milliseconds */
1067
1068 if (delta == 0)
1069 continue;
1070
1071 device_printf(sc->dev, "#%d[%u ms]:pkts %lu, kicks %lu, "
1072 "intr %lu\n", i, delta,
1073 (cur.packets - pq->last_stats.packets),
1074 (cur.kicks - pq->last_stats.kicks),
1075 (cur.intrs - pq->last_stats.intrs));
1076 pq->last_stats = cur;
1077 }
1078 microtime(&sc->last_ts);
1079 callout_schedule(&sc->tick, hz);
1080 }
1081 #endif /* PTNETMAP_STATS */
1082
1083 static void
ptnet_media_status(if_t ifp,struct ifmediareq * ifmr)1084 ptnet_media_status(if_t ifp, struct ifmediareq *ifmr)
1085 {
1086 /* We are always active, as the backend netmap port is
1087 * always open in netmap mode. */
1088 ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE;
1089 ifmr->ifm_active = IFM_ETHER | IFM_10G_T | IFM_FDX;
1090 }
1091
1092 static uint32_t
ptnet_nm_ptctl(struct ptnet_softc * sc,uint32_t cmd)1093 ptnet_nm_ptctl(struct ptnet_softc *sc, uint32_t cmd)
1094 {
1095 /*
1096 * Write a command and read back error status,
1097 * with zero meaning success.
1098 */
1099 bus_write_4(sc->iomem, PTNET_IO_PTCTL, cmd);
1100 return bus_read_4(sc->iomem, PTNET_IO_PTCTL);
1101 }
1102
1103 static int
ptnet_nm_config(struct netmap_adapter * na,struct nm_config_info * info)1104 ptnet_nm_config(struct netmap_adapter *na, struct nm_config_info *info)
1105 {
1106 struct ptnet_softc *sc = if_getsoftc(na->ifp);
1107
1108 info->num_tx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_RINGS);
1109 info->num_rx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_RINGS);
1110 info->num_tx_descs = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_SLOTS);
1111 info->num_rx_descs = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_SLOTS);
1112 info->rx_buf_maxsize = NETMAP_BUF_SIZE(na);
1113
1114 device_printf(sc->dev, "txr %u, rxr %u, txd %u, rxd %u, rxbufsz %u\n",
1115 info->num_tx_rings, info->num_rx_rings,
1116 info->num_tx_descs, info->num_rx_descs,
1117 info->rx_buf_maxsize);
1118
1119 return 0;
1120 }
1121
1122 static void
ptnet_sync_from_csb(struct ptnet_softc * sc,struct netmap_adapter * na)1123 ptnet_sync_from_csb(struct ptnet_softc *sc, struct netmap_adapter *na)
1124 {
1125 int i;
1126
1127 /* Sync krings from the host, reading from
1128 * CSB. */
1129 for (i = 0; i < sc->num_rings; i++) {
1130 struct nm_csb_atok *atok = sc->queues[i].atok;
1131 struct nm_csb_ktoa *ktoa = sc->queues[i].ktoa;
1132 struct netmap_kring *kring;
1133
1134 if (i < na->num_tx_rings) {
1135 kring = na->tx_rings[i];
1136 } else {
1137 kring = na->rx_rings[i - na->num_tx_rings];
1138 }
1139 kring->rhead = kring->ring->head = atok->head;
1140 kring->rcur = kring->ring->cur = atok->cur;
1141 kring->nr_hwcur = ktoa->hwcur;
1142 kring->nr_hwtail = kring->rtail =
1143 kring->ring->tail = ktoa->hwtail;
1144
1145 nm_prdis("%d,%d: csb {hc %u h %u c %u ht %u}", t, i,
1146 ktoa->hwcur, atok->head, atok->cur,
1147 ktoa->hwtail);
1148 nm_prdis("%d,%d: kring {hc %u rh %u rc %u h %u c %u ht %u rt %u t %u}",
1149 t, i, kring->nr_hwcur, kring->rhead, kring->rcur,
1150 kring->ring->head, kring->ring->cur, kring->nr_hwtail,
1151 kring->rtail, kring->ring->tail);
1152 }
1153 }
1154
1155 static void
ptnet_update_vnet_hdr(struct ptnet_softc * sc)1156 ptnet_update_vnet_hdr(struct ptnet_softc *sc)
1157 {
1158 unsigned int wanted_hdr_len = ptnet_vnet_hdr ? PTNET_HDR_SIZE : 0;
1159
1160 bus_write_4(sc->iomem, PTNET_IO_VNET_HDR_LEN, wanted_hdr_len);
1161 sc->vnet_hdr_len = bus_read_4(sc->iomem, PTNET_IO_VNET_HDR_LEN);
1162 sc->ptna->hwup.up.virt_hdr_len = sc->vnet_hdr_len;
1163 }
1164
1165 static int
ptnet_nm_register(struct netmap_adapter * na,int onoff)1166 ptnet_nm_register(struct netmap_adapter *na, int onoff)
1167 {
1168 /* device-specific */
1169 if_t ifp = na->ifp;
1170 struct ptnet_softc *sc = if_getsoftc(ifp);
1171 int native = (na == &sc->ptna->hwup.up);
1172 struct ptnet_queue *pq;
1173 int ret = 0;
1174 int i;
1175
1176 if (!onoff) {
1177 sc->ptna->backend_users--;
1178 }
1179
1180 /* If this is the last netmap client, guest interrupt enable flags may
1181 * be in arbitrary state. Since these flags are going to be used also
1182 * by the netdevice driver, we have to make sure to start with
1183 * notifications enabled. Also, schedule NAPI to flush pending packets
1184 * in the RX rings, since we will not receive further interrupts
1185 * until these will be processed. */
1186 if (native && !onoff && na->active_fds == 0) {
1187 nm_prinf("Exit netmap mode, re-enable interrupts");
1188 for (i = 0; i < sc->num_rings; i++) {
1189 pq = sc->queues + i;
1190 pq->atok->appl_need_kick = 1;
1191 }
1192 }
1193
1194 if (onoff) {
1195 if (sc->ptna->backend_users == 0) {
1196 /* Initialize notification enable fields in the CSB. */
1197 for (i = 0; i < sc->num_rings; i++) {
1198 pq = sc->queues + i;
1199 pq->ktoa->kern_need_kick = 1;
1200 pq->atok->appl_need_kick =
1201 (!(ifp->if_capenable & IFCAP_POLLING)
1202 && i >= sc->num_tx_rings);
1203 }
1204
1205 /* Set the virtio-net header length. */
1206 ptnet_update_vnet_hdr(sc);
1207
1208 /* Make sure the host adapter passed through is ready
1209 * for txsync/rxsync. */
1210 ret = ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_CREATE);
1211 if (ret) {
1212 return ret;
1213 }
1214
1215 /* Align the guest krings and rings to the state stored
1216 * in the CSB. */
1217 ptnet_sync_from_csb(sc, na);
1218 }
1219
1220 /* If not native, don't call nm_set_native_flags, since we don't want
1221 * to replace if_transmit method, nor set NAF_NETMAP_ON */
1222 if (native) {
1223 netmap_krings_mode_commit(na, onoff);
1224 nm_set_native_flags(na);
1225 }
1226
1227 } else {
1228 if (native) {
1229 nm_clear_native_flags(na);
1230 netmap_krings_mode_commit(na, onoff);
1231 }
1232
1233 if (sc->ptna->backend_users == 0) {
1234 ret = ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_DELETE);
1235 }
1236 }
1237
1238 if (onoff) {
1239 sc->ptna->backend_users++;
1240 }
1241
1242 return ret;
1243 }
1244
1245 static int
ptnet_nm_txsync(struct netmap_kring * kring,int flags)1246 ptnet_nm_txsync(struct netmap_kring *kring, int flags)
1247 {
1248 struct ptnet_softc *sc = if_getsoftc(kring->na->ifp);
1249 struct ptnet_queue *pq = sc->queues + kring->ring_id;
1250 bool notify;
1251
1252 notify = netmap_pt_guest_txsync(pq->atok, pq->ktoa, kring, flags);
1253 if (notify) {
1254 ptnet_kick(pq);
1255 }
1256
1257 return 0;
1258 }
1259
1260 static int
ptnet_nm_rxsync(struct netmap_kring * kring,int flags)1261 ptnet_nm_rxsync(struct netmap_kring *kring, int flags)
1262 {
1263 struct ptnet_softc *sc = if_getsoftc(kring->na->ifp);
1264 struct ptnet_queue *pq = sc->rxqueues + kring->ring_id;
1265 bool notify;
1266
1267 notify = netmap_pt_guest_rxsync(pq->atok, pq->ktoa, kring, flags);
1268 if (notify) {
1269 ptnet_kick(pq);
1270 }
1271
1272 return 0;
1273 }
1274
1275 static void
ptnet_nm_intr(struct netmap_adapter * na,int onoff)1276 ptnet_nm_intr(struct netmap_adapter *na, int onoff)
1277 {
1278 struct ptnet_softc *sc = if_getsoftc(na->ifp);
1279 int i;
1280
1281 for (i = 0; i < sc->num_rings; i++) {
1282 struct ptnet_queue *pq = sc->queues + i;
1283 pq->atok->appl_need_kick = onoff;
1284 }
1285 }
1286
1287 static void
ptnet_tx_intr(void * opaque)1288 ptnet_tx_intr(void *opaque)
1289 {
1290 struct ptnet_queue *pq = opaque;
1291 struct ptnet_softc *sc = pq->sc;
1292
1293 DBG(device_printf(sc->dev, "Tx interrupt #%d\n", pq->kring_id));
1294 #ifdef PTNETMAP_STATS
1295 pq->stats.intrs ++;
1296 #endif /* PTNETMAP_STATS */
1297
1298 if (netmap_tx_irq(sc->ifp, pq->kring_id) != NM_IRQ_PASS) {
1299 return;
1300 }
1301
1302 /* Schedule the tasqueue to flush process transmissions requests.
1303 * However, vtnet, if_em and if_igb just call ptnet_transmit() here,
1304 * at least when using MSI-X interrupts. The if_em driver, instead
1305 * schedule taskqueue when using legacy interrupts. */
1306 taskqueue_enqueue(pq->taskq, &pq->task);
1307 }
1308
1309 static void
ptnet_rx_intr(void * opaque)1310 ptnet_rx_intr(void *opaque)
1311 {
1312 struct ptnet_queue *pq = opaque;
1313 struct ptnet_softc *sc = pq->sc;
1314 unsigned int unused;
1315
1316 DBG(device_printf(sc->dev, "Rx interrupt #%d\n", pq->kring_id));
1317 #ifdef PTNETMAP_STATS
1318 pq->stats.intrs ++;
1319 #endif /* PTNETMAP_STATS */
1320
1321 if (netmap_rx_irq(sc->ifp, pq->kring_id, &unused) != NM_IRQ_PASS) {
1322 return;
1323 }
1324
1325 /* Like vtnet, if_igb and if_em drivers when using MSI-X interrupts,
1326 * receive-side processing is executed directly in the interrupt
1327 * service routine. Alternatively, we may schedule the taskqueue. */
1328 ptnet_rx_eof(pq, PTNET_RX_BUDGET, true);
1329 }
1330
1331 /* The following offloadings-related functions are taken from the vtnet
1332 * driver, but the same functionality is required for the ptnet driver.
1333 * As a temporary solution, I copied this code from vtnet and I started
1334 * to generalize it (taking away driver-specific statistic accounting),
1335 * making as little modifications as possible.
1336 * In the future we need to share these functions between vtnet and ptnet.
1337 */
1338 static int
ptnet_tx_offload_ctx(struct mbuf * m,int * etype,int * proto,int * start)1339 ptnet_tx_offload_ctx(struct mbuf *m, int *etype, int *proto, int *start)
1340 {
1341 struct ether_vlan_header *evh;
1342 int offset;
1343
1344 evh = mtod(m, struct ether_vlan_header *);
1345 if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
1346 /* BMV: We should handle nested VLAN tags too. */
1347 *etype = ntohs(evh->evl_proto);
1348 offset = sizeof(struct ether_vlan_header);
1349 } else {
1350 *etype = ntohs(evh->evl_encap_proto);
1351 offset = sizeof(struct ether_header);
1352 }
1353
1354 switch (*etype) {
1355 #if defined(INET)
1356 case ETHERTYPE_IP: {
1357 struct ip *ip, iphdr;
1358 if (__predict_false(m->m_len < offset + sizeof(struct ip))) {
1359 m_copydata(m, offset, sizeof(struct ip),
1360 (caddr_t) &iphdr);
1361 ip = &iphdr;
1362 } else
1363 ip = (struct ip *)(m->m_data + offset);
1364 *proto = ip->ip_p;
1365 *start = offset + (ip->ip_hl << 2);
1366 break;
1367 }
1368 #endif
1369 #if defined(INET6)
1370 case ETHERTYPE_IPV6:
1371 *proto = -1;
1372 *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto);
1373 /* Assert the network stack sent us a valid packet. */
1374 KASSERT(*start > offset,
1375 ("%s: mbuf %p start %d offset %d proto %d", __func__, m,
1376 *start, offset, *proto));
1377 break;
1378 #endif
1379 default:
1380 /* Here we should increment the tx_csum_bad_ethtype counter. */
1381 return (EINVAL);
1382 }
1383
1384 return (0);
1385 }
1386
1387 static int
ptnet_tx_offload_tso(if_t ifp,struct mbuf * m,int eth_type,int offset,bool allow_ecn,struct virtio_net_hdr * hdr)1388 ptnet_tx_offload_tso(if_t ifp, struct mbuf *m, int eth_type,
1389 int offset, bool allow_ecn, struct virtio_net_hdr *hdr)
1390 {
1391 static struct timeval lastecn;
1392 static int curecn;
1393 struct tcphdr *tcp, tcphdr;
1394
1395 if (__predict_false(m->m_len < offset + sizeof(struct tcphdr))) {
1396 m_copydata(m, offset, sizeof(struct tcphdr), (caddr_t) &tcphdr);
1397 tcp = &tcphdr;
1398 } else
1399 tcp = (struct tcphdr *)(m->m_data + offset);
1400
1401 hdr->hdr_len = offset + (tcp->th_off << 2);
1402 hdr->gso_size = m->m_pkthdr.tso_segsz;
1403 hdr->gso_type = eth_type == ETHERTYPE_IP ? VIRTIO_NET_HDR_GSO_TCPV4 :
1404 VIRTIO_NET_HDR_GSO_TCPV6;
1405
1406 if (tcp->th_flags & TH_CWR) {
1407 /*
1408 * Drop if VIRTIO_NET_F_HOST_ECN was not negotiated. In FreeBSD,
1409 * ECN support is not on a per-interface basis, but globally via
1410 * the net.inet.tcp.ecn.enable sysctl knob. The default is off.
1411 */
1412 if (!allow_ecn) {
1413 if (ppsratecheck(&lastecn, &curecn, 1))
1414 if_printf(ifp,
1415 "TSO with ECN not negotiated with host\n");
1416 return (ENOTSUP);
1417 }
1418 hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN;
1419 }
1420
1421 /* Here we should increment tx_tso counter. */
1422
1423 return (0);
1424 }
1425
1426 static struct mbuf *
ptnet_tx_offload(if_t ifp,struct mbuf * m,bool allow_ecn,struct virtio_net_hdr * hdr)1427 ptnet_tx_offload(if_t ifp, struct mbuf *m, bool allow_ecn,
1428 struct virtio_net_hdr *hdr)
1429 {
1430 int flags, etype, csum_start, proto, error;
1431
1432 flags = m->m_pkthdr.csum_flags;
1433
1434 error = ptnet_tx_offload_ctx(m, &etype, &proto, &csum_start);
1435 if (error)
1436 goto drop;
1437
1438 if ((etype == ETHERTYPE_IP && flags & PTNET_CSUM_OFFLOAD) ||
1439 (etype == ETHERTYPE_IPV6 && flags & PTNET_CSUM_OFFLOAD_IPV6)) {
1440 /*
1441 * We could compare the IP protocol vs the CSUM_ flag too,
1442 * but that really should not be necessary.
1443 */
1444 hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM;
1445 hdr->csum_start = csum_start;
1446 hdr->csum_offset = m->m_pkthdr.csum_data;
1447 /* Here we should increment the tx_csum counter. */
1448 }
1449
1450 if (flags & CSUM_TSO) {
1451 if (__predict_false(proto != IPPROTO_TCP)) {
1452 /* Likely failed to correctly parse the mbuf.
1453 * Here we should increment the tx_tso_not_tcp
1454 * counter. */
1455 goto drop;
1456 }
1457
1458 KASSERT(hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM,
1459 ("%s: mbuf %p TSO without checksum offload %#x",
1460 __func__, m, flags));
1461
1462 error = ptnet_tx_offload_tso(ifp, m, etype, csum_start,
1463 allow_ecn, hdr);
1464 if (error)
1465 goto drop;
1466 }
1467
1468 return (m);
1469
1470 drop:
1471 m_freem(m);
1472 return (NULL);
1473 }
1474
1475 static void
ptnet_vlan_tag_remove(struct mbuf * m)1476 ptnet_vlan_tag_remove(struct mbuf *m)
1477 {
1478 struct ether_vlan_header *evh;
1479
1480 evh = mtod(m, struct ether_vlan_header *);
1481 m->m_pkthdr.ether_vtag = ntohs(evh->evl_tag);
1482 m->m_flags |= M_VLANTAG;
1483
1484 /* Strip the 802.1Q header. */
1485 bcopy((char *) evh, (char *) evh + ETHER_VLAN_ENCAP_LEN,
1486 ETHER_HDR_LEN - ETHER_TYPE_LEN);
1487 m_adj(m, ETHER_VLAN_ENCAP_LEN);
1488 }
1489
1490 /*
1491 * Use the checksum offset in the VirtIO header to set the
1492 * correct CSUM_* flags.
1493 */
1494 static int
ptnet_rx_csum_by_offset(struct mbuf * m,uint16_t eth_type,int ip_start,struct virtio_net_hdr * hdr)1495 ptnet_rx_csum_by_offset(struct mbuf *m, uint16_t eth_type, int ip_start,
1496 struct virtio_net_hdr *hdr)
1497 {
1498 #if defined(INET) || defined(INET6)
1499 int offset = hdr->csum_start + hdr->csum_offset;
1500 #endif
1501
1502 /* Only do a basic sanity check on the offset. */
1503 switch (eth_type) {
1504 #if defined(INET)
1505 case ETHERTYPE_IP:
1506 if (__predict_false(offset < ip_start + sizeof(struct ip)))
1507 return (1);
1508 break;
1509 #endif
1510 #if defined(INET6)
1511 case ETHERTYPE_IPV6:
1512 if (__predict_false(offset < ip_start + sizeof(struct ip6_hdr)))
1513 return (1);
1514 break;
1515 #endif
1516 default:
1517 /* Here we should increment the rx_csum_bad_ethtype counter. */
1518 return (1);
1519 }
1520
1521 /*
1522 * Use the offset to determine the appropriate CSUM_* flags. This is
1523 * a bit dirty, but we can get by with it since the checksum offsets
1524 * happen to be different. We assume the host host does not do IPv4
1525 * header checksum offloading.
1526 */
1527 switch (hdr->csum_offset) {
1528 case offsetof(struct udphdr, uh_sum):
1529 case offsetof(struct tcphdr, th_sum):
1530 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1531 m->m_pkthdr.csum_data = 0xFFFF;
1532 break;
1533 default:
1534 /* Here we should increment the rx_csum_bad_offset counter. */
1535 return (1);
1536 }
1537
1538 return (0);
1539 }
1540
1541 static int
ptnet_rx_csum_by_parse(struct mbuf * m,uint16_t eth_type,int ip_start,struct virtio_net_hdr * hdr)1542 ptnet_rx_csum_by_parse(struct mbuf *m, uint16_t eth_type, int ip_start,
1543 struct virtio_net_hdr *hdr)
1544 {
1545 int offset, proto;
1546
1547 switch (eth_type) {
1548 #if defined(INET)
1549 case ETHERTYPE_IP: {
1550 struct ip *ip;
1551 if (__predict_false(m->m_len < ip_start + sizeof(struct ip)))
1552 return (1);
1553 ip = (struct ip *)(m->m_data + ip_start);
1554 proto = ip->ip_p;
1555 offset = ip_start + (ip->ip_hl << 2);
1556 break;
1557 }
1558 #endif
1559 #if defined(INET6)
1560 case ETHERTYPE_IPV6:
1561 if (__predict_false(m->m_len < ip_start +
1562 sizeof(struct ip6_hdr)))
1563 return (1);
1564 offset = ip6_lasthdr(m, ip_start, IPPROTO_IPV6, &proto);
1565 if (__predict_false(offset < 0))
1566 return (1);
1567 break;
1568 #endif
1569 default:
1570 /* Here we should increment the rx_csum_bad_ethtype counter. */
1571 return (1);
1572 }
1573
1574 switch (proto) {
1575 case IPPROTO_TCP:
1576 if (__predict_false(m->m_len < offset + sizeof(struct tcphdr)))
1577 return (1);
1578 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1579 m->m_pkthdr.csum_data = 0xFFFF;
1580 break;
1581 case IPPROTO_UDP:
1582 if (__predict_false(m->m_len < offset + sizeof(struct udphdr)))
1583 return (1);
1584 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1585 m->m_pkthdr.csum_data = 0xFFFF;
1586 break;
1587 default:
1588 /*
1589 * For the remaining protocols, FreeBSD does not support
1590 * checksum offloading, so the checksum will be recomputed.
1591 */
1592 #if 0
1593 if_printf(ifp, "cksum offload of unsupported "
1594 "protocol eth_type=%#x proto=%d csum_start=%d "
1595 "csum_offset=%d\n", __func__, eth_type, proto,
1596 hdr->csum_start, hdr->csum_offset);
1597 #endif
1598 break;
1599 }
1600
1601 return (0);
1602 }
1603
1604 /*
1605 * Set the appropriate CSUM_* flags. Unfortunately, the information
1606 * provided is not directly useful to us. The VirtIO header gives the
1607 * offset of the checksum, which is all Linux needs, but this is not
1608 * how FreeBSD does things. We are forced to peek inside the packet
1609 * a bit.
1610 *
1611 * It would be nice if VirtIO gave us the L4 protocol or if FreeBSD
1612 * could accept the offsets and let the stack figure it out.
1613 */
1614 static int
ptnet_rx_csum(struct mbuf * m,struct virtio_net_hdr * hdr)1615 ptnet_rx_csum(struct mbuf *m, struct virtio_net_hdr *hdr)
1616 {
1617 struct ether_header *eh;
1618 struct ether_vlan_header *evh;
1619 uint16_t eth_type;
1620 int offset, error;
1621
1622 eh = mtod(m, struct ether_header *);
1623 eth_type = ntohs(eh->ether_type);
1624 if (eth_type == ETHERTYPE_VLAN) {
1625 /* BMV: We should handle nested VLAN tags too. */
1626 evh = mtod(m, struct ether_vlan_header *);
1627 eth_type = ntohs(evh->evl_proto);
1628 offset = sizeof(struct ether_vlan_header);
1629 } else
1630 offset = sizeof(struct ether_header);
1631
1632 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
1633 error = ptnet_rx_csum_by_offset(m, eth_type, offset, hdr);
1634 else
1635 error = ptnet_rx_csum_by_parse(m, eth_type, offset, hdr);
1636
1637 return (error);
1638 }
1639 /* End of offloading-related functions to be shared with vtnet. */
1640
1641 static void
ptnet_ring_update(struct ptnet_queue * pq,struct netmap_kring * kring,unsigned int head,unsigned int sync_flags)1642 ptnet_ring_update(struct ptnet_queue *pq, struct netmap_kring *kring,
1643 unsigned int head, unsigned int sync_flags)
1644 {
1645 struct netmap_ring *ring = kring->ring;
1646 struct nm_csb_atok *atok = pq->atok;
1647 struct nm_csb_ktoa *ktoa = pq->ktoa;
1648
1649 /* Some packets have been pushed to the netmap ring. We have
1650 * to tell the host to process the new packets, updating cur
1651 * and head in the CSB. */
1652 ring->head = ring->cur = head;
1653
1654 /* Mimic nm_txsync_prologue/nm_rxsync_prologue. */
1655 kring->rcur = kring->rhead = head;
1656
1657 nm_sync_kloop_appl_write(atok, kring->rcur, kring->rhead);
1658
1659 /* Kick the host if needed. */
1660 if (NM_ACCESS_ONCE(ktoa->kern_need_kick)) {
1661 atok->sync_flags = sync_flags;
1662 ptnet_kick(pq);
1663 }
1664 }
1665
1666 #define PTNET_TX_NOSPACE(_h, _k, _min) \
1667 ((((_h) < (_k)->rtail) ? 0 : (_k)->nkr_num_slots) + \
1668 (_k)->rtail - (_h)) < (_min)
1669
1670 /* This function may be called by the network stack, or by
1671 * by the taskqueue thread. */
1672 static int
ptnet_drain_transmit_queue(struct ptnet_queue * pq,unsigned int budget,bool may_resched)1673 ptnet_drain_transmit_queue(struct ptnet_queue *pq, unsigned int budget,
1674 bool may_resched)
1675 {
1676 struct ptnet_softc *sc = pq->sc;
1677 bool have_vnet_hdr = sc->vnet_hdr_len;
1678 struct netmap_adapter *na = &sc->ptna->dr.up;
1679 if_t ifp = sc->ifp;
1680 unsigned int batch_count = 0;
1681 struct nm_csb_atok *atok;
1682 struct nm_csb_ktoa *ktoa;
1683 struct netmap_kring *kring;
1684 struct netmap_ring *ring;
1685 struct netmap_slot *slot;
1686 unsigned int count = 0;
1687 unsigned int minspace;
1688 unsigned int head;
1689 unsigned int lim;
1690 struct mbuf *mhead;
1691 struct mbuf *mf;
1692 int nmbuf_bytes;
1693 uint8_t *nmbuf;
1694
1695 if (!PTNET_Q_TRYLOCK(pq)) {
1696 /* We failed to acquire the lock, schedule the taskqueue. */
1697 nm_prlim(1, "Deferring TX work");
1698 if (may_resched) {
1699 taskqueue_enqueue(pq->taskq, &pq->task);
1700 }
1701
1702 return 0;
1703 }
1704
1705 if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) {
1706 PTNET_Q_UNLOCK(pq);
1707 nm_prlim(1, "Interface is down");
1708 return ENETDOWN;
1709 }
1710
1711 atok = pq->atok;
1712 ktoa = pq->ktoa;
1713 kring = na->tx_rings[pq->kring_id];
1714 ring = kring->ring;
1715 lim = kring->nkr_num_slots - 1;
1716 head = ring->head;
1717 minspace = sc->min_tx_space;
1718
1719 while (count < budget) {
1720 if (PTNET_TX_NOSPACE(head, kring, minspace)) {
1721 /* We ran out of slot, let's see if the host has
1722 * freed up some, by reading hwcur and hwtail from
1723 * the CSB. */
1724 ptnet_sync_tail(ktoa, kring);
1725
1726 if (PTNET_TX_NOSPACE(head, kring, minspace)) {
1727 /* Still no slots available. Reactivate the
1728 * interrupts so that we can be notified
1729 * when some free slots are made available by
1730 * the host. */
1731 atok->appl_need_kick = 1;
1732
1733 /* Double check. We need a full barrier to
1734 * prevent the store to atok->appl_need_kick
1735 * to be reordered with the load from
1736 * ktoa->hwcur and ktoa->hwtail (store-load
1737 * barrier). */
1738 nm_stld_barrier();
1739 ptnet_sync_tail(ktoa, kring);
1740 if (likely(PTNET_TX_NOSPACE(head, kring,
1741 minspace))) {
1742 break;
1743 }
1744
1745 nm_prlim(1, "Found more slots by doublecheck");
1746 /* More slots were freed before reactivating
1747 * the interrupts. */
1748 atok->appl_need_kick = 0;
1749 }
1750 }
1751
1752 mhead = drbr_peek(ifp, pq->bufring);
1753 if (!mhead) {
1754 break;
1755 }
1756
1757 /* Initialize transmission state variables. */
1758 slot = ring->slot + head;
1759 nmbuf = NMB(na, slot);
1760 nmbuf_bytes = 0;
1761
1762 /* If needed, prepare the virtio-net header at the beginning
1763 * of the first slot. */
1764 if (have_vnet_hdr) {
1765 struct virtio_net_hdr *vh =
1766 (struct virtio_net_hdr *)nmbuf;
1767
1768 /* For performance, we could replace this memset() with
1769 * two 8-bytes-wide writes. */
1770 memset(nmbuf, 0, PTNET_HDR_SIZE);
1771 if (mhead->m_pkthdr.csum_flags & PTNET_ALL_OFFLOAD) {
1772 mhead = ptnet_tx_offload(ifp, mhead, false,
1773 vh);
1774 if (unlikely(!mhead)) {
1775 /* Packet dropped because errors
1776 * occurred while preparing the vnet
1777 * header. Let's go ahead with the next
1778 * packet. */
1779 pq->stats.errors ++;
1780 drbr_advance(ifp, pq->bufring);
1781 continue;
1782 }
1783 }
1784 nm_prdis(1, "%s: [csum_flags %lX] vnet hdr: flags %x "
1785 "csum_start %u csum_ofs %u hdr_len = %u "
1786 "gso_size %u gso_type %x", __func__,
1787 mhead->m_pkthdr.csum_flags, vh->flags,
1788 vh->csum_start, vh->csum_offset, vh->hdr_len,
1789 vh->gso_size, vh->gso_type);
1790
1791 nmbuf += PTNET_HDR_SIZE;
1792 nmbuf_bytes += PTNET_HDR_SIZE;
1793 }
1794
1795 for (mf = mhead; mf; mf = mf->m_next) {
1796 uint8_t *mdata = mf->m_data;
1797 int mlen = mf->m_len;
1798
1799 for (;;) {
1800 int copy = NETMAP_BUF_SIZE(na) - nmbuf_bytes;
1801
1802 if (mlen < copy) {
1803 copy = mlen;
1804 }
1805 memcpy(nmbuf, mdata, copy);
1806
1807 mdata += copy;
1808 mlen -= copy;
1809 nmbuf += copy;
1810 nmbuf_bytes += copy;
1811
1812 if (!mlen) {
1813 break;
1814 }
1815
1816 slot->len = nmbuf_bytes;
1817 slot->flags = NS_MOREFRAG;
1818
1819 head = nm_next(head, lim);
1820 KASSERT(head != ring->tail,
1821 ("Unexpectedly run out of TX space"));
1822 slot = ring->slot + head;
1823 nmbuf = NMB(na, slot);
1824 nmbuf_bytes = 0;
1825 }
1826 }
1827
1828 /* Complete last slot and update head. */
1829 slot->len = nmbuf_bytes;
1830 slot->flags = 0;
1831 head = nm_next(head, lim);
1832
1833 /* Consume the packet just processed. */
1834 drbr_advance(ifp, pq->bufring);
1835
1836 /* Copy the packet to listeners. */
1837 ETHER_BPF_MTAP(ifp, mhead);
1838
1839 pq->stats.packets ++;
1840 pq->stats.bytes += mhead->m_pkthdr.len;
1841 if (mhead->m_flags & M_MCAST) {
1842 pq->stats.mcasts ++;
1843 }
1844
1845 m_freem(mhead);
1846
1847 count ++;
1848 if (++batch_count == PTNET_TX_BATCH) {
1849 ptnet_ring_update(pq, kring, head, NAF_FORCE_RECLAIM);
1850 batch_count = 0;
1851 }
1852 }
1853
1854 if (batch_count) {
1855 ptnet_ring_update(pq, kring, head, NAF_FORCE_RECLAIM);
1856 }
1857
1858 if (count >= budget && may_resched) {
1859 DBG(nm_prlim(1, "out of budget: resched, %d mbufs pending\n",
1860 drbr_inuse(ifp, pq->bufring)));
1861 taskqueue_enqueue(pq->taskq, &pq->task);
1862 }
1863
1864 PTNET_Q_UNLOCK(pq);
1865
1866 return count;
1867 }
1868
1869 static int
ptnet_transmit(if_t ifp,struct mbuf * m)1870 ptnet_transmit(if_t ifp, struct mbuf *m)
1871 {
1872 struct ptnet_softc *sc = if_getsoftc(ifp);
1873 struct ptnet_queue *pq;
1874 unsigned int queue_idx;
1875 int err;
1876
1877 DBG(device_printf(sc->dev, "transmit %p\n", m));
1878
1879 /* Insert 802.1Q header if needed. */
1880 if (m->m_flags & M_VLANTAG) {
1881 m = ether_vlanencap(m, m->m_pkthdr.ether_vtag);
1882 if (m == NULL) {
1883 return ENOBUFS;
1884 }
1885 m->m_flags &= ~M_VLANTAG;
1886 }
1887
1888 /* Get the flow-id if available. */
1889 queue_idx = (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) ?
1890 m->m_pkthdr.flowid : curcpu;
1891
1892 if (unlikely(queue_idx >= sc->num_tx_rings)) {
1893 queue_idx %= sc->num_tx_rings;
1894 }
1895
1896 pq = sc->queues + queue_idx;
1897
1898 err = drbr_enqueue(ifp, pq->bufring, m);
1899 if (err) {
1900 /* ENOBUFS when the bufring is full */
1901 nm_prlim(1, "%s: drbr_enqueue() failed %d\n",
1902 __func__, err);
1903 pq->stats.errors ++;
1904 return err;
1905 }
1906
1907 if (ifp->if_capenable & IFCAP_POLLING) {
1908 /* If polling is on, the transmit queues will be
1909 * drained by the poller. */
1910 return 0;
1911 }
1912
1913 err = ptnet_drain_transmit_queue(pq, PTNET_TX_BUDGET, true);
1914
1915 return (err < 0) ? err : 0;
1916 }
1917
1918 static unsigned int
ptnet_rx_discard(struct netmap_kring * kring,unsigned int head)1919 ptnet_rx_discard(struct netmap_kring *kring, unsigned int head)
1920 {
1921 struct netmap_ring *ring = kring->ring;
1922 struct netmap_slot *slot = ring->slot + head;
1923
1924 for (;;) {
1925 head = nm_next(head, kring->nkr_num_slots - 1);
1926 if (!(slot->flags & NS_MOREFRAG) || head == ring->tail) {
1927 break;
1928 }
1929 slot = ring->slot + head;
1930 }
1931
1932 return head;
1933 }
1934
1935 static inline struct mbuf *
ptnet_rx_slot(struct mbuf * mtail,uint8_t * nmbuf,unsigned int nmbuf_len)1936 ptnet_rx_slot(struct mbuf *mtail, uint8_t *nmbuf, unsigned int nmbuf_len)
1937 {
1938 uint8_t *mdata = mtod(mtail, uint8_t *) + mtail->m_len;
1939
1940 do {
1941 unsigned int copy;
1942
1943 if (mtail->m_len == MCLBYTES) {
1944 struct mbuf *mf;
1945
1946 mf = m_getcl(M_NOWAIT, MT_DATA, 0);
1947 if (unlikely(!mf)) {
1948 return NULL;
1949 }
1950
1951 mtail->m_next = mf;
1952 mtail = mf;
1953 mdata = mtod(mtail, uint8_t *);
1954 mtail->m_len = 0;
1955 }
1956
1957 copy = MCLBYTES - mtail->m_len;
1958 if (nmbuf_len < copy) {
1959 copy = nmbuf_len;
1960 }
1961
1962 memcpy(mdata, nmbuf, copy);
1963
1964 nmbuf += copy;
1965 nmbuf_len -= copy;
1966 mdata += copy;
1967 mtail->m_len += copy;
1968 } while (nmbuf_len);
1969
1970 return mtail;
1971 }
1972
1973 static int
ptnet_rx_eof(struct ptnet_queue * pq,unsigned int budget,bool may_resched)1974 ptnet_rx_eof(struct ptnet_queue *pq, unsigned int budget, bool may_resched)
1975 {
1976 struct ptnet_softc *sc = pq->sc;
1977 bool have_vnet_hdr = sc->vnet_hdr_len;
1978 struct nm_csb_atok *atok = pq->atok;
1979 struct nm_csb_ktoa *ktoa = pq->ktoa;
1980 struct netmap_adapter *na = &sc->ptna->dr.up;
1981 struct netmap_kring *kring = na->rx_rings[pq->kring_id];
1982 struct netmap_ring *ring = kring->ring;
1983 unsigned int const lim = kring->nkr_num_slots - 1;
1984 unsigned int batch_count = 0;
1985 if_t ifp = sc->ifp;
1986 unsigned int count = 0;
1987 uint32_t head;
1988
1989 PTNET_Q_LOCK(pq);
1990
1991 if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) {
1992 goto unlock;
1993 }
1994
1995 kring->nr_kflags &= ~NKR_PENDINTR;
1996
1997 head = ring->head;
1998 while (count < budget) {
1999 uint32_t prev_head = head;
2000 struct mbuf *mhead, *mtail;
2001 struct virtio_net_hdr *vh;
2002 struct netmap_slot *slot;
2003 unsigned int nmbuf_len;
2004 uint8_t *nmbuf;
2005 int deliver = 1; /* the mbuf to the network stack. */
2006 host_sync:
2007 if (head == ring->tail) {
2008 /* We ran out of slot, let's see if the host has
2009 * added some, by reading hwcur and hwtail from
2010 * the CSB. */
2011 ptnet_sync_tail(ktoa, kring);
2012
2013 if (head == ring->tail) {
2014 /* Still no slots available. Reactivate
2015 * interrupts as they were disabled by the
2016 * host thread right before issuing the
2017 * last interrupt. */
2018 atok->appl_need_kick = 1;
2019
2020 /* Double check for more completed RX slots.
2021 * We need a full barrier to prevent the store
2022 * to atok->appl_need_kick to be reordered with
2023 * the load from ktoa->hwcur and ktoa->hwtail
2024 * (store-load barrier). */
2025 nm_stld_barrier();
2026 ptnet_sync_tail(ktoa, kring);
2027 if (likely(head == ring->tail)) {
2028 break;
2029 }
2030 atok->appl_need_kick = 0;
2031 }
2032 }
2033
2034 /* Initialize ring state variables, possibly grabbing the
2035 * virtio-net header. */
2036 slot = ring->slot + head;
2037 nmbuf = NMB(na, slot);
2038 nmbuf_len = slot->len;
2039
2040 vh = (struct virtio_net_hdr *)nmbuf;
2041 if (have_vnet_hdr) {
2042 if (unlikely(nmbuf_len < PTNET_HDR_SIZE)) {
2043 /* There is no good reason why host should
2044 * put the header in multiple netmap slots.
2045 * If this is the case, discard. */
2046 nm_prlim(1, "Fragmented vnet-hdr: dropping");
2047 head = ptnet_rx_discard(kring, head);
2048 pq->stats.iqdrops ++;
2049 deliver = 0;
2050 goto skip;
2051 }
2052 nm_prdis(1, "%s: vnet hdr: flags %x csum_start %u "
2053 "csum_ofs %u hdr_len = %u gso_size %u "
2054 "gso_type %x", __func__, vh->flags,
2055 vh->csum_start, vh->csum_offset, vh->hdr_len,
2056 vh->gso_size, vh->gso_type);
2057 nmbuf += PTNET_HDR_SIZE;
2058 nmbuf_len -= PTNET_HDR_SIZE;
2059 }
2060
2061 /* Allocate the head of a new mbuf chain.
2062 * We use m_getcl() to allocate an mbuf with standard cluster
2063 * size (MCLBYTES). In the future we could use m_getjcl()
2064 * to choose different sizes. */
2065 mhead = mtail = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
2066 if (unlikely(mhead == NULL)) {
2067 device_printf(sc->dev, "%s: failed to allocate mbuf "
2068 "head\n", __func__);
2069 pq->stats.errors ++;
2070 break;
2071 }
2072
2073 /* Initialize the mbuf state variables. */
2074 mhead->m_pkthdr.len = nmbuf_len;
2075 mtail->m_len = 0;
2076
2077 /* Scan all the netmap slots containing the current packet. */
2078 for (;;) {
2079 DBG(device_printf(sc->dev, "%s: h %u t %u rcv frag "
2080 "len %u, flags %u\n", __func__,
2081 head, ring->tail, slot->len,
2082 slot->flags));
2083
2084 mtail = ptnet_rx_slot(mtail, nmbuf, nmbuf_len);
2085 if (unlikely(!mtail)) {
2086 /* Ouch. We ran out of memory while processing
2087 * a packet. We have to restore the previous
2088 * head position, free the mbuf chain, and
2089 * schedule the taskqueue to give the packet
2090 * another chance. */
2091 device_printf(sc->dev, "%s: failed to allocate"
2092 " mbuf frag, reset head %u --> %u\n",
2093 __func__, head, prev_head);
2094 head = prev_head;
2095 m_freem(mhead);
2096 pq->stats.errors ++;
2097 if (may_resched) {
2098 taskqueue_enqueue(pq->taskq,
2099 &pq->task);
2100 }
2101 goto escape;
2102 }
2103
2104 /* We have to increment head irrespective of the
2105 * NS_MOREFRAG being set or not. */
2106 head = nm_next(head, lim);
2107
2108 if (!(slot->flags & NS_MOREFRAG)) {
2109 break;
2110 }
2111
2112 if (unlikely(head == ring->tail)) {
2113 /* The very last slot prepared by the host has
2114 * the NS_MOREFRAG set. Drop it and continue
2115 * the outer cycle (to do the double-check). */
2116 nm_prlim(1, "Incomplete packet: dropping");
2117 m_freem(mhead);
2118 pq->stats.iqdrops ++;
2119 goto host_sync;
2120 }
2121
2122 slot = ring->slot + head;
2123 nmbuf = NMB(na, slot);
2124 nmbuf_len = slot->len;
2125 mhead->m_pkthdr.len += nmbuf_len;
2126 }
2127
2128 mhead->m_pkthdr.rcvif = ifp;
2129 mhead->m_pkthdr.csum_flags = 0;
2130
2131 /* Store the queue idx in the packet header. */
2132 mhead->m_pkthdr.flowid = pq->kring_id;
2133 M_HASHTYPE_SET(mhead, M_HASHTYPE_OPAQUE);
2134
2135 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
2136 struct ether_header *eh;
2137
2138 eh = mtod(mhead, struct ether_header *);
2139 if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
2140 ptnet_vlan_tag_remove(mhead);
2141 /*
2142 * With the 802.1Q header removed, update the
2143 * checksum starting location accordingly.
2144 */
2145 if (vh->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
2146 vh->csum_start -= ETHER_VLAN_ENCAP_LEN;
2147 }
2148 }
2149
2150 if (have_vnet_hdr && (vh->flags & (VIRTIO_NET_HDR_F_NEEDS_CSUM
2151 | VIRTIO_NET_HDR_F_DATA_VALID))) {
2152 if (unlikely(ptnet_rx_csum(mhead, vh))) {
2153 m_freem(mhead);
2154 nm_prlim(1, "Csum offload error: dropping");
2155 pq->stats.iqdrops ++;
2156 deliver = 0;
2157 }
2158 }
2159
2160 skip:
2161 count ++;
2162 if (++batch_count >= PTNET_RX_BATCH) {
2163 /* Some packets have been (or will be) pushed to the network
2164 * stack. We need to update the CSB to tell the host about
2165 * the new ring->cur and ring->head (RX buffer refill). */
2166 ptnet_ring_update(pq, kring, head, NAF_FORCE_READ);
2167 batch_count = 0;
2168 }
2169
2170 if (likely(deliver)) {
2171 pq->stats.packets ++;
2172 pq->stats.bytes += mhead->m_pkthdr.len;
2173
2174 PTNET_Q_UNLOCK(pq);
2175 (*ifp->if_input)(ifp, mhead);
2176 PTNET_Q_LOCK(pq);
2177 /* The ring->head index (and related indices) are
2178 * updated under pq lock by ptnet_ring_update().
2179 * Since we dropped the lock to call if_input(), we
2180 * must reload ring->head and restart processing the
2181 * ring from there. */
2182 head = ring->head;
2183
2184 if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) {
2185 /* The interface has gone down while we didn't
2186 * have the lock. Stop any processing and exit. */
2187 goto unlock;
2188 }
2189 }
2190 }
2191 escape:
2192 if (batch_count) {
2193 ptnet_ring_update(pq, kring, head, NAF_FORCE_READ);
2194
2195 }
2196
2197 if (count >= budget && may_resched) {
2198 /* If we ran out of budget or the double-check found new
2199 * slots to process, schedule the taskqueue. */
2200 DBG(nm_prlim(1, "out of budget: resched h %u t %u\n",
2201 head, ring->tail));
2202 taskqueue_enqueue(pq->taskq, &pq->task);
2203 }
2204 unlock:
2205 PTNET_Q_UNLOCK(pq);
2206
2207 return count;
2208 }
2209
2210 static void
ptnet_rx_task(void * context,int pending)2211 ptnet_rx_task(void *context, int pending)
2212 {
2213 struct ptnet_queue *pq = context;
2214
2215 DBG(nm_prlim(1, "%s: pq #%u\n", __func__, pq->kring_id));
2216 ptnet_rx_eof(pq, PTNET_RX_BUDGET, true);
2217 }
2218
2219 static void
ptnet_tx_task(void * context,int pending)2220 ptnet_tx_task(void *context, int pending)
2221 {
2222 struct ptnet_queue *pq = context;
2223
2224 DBG(nm_prlim(1, "%s: pq #%u\n", __func__, pq->kring_id));
2225 ptnet_drain_transmit_queue(pq, PTNET_TX_BUDGET, true);
2226 }
2227
2228 #ifdef DEVICE_POLLING
2229 /* We don't need to handle differently POLL_AND_CHECK_STATUS and
2230 * POLL_ONLY, since we don't have an Interrupt Status Register. */
2231 static int
ptnet_poll(if_t ifp,enum poll_cmd cmd,int budget)2232 ptnet_poll(if_t ifp, enum poll_cmd cmd, int budget)
2233 {
2234 struct ptnet_softc *sc = if_getsoftc(ifp);
2235 unsigned int queue_budget;
2236 unsigned int count = 0;
2237 bool borrow = false;
2238 int i;
2239
2240 KASSERT(sc->num_rings > 0, ("Found no queues in while polling ptnet"));
2241 queue_budget = MAX(budget / sc->num_rings, 1);
2242 nm_prlim(1, "Per-queue budget is %d", queue_budget);
2243
2244 while (budget) {
2245 unsigned int rcnt = 0;
2246
2247 for (i = 0; i < sc->num_rings; i++) {
2248 struct ptnet_queue *pq = sc->queues + i;
2249
2250 if (borrow) {
2251 queue_budget = MIN(queue_budget, budget);
2252 if (queue_budget == 0) {
2253 break;
2254 }
2255 }
2256
2257 if (i < sc->num_tx_rings) {
2258 rcnt += ptnet_drain_transmit_queue(pq,
2259 queue_budget, false);
2260 } else {
2261 rcnt += ptnet_rx_eof(pq, queue_budget,
2262 false);
2263 }
2264 }
2265
2266 if (!rcnt) {
2267 /* A scan of the queues gave no result, we can
2268 * stop here. */
2269 break;
2270 }
2271
2272 if (rcnt > budget) {
2273 /* This may happen when initial budget < sc->num_rings,
2274 * since one packet budget is given to each queue
2275 * anyway. Just pretend we didn't eat "so much". */
2276 rcnt = budget;
2277 }
2278 count += rcnt;
2279 budget -= rcnt;
2280 borrow = true;
2281 }
2282
2283
2284 return count;
2285 }
2286 #endif /* DEVICE_POLLING */
2287 #endif /* WITH_PTNETMAP */
2288