1 /*
2 * Copyright (C) 2016-2018 Vincenzo Maffione
3 * Copyright (C) 2015 Stefano Garzarella
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 /*
29 * common headers
30 */
31 #if defined(__FreeBSD__)
32 #include <sys/cdefs.h>
33 #include <sys/param.h>
34 #include <sys/kernel.h>
35 #include <sys/types.h>
36 #include <sys/selinfo.h>
37 #include <sys/socket.h>
38 #include <net/if.h>
39 #include <net/if_var.h>
40 #include <machine/bus.h>
41
42 #define usleep_range(_1, _2) \
43 pause_sbt("sync-kloop-sleep", SBT_1US * _1, SBT_1US * 1, C_ABSOLUTE)
44
45 #elif defined(linux)
46 #include <bsd_glue.h>
47 #include <linux/file.h>
48 #include <linux/eventfd.h>
49 #endif
50
51 #include <net/netmap.h>
52 #include <dev/netmap/netmap_kern.h>
53 #include <net/netmap_virt.h>
54 #include <dev/netmap/netmap_mem2.h>
55
56 /* Support for eventfd-based notifications. */
57 #if defined(linux)
58 #define SYNC_KLOOP_POLL
59 #endif
60
61 /* Write kring pointers (hwcur, hwtail) to the CSB.
62 * This routine is coupled with ptnetmap_guest_read_kring_csb(). */
63 static inline void
sync_kloop_kernel_write(struct nm_csb_ktoa __user * ptr,uint32_t hwcur,uint32_t hwtail)64 sync_kloop_kernel_write(struct nm_csb_ktoa __user *ptr, uint32_t hwcur,
65 uint32_t hwtail)
66 {
67 /* Issue a first store-store barrier to make sure writes to the
68 * netmap ring do not overcome updates on ktoa->hwcur and ktoa->hwtail. */
69 nm_stst_barrier();
70
71 /*
72 * The same scheme used in nm_sync_kloop_appl_write() applies here.
73 * We allow the application to read a value of hwcur more recent than the value
74 * of hwtail, since this would anyway result in a consistent view of the
75 * ring state (and hwcur can never wraparound hwtail, since hwcur must be
76 * behind head).
77 *
78 * The following memory barrier scheme is used to make this happen:
79 *
80 * Application Kernel
81 *
82 * STORE(hwcur) LOAD(hwtail)
83 * wmb() <-------------> rmb()
84 * STORE(hwtail) LOAD(hwcur)
85 */
86 CSB_WRITE(ptr, hwcur, hwcur);
87 nm_stst_barrier();
88 CSB_WRITE(ptr, hwtail, hwtail);
89 }
90
91 /* Read kring pointers (head, cur, sync_flags) from the CSB.
92 * This routine is coupled with ptnetmap_guest_write_kring_csb(). */
93 static inline void
sync_kloop_kernel_read(struct nm_csb_atok __user * ptr,struct netmap_ring * shadow_ring,uint32_t num_slots)94 sync_kloop_kernel_read(struct nm_csb_atok __user *ptr,
95 struct netmap_ring *shadow_ring,
96 uint32_t num_slots)
97 {
98 /*
99 * We place a memory barrier to make sure that the update of head never
100 * overtakes the update of cur.
101 * (see explanation in sync_kloop_kernel_write).
102 */
103 CSB_READ(ptr, head, shadow_ring->head);
104 nm_ldld_barrier();
105 CSB_READ(ptr, cur, shadow_ring->cur);
106 CSB_READ(ptr, sync_flags, shadow_ring->flags);
107
108 /* Make sure that loads from atok->head and atok->cur are not delayed
109 * after the loads from the netmap ring. */
110 nm_ldld_barrier();
111 }
112
113 /* Enable or disable application --> kernel kicks. */
114 static inline void
csb_ktoa_kick_enable(struct nm_csb_ktoa __user * csb_ktoa,uint32_t val)115 csb_ktoa_kick_enable(struct nm_csb_ktoa __user *csb_ktoa, uint32_t val)
116 {
117 CSB_WRITE(csb_ktoa, kern_need_kick, val);
118 }
119
120 #ifdef SYNC_KLOOP_POLL
121 /* Are application interrupt enabled or disabled? */
122 static inline uint32_t
csb_atok_intr_enabled(struct nm_csb_atok __user * csb_atok)123 csb_atok_intr_enabled(struct nm_csb_atok __user *csb_atok)
124 {
125 uint32_t v;
126
127 CSB_READ(csb_atok, appl_need_kick, v);
128
129 return v;
130 }
131 #endif /* SYNC_KLOOP_POLL */
132
133 static inline void
sync_kloop_kring_dump(const char * title,const struct netmap_kring * kring)134 sync_kloop_kring_dump(const char *title, const struct netmap_kring *kring)
135 {
136 nm_prinf("%s, kring %s, hwcur %d, rhead %d, "
137 "rcur %d, rtail %d, hwtail %d",
138 title, kring->name, kring->nr_hwcur, kring->rhead,
139 kring->rcur, kring->rtail, kring->nr_hwtail);
140 }
141
142 /* Arguments for netmap_sync_kloop_tx_ring() and
143 * netmap_sync_kloop_rx_ring().
144 */
145 struct sync_kloop_ring_args {
146 struct netmap_kring *kring;
147 struct nm_csb_atok *csb_atok;
148 struct nm_csb_ktoa *csb_ktoa;
149 #ifdef SYNC_KLOOP_POLL
150 struct eventfd_ctx *irq_ctx;
151 #endif /* SYNC_KLOOP_POLL */
152 /* Are we busy waiting rather than using a schedule() loop ? */
153 bool busy_wait;
154 /* Are we processing in the context of VM exit ? */
155 bool direct;
156 };
157
158 static void
netmap_sync_kloop_tx_ring(const struct sync_kloop_ring_args * a)159 netmap_sync_kloop_tx_ring(const struct sync_kloop_ring_args *a)
160 {
161 struct netmap_kring *kring = a->kring;
162 struct nm_csb_atok *csb_atok = a->csb_atok;
163 struct nm_csb_ktoa *csb_ktoa = a->csb_ktoa;
164 struct netmap_ring shadow_ring; /* shadow copy of the netmap_ring */
165 bool more_txspace = false;
166 uint32_t num_slots;
167 int batch;
168
169 if (unlikely(nm_kr_tryget(kring, 1, NULL))) {
170 return;
171 }
172
173 num_slots = kring->nkr_num_slots;
174
175 /* Disable application --> kernel notifications. */
176 if (!a->direct) {
177 csb_ktoa_kick_enable(csb_ktoa, 0);
178 }
179 /* Copy the application kring pointers from the CSB */
180 sync_kloop_kernel_read(csb_atok, &shadow_ring, num_slots);
181
182 for (;;) {
183 batch = shadow_ring.head - kring->nr_hwcur;
184 if (batch < 0)
185 batch += num_slots;
186
187 #ifdef PTN_TX_BATCH_LIM
188 if (batch > PTN_TX_BATCH_LIM(num_slots)) {
189 /* If application moves ahead too fast, let's cut the move so
190 * that we don't exceed our batch limit. */
191 uint32_t head_lim = kring->nr_hwcur + PTN_TX_BATCH_LIM(num_slots);
192
193 if (head_lim >= num_slots)
194 head_lim -= num_slots;
195 nm_prdis(1, "batch: %d head: %d head_lim: %d", batch, shadow_ring.head,
196 head_lim);
197 shadow_ring.head = head_lim;
198 batch = PTN_TX_BATCH_LIM(num_slots);
199 }
200 #endif /* PTN_TX_BATCH_LIM */
201
202 if (nm_kr_txspace(kring) <= (num_slots >> 1)) {
203 shadow_ring.flags |= NAF_FORCE_RECLAIM;
204 }
205
206 /* Netmap prologue */
207 shadow_ring.tail = kring->rtail;
208 if (unlikely(nm_txsync_prologue(kring, &shadow_ring) >= num_slots)) {
209 /* Reinit ring and enable notifications. */
210 netmap_ring_reinit(kring);
211 if (!a->busy_wait) {
212 csb_ktoa_kick_enable(csb_ktoa, 1);
213 }
214 break;
215 }
216
217 if (unlikely(netmap_debug & NM_DEBUG_TXSYNC)) {
218 sync_kloop_kring_dump("pre txsync", kring);
219 }
220
221 if (unlikely(kring->nm_sync(kring, shadow_ring.flags))) {
222 if (!a->busy_wait) {
223 /* Re-enable notifications. */
224 csb_ktoa_kick_enable(csb_ktoa, 1);
225 }
226 nm_prerr("txsync() failed");
227 break;
228 }
229
230 /*
231 * Finalize
232 * Copy kernel hwcur and hwtail into the CSB for the application sync(), and
233 * do the nm_sync_finalize.
234 */
235 sync_kloop_kernel_write(csb_ktoa, kring->nr_hwcur,
236 kring->nr_hwtail);
237 if (kring->rtail != kring->nr_hwtail) {
238 /* Some more room available in the parent adapter. */
239 kring->rtail = kring->nr_hwtail;
240 more_txspace = true;
241 }
242
243 if (unlikely(netmap_debug & NM_DEBUG_TXSYNC)) {
244 sync_kloop_kring_dump("post txsync", kring);
245 }
246
247 /* Interrupt the application if needed. */
248 #ifdef SYNC_KLOOP_POLL
249 if (a->irq_ctx && more_txspace && csb_atok_intr_enabled(csb_atok)) {
250 /* We could disable kernel --> application kicks here,
251 * to avoid spurious interrupts. */
252 eventfd_signal(a->irq_ctx, 1);
253 more_txspace = false;
254 }
255 #endif /* SYNC_KLOOP_POLL */
256
257 /* Read CSB to see if there is more work to do. */
258 sync_kloop_kernel_read(csb_atok, &shadow_ring, num_slots);
259 if (shadow_ring.head == kring->rhead) {
260 if (a->busy_wait) {
261 break;
262 }
263 /*
264 * No more packets to transmit. We enable notifications and
265 * go to sleep, waiting for a kick from the application when new
266 * new slots are ready for transmission.
267 */
268 /* Re-enable notifications. */
269 csb_ktoa_kick_enable(csb_ktoa, 1);
270 /* Double check, with store-load memory barrier. */
271 nm_stld_barrier();
272 sync_kloop_kernel_read(csb_atok, &shadow_ring, num_slots);
273 if (shadow_ring.head != kring->rhead) {
274 /* We won the race condition, there are more packets to
275 * transmit. Disable notifications and do another cycle */
276 csb_ktoa_kick_enable(csb_ktoa, 0);
277 continue;
278 }
279 break;
280 }
281
282 if (nm_kr_txempty(kring)) {
283 /* No more available TX slots. We stop waiting for a notification
284 * from the backend (netmap_tx_irq). */
285 nm_prdis(1, "TX ring");
286 break;
287 }
288 }
289
290 nm_kr_put(kring);
291
292 #ifdef SYNC_KLOOP_POLL
293 if (a->irq_ctx && more_txspace && csb_atok_intr_enabled(csb_atok)) {
294 eventfd_signal(a->irq_ctx, 1);
295 }
296 #endif /* SYNC_KLOOP_POLL */
297 }
298
299 /* RX cycle without receive any packets */
300 #define SYNC_LOOP_RX_DRY_CYCLES_MAX 2
301
302 static inline int
sync_kloop_norxslots(struct netmap_kring * kring,uint32_t g_head)303 sync_kloop_norxslots(struct netmap_kring *kring, uint32_t g_head)
304 {
305 return (NM_ACCESS_ONCE(kring->nr_hwtail) == nm_prev(g_head,
306 kring->nkr_num_slots - 1));
307 }
308
309 static void
netmap_sync_kloop_rx_ring(const struct sync_kloop_ring_args * a)310 netmap_sync_kloop_rx_ring(const struct sync_kloop_ring_args *a)
311 {
312
313 struct netmap_kring *kring = a->kring;
314 struct nm_csb_atok *csb_atok = a->csb_atok;
315 struct nm_csb_ktoa *csb_ktoa = a->csb_ktoa;
316 struct netmap_ring shadow_ring; /* shadow copy of the netmap_ring */
317 int dry_cycles = 0;
318 bool some_recvd = false;
319 uint32_t num_slots;
320
321 if (unlikely(nm_kr_tryget(kring, 1, NULL))) {
322 return;
323 }
324
325 num_slots = kring->nkr_num_slots;
326
327 /* Get RX csb_atok and csb_ktoa pointers from the CSB. */
328 num_slots = kring->nkr_num_slots;
329
330 /* Disable notifications. */
331 if (!a->direct) {
332 csb_ktoa_kick_enable(csb_ktoa, 0);
333 }
334 /* Copy the application kring pointers from the CSB */
335 sync_kloop_kernel_read(csb_atok, &shadow_ring, num_slots);
336
337 for (;;) {
338 uint32_t hwtail;
339
340 /* Netmap prologue */
341 shadow_ring.tail = kring->rtail;
342 if (unlikely(nm_rxsync_prologue(kring, &shadow_ring) >= num_slots)) {
343 /* Reinit ring and enable notifications. */
344 netmap_ring_reinit(kring);
345 if (!a->busy_wait) {
346 csb_ktoa_kick_enable(csb_ktoa, 1);
347 }
348 break;
349 }
350
351 if (unlikely(netmap_debug & NM_DEBUG_RXSYNC)) {
352 sync_kloop_kring_dump("pre rxsync", kring);
353 }
354
355 if (unlikely(kring->nm_sync(kring, shadow_ring.flags))) {
356 if (!a->busy_wait) {
357 /* Re-enable notifications. */
358 csb_ktoa_kick_enable(csb_ktoa, 1);
359 }
360 nm_prerr("rxsync() failed");
361 break;
362 }
363
364 /*
365 * Finalize
366 * Copy kernel hwcur and hwtail into the CSB for the application sync()
367 */
368 hwtail = NM_ACCESS_ONCE(kring->nr_hwtail);
369 sync_kloop_kernel_write(csb_ktoa, kring->nr_hwcur, hwtail);
370 if (kring->rtail != hwtail) {
371 kring->rtail = hwtail;
372 some_recvd = true;
373 dry_cycles = 0;
374 } else {
375 dry_cycles++;
376 }
377
378 if (unlikely(netmap_debug & NM_DEBUG_RXSYNC)) {
379 sync_kloop_kring_dump("post rxsync", kring);
380 }
381
382 #ifdef SYNC_KLOOP_POLL
383 /* Interrupt the application if needed. */
384 if (a->irq_ctx && some_recvd && csb_atok_intr_enabled(csb_atok)) {
385 /* We could disable kernel --> application kicks here,
386 * to avoid spurious interrupts. */
387 eventfd_signal(a->irq_ctx, 1);
388 some_recvd = false;
389 }
390 #endif /* SYNC_KLOOP_POLL */
391
392 /* Read CSB to see if there is more work to do. */
393 sync_kloop_kernel_read(csb_atok, &shadow_ring, num_slots);
394 if (sync_kloop_norxslots(kring, shadow_ring.head)) {
395 if (a->busy_wait) {
396 break;
397 }
398 /*
399 * No more slots available for reception. We enable notification and
400 * go to sleep, waiting for a kick from the application when new receive
401 * slots are available.
402 */
403 /* Re-enable notifications. */
404 csb_ktoa_kick_enable(csb_ktoa, 1);
405 /* Double check, with store-load memory barrier. */
406 nm_stld_barrier();
407 sync_kloop_kernel_read(csb_atok, &shadow_ring, num_slots);
408 if (!sync_kloop_norxslots(kring, shadow_ring.head)) {
409 /* We won the race condition, more slots are available. Disable
410 * notifications and do another cycle. */
411 csb_ktoa_kick_enable(csb_ktoa, 0);
412 continue;
413 }
414 break;
415 }
416
417 hwtail = NM_ACCESS_ONCE(kring->nr_hwtail);
418 if (unlikely(hwtail == kring->rhead ||
419 dry_cycles >= SYNC_LOOP_RX_DRY_CYCLES_MAX)) {
420 /* No more packets to be read from the backend. We stop and
421 * wait for a notification from the backend (netmap_rx_irq). */
422 nm_prdis(1, "nr_hwtail: %d rhead: %d dry_cycles: %d",
423 hwtail, kring->rhead, dry_cycles);
424 break;
425 }
426 }
427
428 nm_kr_put(kring);
429
430 #ifdef SYNC_KLOOP_POLL
431 /* Interrupt the application if needed. */
432 if (a->irq_ctx && some_recvd && csb_atok_intr_enabled(csb_atok)) {
433 eventfd_signal(a->irq_ctx, 1);
434 }
435 #endif /* SYNC_KLOOP_POLL */
436 }
437
438 #ifdef SYNC_KLOOP_POLL
439 struct sync_kloop_poll_ctx;
440 struct sync_kloop_poll_entry {
441 /* Support for receiving notifications from
442 * a netmap ring or from the application. */
443 struct file *filp;
444 wait_queue_t wait;
445 wait_queue_head_t *wqh;
446
447 /* Support for sending notifications to the application. */
448 struct eventfd_ctx *irq_ctx;
449 struct file *irq_filp;
450
451 /* Arguments for the ring processing function. Useful
452 * in case of custom wake-up function. */
453 struct sync_kloop_ring_args *args;
454 struct sync_kloop_poll_ctx *parent;
455
456 };
457
458 struct sync_kloop_poll_ctx {
459 poll_table wait_table;
460 unsigned int next_entry;
461 int (*next_wake_fun)(wait_queue_t *, unsigned, int, void *);
462 unsigned int num_entries;
463 unsigned int num_tx_rings;
464 unsigned int num_rings;
465 /* First num_tx_rings entries are for the TX kicks.
466 * Then the RX kicks entries follow. The last two
467 * entries are for TX irq, and RX irq. */
468 struct sync_kloop_poll_entry entries[0];
469 };
470
471 static void
sync_kloop_poll_table_queue_proc(struct file * file,wait_queue_head_t * wqh,poll_table * pt)472 sync_kloop_poll_table_queue_proc(struct file *file, wait_queue_head_t *wqh,
473 poll_table *pt)
474 {
475 struct sync_kloop_poll_ctx *poll_ctx =
476 container_of(pt, struct sync_kloop_poll_ctx, wait_table);
477 struct sync_kloop_poll_entry *entry = poll_ctx->entries +
478 poll_ctx->next_entry;
479
480 BUG_ON(poll_ctx->next_entry >= poll_ctx->num_entries);
481 entry->wqh = wqh;
482 entry->filp = file;
483 /* Use the default wake up function. */
484 if (poll_ctx->next_wake_fun == NULL) {
485 init_waitqueue_entry(&entry->wait, current);
486 } else {
487 init_waitqueue_func_entry(&entry->wait,
488 poll_ctx->next_wake_fun);
489 }
490 add_wait_queue(wqh, &entry->wait);
491 }
492
493 static int
sync_kloop_tx_kick_wake_fun(wait_queue_t * wait,unsigned mode,int wake_flags,void * key)494 sync_kloop_tx_kick_wake_fun(wait_queue_t *wait, unsigned mode,
495 int wake_flags, void *key)
496 {
497 struct sync_kloop_poll_entry *entry =
498 container_of(wait, struct sync_kloop_poll_entry, wait);
499
500 netmap_sync_kloop_tx_ring(entry->args);
501
502 return 0;
503 }
504
505 static int
sync_kloop_tx_irq_wake_fun(wait_queue_t * wait,unsigned mode,int wake_flags,void * key)506 sync_kloop_tx_irq_wake_fun(wait_queue_t *wait, unsigned mode,
507 int wake_flags, void *key)
508 {
509 struct sync_kloop_poll_entry *entry =
510 container_of(wait, struct sync_kloop_poll_entry, wait);
511 struct sync_kloop_poll_ctx *poll_ctx = entry->parent;
512 int i;
513
514 for (i = 0; i < poll_ctx->num_tx_rings; i++) {
515 struct eventfd_ctx *irq_ctx = poll_ctx->entries[i].irq_ctx;
516
517 if (irq_ctx) {
518 eventfd_signal(irq_ctx, 1);
519 }
520 }
521
522 return 0;
523 }
524
525 static int
sync_kloop_rx_kick_wake_fun(wait_queue_t * wait,unsigned mode,int wake_flags,void * key)526 sync_kloop_rx_kick_wake_fun(wait_queue_t *wait, unsigned mode,
527 int wake_flags, void *key)
528 {
529 struct sync_kloop_poll_entry *entry =
530 container_of(wait, struct sync_kloop_poll_entry, wait);
531
532 netmap_sync_kloop_rx_ring(entry->args);
533
534 return 0;
535 }
536
537 static int
sync_kloop_rx_irq_wake_fun(wait_queue_t * wait,unsigned mode,int wake_flags,void * key)538 sync_kloop_rx_irq_wake_fun(wait_queue_t *wait, unsigned mode,
539 int wake_flags, void *key)
540 {
541 struct sync_kloop_poll_entry *entry =
542 container_of(wait, struct sync_kloop_poll_entry, wait);
543 struct sync_kloop_poll_ctx *poll_ctx = entry->parent;
544 int i;
545
546 for (i = poll_ctx->num_tx_rings; i < poll_ctx->num_rings; i++) {
547 struct eventfd_ctx *irq_ctx = poll_ctx->entries[i].irq_ctx;
548
549 if (irq_ctx) {
550 eventfd_signal(irq_ctx, 1);
551 }
552 }
553
554 return 0;
555 }
556 #endif /* SYNC_KLOOP_POLL */
557
558 int
netmap_sync_kloop(struct netmap_priv_d * priv,struct nmreq_header * hdr)559 netmap_sync_kloop(struct netmap_priv_d *priv, struct nmreq_header *hdr)
560 {
561 struct nmreq_sync_kloop_start *req =
562 (struct nmreq_sync_kloop_start *)(uintptr_t)hdr->nr_body;
563 struct nmreq_opt_sync_kloop_eventfds *eventfds_opt = NULL;
564 #ifdef SYNC_KLOOP_POLL
565 struct sync_kloop_poll_ctx *poll_ctx = NULL;
566 #endif /* SYNC_KLOOP_POLL */
567 int num_rx_rings, num_tx_rings, num_rings;
568 struct sync_kloop_ring_args *args = NULL;
569 uint32_t sleep_us = req->sleep_us;
570 struct nm_csb_atok* csb_atok_base;
571 struct nm_csb_ktoa* csb_ktoa_base;
572 struct netmap_adapter *na;
573 struct nmreq_option *opt;
574 bool na_could_sleep = false;
575 bool busy_wait = true;
576 bool direct_tx = false;
577 bool direct_rx = false;
578 int err = 0;
579 int i;
580
581 if (sleep_us > 1000000) {
582 /* We do not accept sleeping for more than a second. */
583 return EINVAL;
584 }
585
586 if (priv->np_nifp == NULL) {
587 return ENXIO;
588 }
589 mb(); /* make sure following reads are not from cache */
590
591 na = priv->np_na;
592 if (!nm_netmap_on(na)) {
593 return ENXIO;
594 }
595
596 NMG_LOCK();
597 /* Make sure the application is working in CSB mode. */
598 if (!priv->np_csb_atok_base || !priv->np_csb_ktoa_base) {
599 NMG_UNLOCK();
600 nm_prerr("sync-kloop on %s requires "
601 "NETMAP_REQ_OPT_CSB option", na->name);
602 return EINVAL;
603 }
604
605 csb_atok_base = priv->np_csb_atok_base;
606 csb_ktoa_base = priv->np_csb_ktoa_base;
607
608 /* Make sure that no kloop is currently running. */
609 if (priv->np_kloop_state & NM_SYNC_KLOOP_RUNNING) {
610 err = EBUSY;
611 }
612 priv->np_kloop_state |= NM_SYNC_KLOOP_RUNNING;
613 NMG_UNLOCK();
614 if (err) {
615 return err;
616 }
617
618 num_rx_rings = priv->np_qlast[NR_RX] - priv->np_qfirst[NR_RX];
619 num_tx_rings = priv->np_qlast[NR_TX] - priv->np_qfirst[NR_TX];
620 num_rings = num_tx_rings + num_rx_rings;
621
622 args = nm_os_malloc(num_rings * sizeof(args[0]));
623 if (!args) {
624 err = ENOMEM;
625 goto out;
626 }
627
628 /* Prepare the arguments for netmap_sync_kloop_tx_ring()
629 * and netmap_sync_kloop_rx_ring(). */
630 for (i = 0; i < num_tx_rings; i++) {
631 struct sync_kloop_ring_args *a = args + i;
632
633 a->kring = NMR(na, NR_TX)[i + priv->np_qfirst[NR_TX]];
634 a->csb_atok = csb_atok_base + i;
635 a->csb_ktoa = csb_ktoa_base + i;
636 a->busy_wait = busy_wait;
637 a->direct = direct_tx;
638 }
639 for (i = 0; i < num_rx_rings; i++) {
640 struct sync_kloop_ring_args *a = args + num_tx_rings + i;
641
642 a->kring = NMR(na, NR_RX)[i + priv->np_qfirst[NR_RX]];
643 a->csb_atok = csb_atok_base + num_tx_rings + i;
644 a->csb_ktoa = csb_ktoa_base + num_tx_rings + i;
645 a->busy_wait = busy_wait;
646 a->direct = direct_rx;
647 }
648
649 /* Validate notification options. */
650 opt = nmreq_getoption(hdr, NETMAP_REQ_OPT_SYNC_KLOOP_MODE);
651 if (opt != NULL) {
652 struct nmreq_opt_sync_kloop_mode *mode_opt =
653 (struct nmreq_opt_sync_kloop_mode *)opt;
654
655 direct_tx = !!(mode_opt->mode & NM_OPT_SYNC_KLOOP_DIRECT_TX);
656 direct_rx = !!(mode_opt->mode & NM_OPT_SYNC_KLOOP_DIRECT_RX);
657 if (mode_opt->mode & ~(NM_OPT_SYNC_KLOOP_DIRECT_TX |
658 NM_OPT_SYNC_KLOOP_DIRECT_RX)) {
659 opt->nro_status = err = EINVAL;
660 goto out;
661 }
662 opt->nro_status = 0;
663 }
664 opt = nmreq_getoption(hdr, NETMAP_REQ_OPT_SYNC_KLOOP_EVENTFDS);
665 if (opt != NULL) {
666 if (opt->nro_size != sizeof(*eventfds_opt) +
667 sizeof(eventfds_opt->eventfds[0]) * num_rings) {
668 /* Option size not consistent with the number of
669 * entries. */
670 opt->nro_status = err = EINVAL;
671 goto out;
672 }
673 #ifdef SYNC_KLOOP_POLL
674 eventfds_opt = (struct nmreq_opt_sync_kloop_eventfds *)opt;
675 opt->nro_status = 0;
676
677 /* Check if some ioeventfd entry is not defined, and force sleep
678 * synchronization in that case. */
679 busy_wait = false;
680 for (i = 0; i < num_rings; i++) {
681 if (eventfds_opt->eventfds[i].ioeventfd < 0) {
682 busy_wait = true;
683 break;
684 }
685 }
686
687 if (busy_wait && (direct_tx || direct_rx)) {
688 /* For direct processing we need all the
689 * ioeventfds to be valid. */
690 opt->nro_status = err = EINVAL;
691 goto out;
692 }
693
694 /* We need 2 poll entries for TX and RX notifications coming
695 * from the netmap adapter, plus one entries per ring for the
696 * notifications coming from the application. */
697 poll_ctx = nm_os_malloc(sizeof(*poll_ctx) +
698 (num_rings + 2) * sizeof(poll_ctx->entries[0]));
699 init_poll_funcptr(&poll_ctx->wait_table,
700 sync_kloop_poll_table_queue_proc);
701 poll_ctx->num_entries = 2 + num_rings;
702 poll_ctx->num_tx_rings = num_tx_rings;
703 poll_ctx->num_rings = num_rings;
704 poll_ctx->next_entry = 0;
705 poll_ctx->next_wake_fun = NULL;
706
707 if (direct_tx && (na->na_flags & NAF_BDG_MAYSLEEP)) {
708 /* In direct mode, VALE txsync is called from
709 * wake-up context, where it is not possible
710 * to sleep.
711 */
712 na->na_flags &= ~NAF_BDG_MAYSLEEP;
713 na_could_sleep = true;
714 }
715
716 for (i = 0; i < num_rings + 2; i++) {
717 poll_ctx->entries[i].args = args + i;
718 poll_ctx->entries[i].parent = poll_ctx;
719 }
720
721 /* Poll for notifications coming from the applications through
722 * eventfds. */
723 for (i = 0; i < num_rings; i++, poll_ctx->next_entry++) {
724 struct eventfd_ctx *irq = NULL;
725 struct file *filp = NULL;
726 unsigned long mask;
727 bool tx_ring = (i < num_tx_rings);
728
729 if (eventfds_opt->eventfds[i].irqfd >= 0) {
730 filp = eventfd_fget(
731 eventfds_opt->eventfds[i].irqfd);
732 if (IS_ERR(filp)) {
733 err = PTR_ERR(filp);
734 goto out;
735 }
736 irq = eventfd_ctx_fileget(filp);
737 if (IS_ERR(irq)) {
738 err = PTR_ERR(irq);
739 goto out;
740 }
741 }
742 poll_ctx->entries[i].irq_filp = filp;
743 poll_ctx->entries[i].irq_ctx = irq;
744 poll_ctx->entries[i].args->busy_wait = busy_wait;
745 /* Don't let netmap_sync_kloop_*x_ring() use
746 * IRQs in direct mode. */
747 poll_ctx->entries[i].args->irq_ctx =
748 ((tx_ring && direct_tx) ||
749 (!tx_ring && direct_rx)) ? NULL :
750 poll_ctx->entries[i].irq_ctx;
751 poll_ctx->entries[i].args->direct =
752 (tx_ring ? direct_tx : direct_rx);
753
754 if (!busy_wait) {
755 filp = eventfd_fget(
756 eventfds_opt->eventfds[i].ioeventfd);
757 if (IS_ERR(filp)) {
758 err = PTR_ERR(filp);
759 goto out;
760 }
761 if (tx_ring && direct_tx) {
762 /* Override the wake up function
763 * so that it can directly call
764 * netmap_sync_kloop_tx_ring().
765 */
766 poll_ctx->next_wake_fun =
767 sync_kloop_tx_kick_wake_fun;
768 } else if (!tx_ring && direct_rx) {
769 /* Same for direct RX. */
770 poll_ctx->next_wake_fun =
771 sync_kloop_rx_kick_wake_fun;
772 } else {
773 poll_ctx->next_wake_fun = NULL;
774 }
775 mask = filp->f_op->poll(filp,
776 &poll_ctx->wait_table);
777 if (mask & POLLERR) {
778 err = EINVAL;
779 goto out;
780 }
781 }
782 }
783
784 /* Poll for notifications coming from the netmap rings bound to
785 * this file descriptor. */
786 if (!busy_wait) {
787 NMG_LOCK();
788 /* In direct mode, override the wake up function so
789 * that it can forward the netmap_tx_irq() to the
790 * guest. */
791 poll_ctx->next_wake_fun = direct_tx ?
792 sync_kloop_tx_irq_wake_fun : NULL;
793 poll_wait(priv->np_filp, priv->np_si[NR_TX],
794 &poll_ctx->wait_table);
795 poll_ctx->next_entry++;
796
797 poll_ctx->next_wake_fun = direct_rx ?
798 sync_kloop_rx_irq_wake_fun : NULL;
799 poll_wait(priv->np_filp, priv->np_si[NR_RX],
800 &poll_ctx->wait_table);
801 poll_ctx->next_entry++;
802 NMG_UNLOCK();
803 }
804 #else /* SYNC_KLOOP_POLL */
805 opt->nro_status = EOPNOTSUPP;
806 goto out;
807 #endif /* SYNC_KLOOP_POLL */
808 }
809
810 nm_prinf("kloop busy_wait %u, direct_tx %u, direct_rx %u, "
811 "na_could_sleep %u", busy_wait, direct_tx, direct_rx,
812 na_could_sleep);
813
814 /* Main loop. */
815 for (;;) {
816 if (unlikely(NM_ACCESS_ONCE(priv->np_kloop_state) & NM_SYNC_KLOOP_STOPPING)) {
817 break;
818 }
819
820 #ifdef SYNC_KLOOP_POLL
821 if (!busy_wait) {
822 /* It is important to set the task state as
823 * interruptible before processing any TX/RX ring,
824 * so that if a notification on ring Y comes after
825 * we have processed ring Y, but before we call
826 * schedule(), we don't miss it. This is true because
827 * the wake up function will change the task state,
828 * and therefore the schedule_timeout() call below
829 * will observe the change).
830 */
831 set_current_state(TASK_INTERRUPTIBLE);
832 }
833 #endif /* SYNC_KLOOP_POLL */
834
835 /* Process all the TX rings bound to this file descriptor. */
836 for (i = 0; !direct_tx && i < num_tx_rings; i++) {
837 struct sync_kloop_ring_args *a = args + i;
838 netmap_sync_kloop_tx_ring(a);
839 }
840
841 /* Process all the RX rings bound to this file descriptor. */
842 for (i = 0; !direct_rx && i < num_rx_rings; i++) {
843 struct sync_kloop_ring_args *a = args + num_tx_rings + i;
844 netmap_sync_kloop_rx_ring(a);
845 }
846
847 if (busy_wait) {
848 /* Default synchronization method: sleep for a while. */
849 usleep_range(sleep_us, sleep_us);
850 }
851 #ifdef SYNC_KLOOP_POLL
852 else {
853 /* Yield to the scheduler waiting for a notification
854 * to come either from netmap or the application. */
855 schedule_timeout(msecs_to_jiffies(3000));
856 }
857 #endif /* SYNC_KLOOP_POLL */
858 }
859 out:
860 #ifdef SYNC_KLOOP_POLL
861 if (poll_ctx) {
862 /* Stop polling from netmap and the eventfds, and deallocate
863 * the poll context. */
864 if (!busy_wait) {
865 __set_current_state(TASK_RUNNING);
866 }
867 for (i = 0; i < poll_ctx->next_entry; i++) {
868 struct sync_kloop_poll_entry *entry =
869 poll_ctx->entries + i;
870
871 if (entry->wqh)
872 remove_wait_queue(entry->wqh, &entry->wait);
873 /* We did not get a reference to the eventfds, but
874 * don't do that on netmap file descriptors (since
875 * a reference was not taken. */
876 if (entry->filp && entry->filp != priv->np_filp)
877 fput(entry->filp);
878 if (entry->irq_ctx)
879 eventfd_ctx_put(entry->irq_ctx);
880 if (entry->irq_filp)
881 fput(entry->irq_filp);
882 }
883 nm_os_free(poll_ctx);
884 poll_ctx = NULL;
885 }
886 #endif /* SYNC_KLOOP_POLL */
887
888 if (args) {
889 nm_os_free(args);
890 args = NULL;
891 }
892
893 /* Reset the kloop state. */
894 NMG_LOCK();
895 priv->np_kloop_state = 0;
896 if (na_could_sleep) {
897 na->na_flags |= NAF_BDG_MAYSLEEP;
898 }
899 NMG_UNLOCK();
900
901 return err;
902 }
903
904 int
netmap_sync_kloop_stop(struct netmap_priv_d * priv)905 netmap_sync_kloop_stop(struct netmap_priv_d *priv)
906 {
907 struct netmap_adapter *na;
908 bool running = true;
909 int err = 0;
910
911 if (priv->np_nifp == NULL) {
912 return ENXIO;
913 }
914 mb(); /* make sure following reads are not from cache */
915
916 na = priv->np_na;
917 if (!nm_netmap_on(na)) {
918 return ENXIO;
919 }
920
921 /* Set the kloop stopping flag. */
922 NMG_LOCK();
923 priv->np_kloop_state |= NM_SYNC_KLOOP_STOPPING;
924 NMG_UNLOCK();
925
926 /* Send a notification to the kloop, in case it is blocked in
927 * schedule_timeout(). We can use either RX or TX, because the
928 * kloop is waiting on both. */
929 nm_os_selwakeup(priv->np_si[NR_RX]);
930
931 /* Wait for the kloop to actually terminate. */
932 while (running) {
933 usleep_range(1000, 1500);
934 NMG_LOCK();
935 running = (NM_ACCESS_ONCE(priv->np_kloop_state)
936 & NM_SYNC_KLOOP_RUNNING);
937 NMG_UNLOCK();
938 }
939
940 return err;
941 }
942
943 #ifdef WITH_PTNETMAP
944 /*
945 * Guest ptnetmap txsync()/rxsync() routines, used in ptnet device drivers.
946 * These routines are reused across the different operating systems supported
947 * by netmap.
948 */
949
950 /*
951 * Reconcile host and guest views of the transmit ring.
952 *
953 * Guest user wants to transmit packets up to the one before ring->head,
954 * and guest kernel knows tx_ring->hwcur is the first packet unsent
955 * by the host kernel.
956 *
957 * We push out as many packets as possible, and possibly
958 * reclaim buffers from previously completed transmission.
959 *
960 * Notifications from the host are enabled only if the user guest would
961 * block (no space in the ring).
962 */
963 bool
netmap_pt_guest_txsync(struct nm_csb_atok * atok,struct nm_csb_ktoa * ktoa,struct netmap_kring * kring,int flags)964 netmap_pt_guest_txsync(struct nm_csb_atok *atok, struct nm_csb_ktoa *ktoa,
965 struct netmap_kring *kring, int flags)
966 {
967 bool notify = false;
968
969 /* Disable notifications */
970 atok->appl_need_kick = 0;
971
972 /*
973 * First part: tell the host to process the new packets,
974 * updating the CSB.
975 */
976 kring->nr_hwcur = ktoa->hwcur;
977 nm_sync_kloop_appl_write(atok, kring->rcur, kring->rhead);
978
979 /* Ask for a kick from a guest to the host if needed. */
980 if (((kring->rhead != kring->nr_hwcur || nm_kr_wouldblock(kring))
981 && NM_ACCESS_ONCE(ktoa->kern_need_kick)) ||
982 (flags & NAF_FORCE_RECLAIM)) {
983 atok->sync_flags = flags;
984 notify = true;
985 }
986
987 /*
988 * Second part: reclaim buffers for completed transmissions.
989 */
990 if (nm_kr_wouldblock(kring) || (flags & NAF_FORCE_RECLAIM)) {
991 nm_sync_kloop_appl_read(ktoa, &kring->nr_hwtail,
992 &kring->nr_hwcur);
993 }
994
995 /*
996 * No more room in the ring for new transmissions. The user thread will
997 * go to sleep and we need to be notified by the host when more free
998 * space is available.
999 */
1000 if (nm_kr_wouldblock(kring) && !(kring->nr_kflags & NKR_NOINTR)) {
1001 /* Re-enable notifications. */
1002 atok->appl_need_kick = 1;
1003 /* Double check, with store-load memory barrier. */
1004 nm_stld_barrier();
1005 nm_sync_kloop_appl_read(ktoa, &kring->nr_hwtail,
1006 &kring->nr_hwcur);
1007 /* If there is new free space, disable notifications */
1008 if (unlikely(!nm_kr_wouldblock(kring))) {
1009 atok->appl_need_kick = 0;
1010 }
1011 }
1012
1013 nm_prdis(1, "%s CSB(head:%u cur:%u hwtail:%u) KRING(head:%u cur:%u tail:%u)",
1014 kring->name, atok->head, atok->cur, ktoa->hwtail,
1015 kring->rhead, kring->rcur, kring->nr_hwtail);
1016
1017 return notify;
1018 }
1019
1020 /*
1021 * Reconcile host and guest view of the receive ring.
1022 *
1023 * Update hwcur/hwtail from host (reading from CSB).
1024 *
1025 * If guest user has released buffers up to the one before ring->head, we
1026 * also give them to the host.
1027 *
1028 * Notifications from the host are enabled only if the user guest would
1029 * block (no more completed slots in the ring).
1030 */
1031 bool
netmap_pt_guest_rxsync(struct nm_csb_atok * atok,struct nm_csb_ktoa * ktoa,struct netmap_kring * kring,int flags)1032 netmap_pt_guest_rxsync(struct nm_csb_atok *atok, struct nm_csb_ktoa *ktoa,
1033 struct netmap_kring *kring, int flags)
1034 {
1035 bool notify = false;
1036
1037 /* Disable notifications */
1038 atok->appl_need_kick = 0;
1039
1040 /*
1041 * First part: import newly received packets, by updating the kring
1042 * hwtail to the hwtail known from the host (read from the CSB).
1043 * This also updates the kring hwcur.
1044 */
1045 nm_sync_kloop_appl_read(ktoa, &kring->nr_hwtail, &kring->nr_hwcur);
1046 kring->nr_kflags &= ~NKR_PENDINTR;
1047
1048 /*
1049 * Second part: tell the host about the slots that guest user has
1050 * released, by updating cur and head in the CSB.
1051 */
1052 if (kring->rhead != kring->nr_hwcur) {
1053 nm_sync_kloop_appl_write(atok, kring->rcur, kring->rhead);
1054 }
1055
1056 /*
1057 * No more completed RX slots. The user thread will go to sleep and
1058 * we need to be notified by the host when more RX slots have been
1059 * completed.
1060 */
1061 if (nm_kr_wouldblock(kring) && !(kring->nr_kflags & NKR_NOINTR)) {
1062 /* Re-enable notifications. */
1063 atok->appl_need_kick = 1;
1064 /* Double check, with store-load memory barrier. */
1065 nm_stld_barrier();
1066 nm_sync_kloop_appl_read(ktoa, &kring->nr_hwtail,
1067 &kring->nr_hwcur);
1068 /* If there are new slots, disable notifications. */
1069 if (!nm_kr_wouldblock(kring)) {
1070 atok->appl_need_kick = 0;
1071 }
1072 }
1073
1074 /* Ask for a kick from the guest to the host if needed. */
1075 if ((kring->rhead != kring->nr_hwcur || nm_kr_wouldblock(kring))
1076 && NM_ACCESS_ONCE(ktoa->kern_need_kick)) {
1077 atok->sync_flags = flags;
1078 notify = true;
1079 }
1080
1081 nm_prdis(1, "%s CSB(head:%u cur:%u hwtail:%u) KRING(head:%u cur:%u tail:%u)",
1082 kring->name, atok->head, atok->cur, ktoa->hwtail,
1083 kring->rhead, kring->rcur, kring->nr_hwtail);
1084
1085 return notify;
1086 }
1087
1088 /*
1089 * Callbacks for ptnet drivers: nm_krings_create, nm_krings_delete, nm_dtor.
1090 */
1091 int
ptnet_nm_krings_create(struct netmap_adapter * na)1092 ptnet_nm_krings_create(struct netmap_adapter *na)
1093 {
1094 struct netmap_pt_guest_adapter *ptna =
1095 (struct netmap_pt_guest_adapter *)na; /* Upcast. */
1096 struct netmap_adapter *na_nm = &ptna->hwup.up;
1097 struct netmap_adapter *na_dr = &ptna->dr.up;
1098 int ret;
1099
1100 if (ptna->backend_users) {
1101 return 0;
1102 }
1103
1104 /* Create krings on the public netmap adapter. */
1105 ret = netmap_hw_krings_create(na_nm);
1106 if (ret) {
1107 return ret;
1108 }
1109
1110 /* Copy krings into the netmap adapter private to the driver. */
1111 na_dr->tx_rings = na_nm->tx_rings;
1112 na_dr->rx_rings = na_nm->rx_rings;
1113
1114 return 0;
1115 }
1116
1117 void
ptnet_nm_krings_delete(struct netmap_adapter * na)1118 ptnet_nm_krings_delete(struct netmap_adapter *na)
1119 {
1120 struct netmap_pt_guest_adapter *ptna =
1121 (struct netmap_pt_guest_adapter *)na; /* Upcast. */
1122 struct netmap_adapter *na_nm = &ptna->hwup.up;
1123 struct netmap_adapter *na_dr = &ptna->dr.up;
1124
1125 if (ptna->backend_users) {
1126 return;
1127 }
1128
1129 na_dr->tx_rings = NULL;
1130 na_dr->rx_rings = NULL;
1131
1132 netmap_hw_krings_delete(na_nm);
1133 }
1134
1135 void
ptnet_nm_dtor(struct netmap_adapter * na)1136 ptnet_nm_dtor(struct netmap_adapter *na)
1137 {
1138 struct netmap_pt_guest_adapter *ptna =
1139 (struct netmap_pt_guest_adapter *)na;
1140
1141 netmap_mem_put(ptna->dr.up.nm_mem);
1142 memset(&ptna->dr, 0, sizeof(ptna->dr));
1143 netmap_mem_pt_guest_ifp_del(na->nm_mem, na->ifp);
1144 }
1145
1146 int
netmap_pt_guest_attach(struct netmap_adapter * arg,unsigned int nifp_offset,unsigned int memid)1147 netmap_pt_guest_attach(struct netmap_adapter *arg,
1148 unsigned int nifp_offset, unsigned int memid)
1149 {
1150 struct netmap_pt_guest_adapter *ptna;
1151 struct ifnet *ifp = arg ? arg->ifp : NULL;
1152 int error;
1153
1154 /* get allocator */
1155 arg->nm_mem = netmap_mem_pt_guest_new(ifp, nifp_offset, memid);
1156 if (arg->nm_mem == NULL)
1157 return ENOMEM;
1158 arg->na_flags |= NAF_MEM_OWNER;
1159 error = netmap_attach_ext(arg, sizeof(struct netmap_pt_guest_adapter), 1);
1160 if (error)
1161 return error;
1162
1163 /* get the netmap_pt_guest_adapter */
1164 ptna = (struct netmap_pt_guest_adapter *) NA(ifp);
1165
1166 /* Initialize a separate pass-through netmap adapter that is going to
1167 * be used by the ptnet driver only, and so never exposed to netmap
1168 * applications. We only need a subset of the available fields. */
1169 memset(&ptna->dr, 0, sizeof(ptna->dr));
1170 ptna->dr.up.ifp = ifp;
1171 ptna->dr.up.nm_mem = netmap_mem_get(ptna->hwup.up.nm_mem);
1172 ptna->dr.up.nm_config = ptna->hwup.up.nm_config;
1173
1174 ptna->backend_users = 0;
1175
1176 return 0;
1177 }
1178
1179 #endif /* WITH_PTNETMAP */
1180