1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 /*
30 * Implements the virtqueue interface as basically described
31 * in the original VirtIO paper.
32 */
33
34 #include <sys/cdefs.h>
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/kernel.h>
38 #include <sys/malloc.h>
39 #include <sys/sglist.h>
40 #include <vm/vm.h>
41 #include <vm/pmap.h>
42
43 #include <machine/cpu.h>
44 #include <machine/bus.h>
45 #include <machine/atomic.h>
46 #include <machine/resource.h>
47 #include <sys/bus.h>
48 #include <sys/rman.h>
49
50 #include <dev/virtio/virtio.h>
51 #include <dev/virtio/virtqueue.h>
52 #include <dev/virtio/virtio_ring.h>
53
54 #include "virtio_bus_if.h"
55
56 struct virtqueue {
57 device_t vq_dev;
58 uint16_t vq_queue_index;
59 uint16_t vq_nentries;
60 uint32_t vq_flags;
61 #define VIRTQUEUE_FLAG_MODERN 0x0001
62 #define VIRTQUEUE_FLAG_INDIRECT 0x0002
63 #define VIRTQUEUE_FLAG_EVENT_IDX 0x0004
64
65 int vq_max_indirect_size;
66 bus_size_t vq_notify_offset;
67 virtqueue_intr_t *vq_intrhand;
68 void *vq_intrhand_arg;
69
70 struct vring vq_ring;
71 uint16_t vq_free_cnt;
72 uint16_t vq_queued_cnt;
73 /*
74 * Head of the free chain in the descriptor table. If
75 * there are no free descriptors, this will be set to
76 * VQ_RING_DESC_CHAIN_END.
77 */
78 uint16_t vq_desc_head_idx;
79 /*
80 * Last consumed descriptor in the used table,
81 * trails vq_ring.used->idx.
82 */
83 uint16_t vq_used_cons_idx;
84
85 void *vq_ring_mem;
86 int vq_indirect_mem_size;
87 int vq_alignment;
88 int vq_ring_size;
89 char vq_name[VIRTQUEUE_MAX_NAME_SZ];
90
91 struct vq_desc_extra {
92 void *cookie;
93 struct vring_desc *indirect;
94 vm_paddr_t indirect_paddr;
95 uint16_t ndescs;
96 } vq_descx[0];
97 };
98
99 /*
100 * The maximum virtqueue size is 2^15. Use that value as the end of
101 * descriptor chain terminator since it will never be a valid index
102 * in the descriptor table. This is used to verify we are correctly
103 * handling vq_free_cnt.
104 */
105 #define VQ_RING_DESC_CHAIN_END 32768
106
107 #define VQASSERT(_vq, _exp, _msg, ...) \
108 KASSERT((_exp),("%s: %s - "_msg, __func__, (_vq)->vq_name, \
109 ##__VA_ARGS__))
110
111 #define VQ_RING_ASSERT_VALID_IDX(_vq, _idx) \
112 VQASSERT((_vq), (_idx) < (_vq)->vq_nentries, \
113 "invalid ring index: %d, max: %d", (_idx), \
114 (_vq)->vq_nentries)
115
116 #define VQ_RING_ASSERT_CHAIN_TERM(_vq) \
117 VQASSERT((_vq), (_vq)->vq_desc_head_idx == \
118 VQ_RING_DESC_CHAIN_END, "full ring terminated " \
119 "incorrectly: head idx: %d", (_vq)->vq_desc_head_idx)
120
121 static int virtqueue_init_indirect(struct virtqueue *vq, int);
122 static void virtqueue_free_indirect(struct virtqueue *vq);
123 static void virtqueue_init_indirect_list(struct virtqueue *,
124 struct vring_desc *);
125
126 static void vq_ring_init(struct virtqueue *);
127 static void vq_ring_update_avail(struct virtqueue *, uint16_t);
128 static uint16_t vq_ring_enqueue_segments(struct virtqueue *,
129 struct vring_desc *, uint16_t, struct sglist *, int, int);
130 static int vq_ring_use_indirect(struct virtqueue *, int);
131 static void vq_ring_enqueue_indirect(struct virtqueue *, void *,
132 struct sglist *, int, int);
133 static int vq_ring_enable_interrupt(struct virtqueue *, uint16_t);
134 static int vq_ring_must_notify_host(struct virtqueue *);
135 static void vq_ring_notify_host(struct virtqueue *);
136 static void vq_ring_free_chain(struct virtqueue *, uint16_t);
137
138 #define vq_modern(_vq) (((_vq)->vq_flags & VIRTQUEUE_FLAG_MODERN) != 0)
139 #define vq_htog16(_vq, _val) virtio_htog16(vq_modern(_vq), _val)
140 #define vq_htog32(_vq, _val) virtio_htog32(vq_modern(_vq), _val)
141 #define vq_htog64(_vq, _val) virtio_htog64(vq_modern(_vq), _val)
142 #define vq_gtoh16(_vq, _val) virtio_gtoh16(vq_modern(_vq), _val)
143 #define vq_gtoh32(_vq, _val) virtio_gtoh32(vq_modern(_vq), _val)
144 #define vq_gtoh64(_vq, _val) virtio_gtoh64(vq_modern(_vq), _val)
145
146 int
virtqueue_alloc(device_t dev,uint16_t queue,uint16_t size,bus_size_t notify_offset,int align,vm_paddr_t highaddr,struct vq_alloc_info * info,struct virtqueue ** vqp)147 virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size,
148 bus_size_t notify_offset, int align, vm_paddr_t highaddr,
149 struct vq_alloc_info *info, struct virtqueue **vqp)
150 {
151 struct virtqueue *vq;
152 int error;
153
154 *vqp = NULL;
155 error = 0;
156
157 if (size == 0) {
158 device_printf(dev,
159 "virtqueue %d (%s) does not exist (size is zero)\n",
160 queue, info->vqai_name);
161 return (ENODEV);
162 } else if (!powerof2(size)) {
163 device_printf(dev,
164 "virtqueue %d (%s) size is not a power of 2: %d\n",
165 queue, info->vqai_name, size);
166 return (ENXIO);
167 } else if (info->vqai_maxindirsz > VIRTIO_MAX_INDIRECT) {
168 device_printf(dev, "virtqueue %d (%s) requested too many "
169 "indirect descriptors: %d, max %d\n",
170 queue, info->vqai_name, info->vqai_maxindirsz,
171 VIRTIO_MAX_INDIRECT);
172 return (EINVAL);
173 }
174
175 vq = malloc(sizeof(struct virtqueue) +
176 size * sizeof(struct vq_desc_extra), M_DEVBUF, M_NOWAIT | M_ZERO);
177 if (vq == NULL) {
178 device_printf(dev, "cannot allocate virtqueue\n");
179 return (ENOMEM);
180 }
181
182 vq->vq_dev = dev;
183 strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name));
184 vq->vq_queue_index = queue;
185 vq->vq_notify_offset = notify_offset;
186 vq->vq_alignment = align;
187 vq->vq_nentries = size;
188 vq->vq_free_cnt = size;
189 vq->vq_intrhand = info->vqai_intr;
190 vq->vq_intrhand_arg = info->vqai_intr_arg;
191
192 if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_F_VERSION_1) != 0)
193 vq->vq_flags |= VIRTQUEUE_FLAG_MODERN;
194 if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0)
195 vq->vq_flags |= VIRTQUEUE_FLAG_EVENT_IDX;
196
197 if (info->vqai_maxindirsz > 1) {
198 error = virtqueue_init_indirect(vq, info->vqai_maxindirsz);
199 if (error)
200 goto fail;
201 }
202
203 vq->vq_ring_size = round_page(vring_size(size, align));
204 vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF,
205 M_NOWAIT | M_ZERO, 0, highaddr, PAGE_SIZE, 0);
206 if (vq->vq_ring_mem == NULL) {
207 device_printf(dev,
208 "cannot allocate memory for virtqueue ring\n");
209 error = ENOMEM;
210 goto fail;
211 }
212
213 vq_ring_init(vq);
214 virtqueue_disable_intr(vq);
215
216 *vqp = vq;
217
218 fail:
219 if (error)
220 virtqueue_free(vq);
221
222 return (error);
223 }
224
225 static int
virtqueue_init_indirect(struct virtqueue * vq,int indirect_size)226 virtqueue_init_indirect(struct virtqueue *vq, int indirect_size)
227 {
228 device_t dev;
229 struct vq_desc_extra *dxp;
230 int i, size;
231
232 dev = vq->vq_dev;
233
234 if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_INDIRECT_DESC) == 0) {
235 /*
236 * Indirect descriptors requested by the driver but not
237 * negotiated. Return zero to keep the initialization
238 * going: we'll run fine without.
239 */
240 if (bootverbose)
241 device_printf(dev, "virtqueue %d (%s) requested "
242 "indirect descriptors but not negotiated\n",
243 vq->vq_queue_index, vq->vq_name);
244 return (0);
245 }
246
247 size = indirect_size * sizeof(struct vring_desc);
248 vq->vq_max_indirect_size = indirect_size;
249 vq->vq_indirect_mem_size = size;
250 vq->vq_flags |= VIRTQUEUE_FLAG_INDIRECT;
251
252 for (i = 0; i < vq->vq_nentries; i++) {
253 dxp = &vq->vq_descx[i];
254
255 dxp->indirect = malloc(size, M_DEVBUF, M_NOWAIT);
256 if (dxp->indirect == NULL) {
257 device_printf(dev, "cannot allocate indirect list\n");
258 return (ENOMEM);
259 }
260
261 dxp->indirect_paddr = vtophys(dxp->indirect);
262 virtqueue_init_indirect_list(vq, dxp->indirect);
263 }
264
265 return (0);
266 }
267
268 static void
virtqueue_free_indirect(struct virtqueue * vq)269 virtqueue_free_indirect(struct virtqueue *vq)
270 {
271 struct vq_desc_extra *dxp;
272 int i;
273
274 for (i = 0; i < vq->vq_nentries; i++) {
275 dxp = &vq->vq_descx[i];
276
277 if (dxp->indirect == NULL)
278 break;
279
280 free(dxp->indirect, M_DEVBUF);
281 dxp->indirect = NULL;
282 dxp->indirect_paddr = 0;
283 }
284
285 vq->vq_flags &= ~VIRTQUEUE_FLAG_INDIRECT;
286 vq->vq_indirect_mem_size = 0;
287 }
288
289 static void
virtqueue_init_indirect_list(struct virtqueue * vq,struct vring_desc * indirect)290 virtqueue_init_indirect_list(struct virtqueue *vq,
291 struct vring_desc *indirect)
292 {
293 int i;
294
295 bzero(indirect, vq->vq_indirect_mem_size);
296
297 for (i = 0; i < vq->vq_max_indirect_size - 1; i++)
298 indirect[i].next = vq_gtoh16(vq, i + 1);
299 indirect[i].next = vq_gtoh16(vq, VQ_RING_DESC_CHAIN_END);
300 }
301
302 int
virtqueue_reinit(struct virtqueue * vq,uint16_t size)303 virtqueue_reinit(struct virtqueue *vq, uint16_t size)
304 {
305 struct vq_desc_extra *dxp;
306 int i;
307
308 if (vq->vq_nentries != size) {
309 device_printf(vq->vq_dev,
310 "%s: '%s' changed size; old=%hu, new=%hu\n",
311 __func__, vq->vq_name, vq->vq_nentries, size);
312 return (EINVAL);
313 }
314
315 /* Warn if the virtqueue was not properly cleaned up. */
316 if (vq->vq_free_cnt != vq->vq_nentries) {
317 device_printf(vq->vq_dev,
318 "%s: warning '%s' virtqueue not empty, "
319 "leaking %d entries\n", __func__, vq->vq_name,
320 vq->vq_nentries - vq->vq_free_cnt);
321 }
322
323 vq->vq_desc_head_idx = 0;
324 vq->vq_used_cons_idx = 0;
325 vq->vq_queued_cnt = 0;
326 vq->vq_free_cnt = vq->vq_nentries;
327
328 /* To be safe, reset all our allocated memory. */
329 bzero(vq->vq_ring_mem, vq->vq_ring_size);
330 for (i = 0; i < vq->vq_nentries; i++) {
331 dxp = &vq->vq_descx[i];
332 dxp->cookie = NULL;
333 dxp->ndescs = 0;
334 if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
335 virtqueue_init_indirect_list(vq, dxp->indirect);
336 }
337
338 vq_ring_init(vq);
339 virtqueue_disable_intr(vq);
340
341 return (0);
342 }
343
344 void
virtqueue_free(struct virtqueue * vq)345 virtqueue_free(struct virtqueue *vq)
346 {
347
348 if (vq->vq_free_cnt != vq->vq_nentries) {
349 device_printf(vq->vq_dev, "%s: freeing non-empty virtqueue, "
350 "leaking %d entries\n", vq->vq_name,
351 vq->vq_nentries - vq->vq_free_cnt);
352 }
353
354 if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
355 virtqueue_free_indirect(vq);
356
357 if (vq->vq_ring_mem != NULL) {
358 contigfree(vq->vq_ring_mem, vq->vq_ring_size, M_DEVBUF);
359 vq->vq_ring_size = 0;
360 vq->vq_ring_mem = NULL;
361 }
362
363 free(vq, M_DEVBUF);
364 }
365
366 vm_paddr_t
virtqueue_paddr(struct virtqueue * vq)367 virtqueue_paddr(struct virtqueue *vq)
368 {
369
370 return (vtophys(vq->vq_ring_mem));
371 }
372
373 vm_paddr_t
virtqueue_desc_paddr(struct virtqueue * vq)374 virtqueue_desc_paddr(struct virtqueue *vq)
375 {
376
377 return (vtophys(vq->vq_ring.desc));
378 }
379
380 vm_paddr_t
virtqueue_avail_paddr(struct virtqueue * vq)381 virtqueue_avail_paddr(struct virtqueue *vq)
382 {
383
384 return (vtophys(vq->vq_ring.avail));
385 }
386
387 vm_paddr_t
virtqueue_used_paddr(struct virtqueue * vq)388 virtqueue_used_paddr(struct virtqueue *vq)
389 {
390
391 return (vtophys(vq->vq_ring.used));
392 }
393
394 uint16_t
virtqueue_index(struct virtqueue * vq)395 virtqueue_index(struct virtqueue *vq)
396 {
397
398 return (vq->vq_queue_index);
399 }
400
401 int
virtqueue_size(struct virtqueue * vq)402 virtqueue_size(struct virtqueue *vq)
403 {
404
405 return (vq->vq_nentries);
406 }
407
408 int
virtqueue_nfree(struct virtqueue * vq)409 virtqueue_nfree(struct virtqueue *vq)
410 {
411
412 return (vq->vq_free_cnt);
413 }
414
415 int
virtqueue_empty(struct virtqueue * vq)416 virtqueue_empty(struct virtqueue *vq)
417 {
418
419 return (vq->vq_nentries == vq->vq_free_cnt);
420 }
421
422 int
virtqueue_full(struct virtqueue * vq)423 virtqueue_full(struct virtqueue *vq)
424 {
425
426 return (vq->vq_free_cnt == 0);
427 }
428
429 void
virtqueue_notify(struct virtqueue * vq)430 virtqueue_notify(struct virtqueue *vq)
431 {
432
433 /* Ensure updated avail->idx is visible to host. */
434 mb();
435
436 if (vq_ring_must_notify_host(vq))
437 vq_ring_notify_host(vq);
438 vq->vq_queued_cnt = 0;
439 }
440
441 int
virtqueue_nused(struct virtqueue * vq)442 virtqueue_nused(struct virtqueue *vq)
443 {
444 uint16_t used_idx, nused;
445
446 used_idx = vq_htog16(vq, vq->vq_ring.used->idx);
447
448 nused = (uint16_t)(used_idx - vq->vq_used_cons_idx);
449 VQASSERT(vq, nused <= vq->vq_nentries, "used more than available");
450
451 return (nused);
452 }
453
454 int
virtqueue_intr_filter(struct virtqueue * vq)455 virtqueue_intr_filter(struct virtqueue *vq)
456 {
457
458 if (vq->vq_used_cons_idx == vq_htog16(vq, vq->vq_ring.used->idx))
459 return (0);
460
461 virtqueue_disable_intr(vq);
462
463 return (1);
464 }
465
466 void
virtqueue_intr(struct virtqueue * vq)467 virtqueue_intr(struct virtqueue *vq)
468 {
469
470 vq->vq_intrhand(vq->vq_intrhand_arg);
471 }
472
473 int
virtqueue_enable_intr(struct virtqueue * vq)474 virtqueue_enable_intr(struct virtqueue *vq)
475 {
476
477 return (vq_ring_enable_interrupt(vq, 0));
478 }
479
480 int
virtqueue_postpone_intr(struct virtqueue * vq,vq_postpone_t hint)481 virtqueue_postpone_intr(struct virtqueue *vq, vq_postpone_t hint)
482 {
483 uint16_t ndesc, avail_idx;
484
485 avail_idx = vq_htog16(vq, vq->vq_ring.avail->idx);
486 ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx);
487
488 switch (hint) {
489 case VQ_POSTPONE_SHORT:
490 ndesc = ndesc / 4;
491 break;
492 case VQ_POSTPONE_LONG:
493 ndesc = (ndesc * 3) / 4;
494 break;
495 case VQ_POSTPONE_EMPTIED:
496 break;
497 }
498
499 return (vq_ring_enable_interrupt(vq, ndesc));
500 }
501
502 /*
503 * Note this is only considered a hint to the host.
504 */
505 void
virtqueue_disable_intr(struct virtqueue * vq)506 virtqueue_disable_intr(struct virtqueue *vq)
507 {
508
509 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
510 vring_used_event(&vq->vq_ring) = vq_gtoh16(vq,
511 vq->vq_used_cons_idx - vq->vq_nentries - 1);
512 return;
513 }
514
515 vq->vq_ring.avail->flags |= vq_gtoh16(vq, VRING_AVAIL_F_NO_INTERRUPT);
516 }
517
518 int
virtqueue_enqueue(struct virtqueue * vq,void * cookie,struct sglist * sg,int readable,int writable)519 virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg,
520 int readable, int writable)
521 {
522 struct vq_desc_extra *dxp;
523 int needed;
524 uint16_t head_idx, idx;
525
526 needed = readable + writable;
527
528 VQASSERT(vq, cookie != NULL, "enqueuing with no cookie");
529 VQASSERT(vq, needed == sg->sg_nseg,
530 "segment count mismatch, %d, %d", needed, sg->sg_nseg);
531 VQASSERT(vq,
532 needed <= vq->vq_nentries || needed <= vq->vq_max_indirect_size,
533 "too many segments to enqueue: %d, %d/%d", needed,
534 vq->vq_nentries, vq->vq_max_indirect_size);
535
536 if (needed < 1)
537 return (EINVAL);
538 if (vq->vq_free_cnt == 0)
539 return (ENOSPC);
540
541 if (vq_ring_use_indirect(vq, needed)) {
542 vq_ring_enqueue_indirect(vq, cookie, sg, readable, writable);
543 return (0);
544 } else if (vq->vq_free_cnt < needed)
545 return (EMSGSIZE);
546
547 head_idx = vq->vq_desc_head_idx;
548 VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
549 dxp = &vq->vq_descx[head_idx];
550
551 VQASSERT(vq, dxp->cookie == NULL,
552 "cookie already exists for index %d", head_idx);
553 dxp->cookie = cookie;
554 dxp->ndescs = needed;
555
556 idx = vq_ring_enqueue_segments(vq, vq->vq_ring.desc, head_idx,
557 sg, readable, writable);
558
559 vq->vq_desc_head_idx = idx;
560 vq->vq_free_cnt -= needed;
561 if (vq->vq_free_cnt == 0)
562 VQ_RING_ASSERT_CHAIN_TERM(vq);
563 else
564 VQ_RING_ASSERT_VALID_IDX(vq, idx);
565
566 vq_ring_update_avail(vq, head_idx);
567
568 return (0);
569 }
570
571 void *
virtqueue_dequeue(struct virtqueue * vq,uint32_t * len)572 virtqueue_dequeue(struct virtqueue *vq, uint32_t *len)
573 {
574 struct vring_used_elem *uep;
575 void *cookie;
576 uint16_t used_idx, desc_idx;
577
578 if (vq->vq_used_cons_idx == vq_htog16(vq, vq->vq_ring.used->idx))
579 return (NULL);
580
581 used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1);
582 uep = &vq->vq_ring.used->ring[used_idx];
583
584 rmb();
585 desc_idx = (uint16_t) vq_htog32(vq, uep->id);
586 if (len != NULL)
587 *len = vq_htog32(vq, uep->len);
588
589 vq_ring_free_chain(vq, desc_idx);
590
591 cookie = vq->vq_descx[desc_idx].cookie;
592 VQASSERT(vq, cookie != NULL, "no cookie for index %d", desc_idx);
593 vq->vq_descx[desc_idx].cookie = NULL;
594
595 return (cookie);
596 }
597
598 void *
virtqueue_poll(struct virtqueue * vq,uint32_t * len)599 virtqueue_poll(struct virtqueue *vq, uint32_t *len)
600 {
601 void *cookie;
602
603 VIRTIO_BUS_POLL(vq->vq_dev);
604 while ((cookie = virtqueue_dequeue(vq, len)) == NULL) {
605 cpu_spinwait();
606 VIRTIO_BUS_POLL(vq->vq_dev);
607 }
608
609 return (cookie);
610 }
611
612 void *
virtqueue_drain(struct virtqueue * vq,int * last)613 virtqueue_drain(struct virtqueue *vq, int *last)
614 {
615 void *cookie;
616 int idx;
617
618 cookie = NULL;
619 idx = *last;
620
621 while (idx < vq->vq_nentries && cookie == NULL) {
622 if ((cookie = vq->vq_descx[idx].cookie) != NULL) {
623 vq->vq_descx[idx].cookie = NULL;
624 /* Free chain to keep free count consistent. */
625 vq_ring_free_chain(vq, idx);
626 }
627 idx++;
628 }
629
630 *last = idx;
631
632 return (cookie);
633 }
634
635 void
virtqueue_dump(struct virtqueue * vq)636 virtqueue_dump(struct virtqueue *vq)
637 {
638
639 if (vq == NULL)
640 return;
641
642 printf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; "
643 "desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; "
644 "used.idx=%d; used_event_idx=%d; avail.flags=0x%x; used.flags=0x%x\n",
645 vq->vq_name, vq->vq_nentries, vq->vq_free_cnt, virtqueue_nused(vq),
646 vq->vq_queued_cnt, vq->vq_desc_head_idx,
647 vq_htog16(vq, vq->vq_ring.avail->idx), vq->vq_used_cons_idx,
648 vq_htog16(vq, vq->vq_ring.used->idx),
649 vq_htog16(vq, vring_used_event(&vq->vq_ring)),
650 vq_htog16(vq, vq->vq_ring.avail->flags),
651 vq_htog16(vq, vq->vq_ring.used->flags));
652 }
653
654 static void
vq_ring_init(struct virtqueue * vq)655 vq_ring_init(struct virtqueue *vq)
656 {
657 struct vring *vr;
658 char *ring_mem;
659 int i, size;
660
661 ring_mem = vq->vq_ring_mem;
662 size = vq->vq_nentries;
663 vr = &vq->vq_ring;
664
665 vring_init(vr, size, ring_mem, vq->vq_alignment);
666
667 for (i = 0; i < size - 1; i++)
668 vr->desc[i].next = vq_gtoh16(vq, i + 1);
669 vr->desc[i].next = vq_gtoh16(vq, VQ_RING_DESC_CHAIN_END);
670 }
671
672 static void
vq_ring_update_avail(struct virtqueue * vq,uint16_t desc_idx)673 vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx)
674 {
675 uint16_t avail_idx, avail_ring_idx;
676
677 /*
678 * Place the head of the descriptor chain into the next slot and make
679 * it usable to the host. The chain is made available now rather than
680 * deferring to virtqueue_notify() in the hopes that if the host is
681 * currently running on another CPU, we can keep it processing the new
682 * descriptor.
683 */
684 avail_idx = vq_htog16(vq, vq->vq_ring.avail->idx);
685 avail_ring_idx = avail_idx & (vq->vq_nentries - 1);
686 vq->vq_ring.avail->ring[avail_ring_idx] = vq_gtoh16(vq, desc_idx);
687
688 wmb();
689 vq->vq_ring.avail->idx = vq_gtoh16(vq, avail_idx + 1);
690
691 /* Keep pending count until virtqueue_notify(). */
692 vq->vq_queued_cnt++;
693 }
694
695 static uint16_t
vq_ring_enqueue_segments(struct virtqueue * vq,struct vring_desc * desc,uint16_t head_idx,struct sglist * sg,int readable,int writable)696 vq_ring_enqueue_segments(struct virtqueue *vq, struct vring_desc *desc,
697 uint16_t head_idx, struct sglist *sg, int readable, int writable)
698 {
699 struct sglist_seg *seg;
700 struct vring_desc *dp;
701 int i, needed;
702 uint16_t idx;
703
704 needed = readable + writable;
705
706 for (i = 0, idx = head_idx, seg = sg->sg_segs;
707 i < needed;
708 i++, idx = vq_htog16(vq, dp->next), seg++) {
709 VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END,
710 "premature end of free desc chain");
711
712 dp = &desc[idx];
713 dp->addr = vq_gtoh64(vq, seg->ss_paddr);
714 dp->len = vq_gtoh32(vq, seg->ss_len);
715 dp->flags = 0;
716
717 if (i < needed - 1)
718 dp->flags |= vq_gtoh16(vq, VRING_DESC_F_NEXT);
719 if (i >= readable)
720 dp->flags |= vq_gtoh16(vq, VRING_DESC_F_WRITE);
721 }
722
723 return (idx);
724 }
725
726 static int
vq_ring_use_indirect(struct virtqueue * vq,int needed)727 vq_ring_use_indirect(struct virtqueue *vq, int needed)
728 {
729
730 if ((vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) == 0)
731 return (0);
732
733 if (vq->vq_max_indirect_size < needed)
734 return (0);
735
736 if (needed < 2)
737 return (0);
738
739 return (1);
740 }
741
742 static void
vq_ring_enqueue_indirect(struct virtqueue * vq,void * cookie,struct sglist * sg,int readable,int writable)743 vq_ring_enqueue_indirect(struct virtqueue *vq, void *cookie,
744 struct sglist *sg, int readable, int writable)
745 {
746 struct vring_desc *dp;
747 struct vq_desc_extra *dxp;
748 int needed;
749 uint16_t head_idx;
750
751 needed = readable + writable;
752 VQASSERT(vq, needed <= vq->vq_max_indirect_size,
753 "enqueuing too many indirect descriptors");
754
755 head_idx = vq->vq_desc_head_idx;
756 VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
757 dp = &vq->vq_ring.desc[head_idx];
758 dxp = &vq->vq_descx[head_idx];
759
760 VQASSERT(vq, dxp->cookie == NULL,
761 "cookie already exists for index %d", head_idx);
762 dxp->cookie = cookie;
763 dxp->ndescs = 1;
764
765 dp->addr = vq_gtoh64(vq, dxp->indirect_paddr);
766 dp->len = vq_gtoh32(vq, needed * sizeof(struct vring_desc));
767 dp->flags = vq_gtoh16(vq, VRING_DESC_F_INDIRECT);
768
769 vq_ring_enqueue_segments(vq, dxp->indirect, 0,
770 sg, readable, writable);
771
772 vq->vq_desc_head_idx = vq_htog16(vq, dp->next);
773 vq->vq_free_cnt--;
774 if (vq->vq_free_cnt == 0)
775 VQ_RING_ASSERT_CHAIN_TERM(vq);
776 else
777 VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx);
778
779 vq_ring_update_avail(vq, head_idx);
780 }
781
782 static int
vq_ring_enable_interrupt(struct virtqueue * vq,uint16_t ndesc)783 vq_ring_enable_interrupt(struct virtqueue *vq, uint16_t ndesc)
784 {
785
786 /*
787 * Enable interrupts, making sure we get the latest index of
788 * what's already been consumed.
789 */
790 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
791 vring_used_event(&vq->vq_ring) =
792 vq_gtoh16(vq, vq->vq_used_cons_idx + ndesc);
793 } else {
794 vq->vq_ring.avail->flags &=
795 vq_gtoh16(vq, ~VRING_AVAIL_F_NO_INTERRUPT);
796 }
797
798 mb();
799
800 /*
801 * Enough items may have already been consumed to meet our threshold
802 * since we last checked. Let our caller know so it processes the new
803 * entries.
804 */
805 if (virtqueue_nused(vq) > ndesc)
806 return (1);
807
808 return (0);
809 }
810
811 static int
vq_ring_must_notify_host(struct virtqueue * vq)812 vq_ring_must_notify_host(struct virtqueue *vq)
813 {
814 uint16_t new_idx, prev_idx, event_idx, flags;
815
816 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
817 new_idx = vq_htog16(vq, vq->vq_ring.avail->idx);
818 prev_idx = new_idx - vq->vq_queued_cnt;
819 event_idx = vq_htog16(vq, vring_avail_event(&vq->vq_ring));
820
821 return (vring_need_event(event_idx, new_idx, prev_idx) != 0);
822 }
823
824 flags = vq->vq_ring.used->flags;
825 return ((flags & vq_gtoh16(vq, VRING_USED_F_NO_NOTIFY)) == 0);
826 }
827
828 static void
vq_ring_notify_host(struct virtqueue * vq)829 vq_ring_notify_host(struct virtqueue *vq)
830 {
831
832 VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index,
833 vq->vq_notify_offset);
834 }
835
836 static void
vq_ring_free_chain(struct virtqueue * vq,uint16_t desc_idx)837 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
838 {
839 struct vring_desc *dp;
840 struct vq_desc_extra *dxp;
841
842 VQ_RING_ASSERT_VALID_IDX(vq, desc_idx);
843 dp = &vq->vq_ring.desc[desc_idx];
844 dxp = &vq->vq_descx[desc_idx];
845
846 if (vq->vq_free_cnt == 0)
847 VQ_RING_ASSERT_CHAIN_TERM(vq);
848
849 vq->vq_free_cnt += dxp->ndescs;
850 dxp->ndescs--;
851
852 if ((dp->flags & vq_gtoh16(vq, VRING_DESC_F_INDIRECT)) == 0) {
853 while (dp->flags & vq_gtoh16(vq, VRING_DESC_F_NEXT)) {
854 uint16_t next_idx = vq_htog16(vq, dp->next);
855 VQ_RING_ASSERT_VALID_IDX(vq, next_idx);
856 dp = &vq->vq_ring.desc[next_idx];
857 dxp->ndescs--;
858 }
859 }
860
861 VQASSERT(vq, dxp->ndescs == 0,
862 "failed to free entire desc chain, remaining: %d", dxp->ndescs);
863
864 /*
865 * We must append the existing free chain, if any, to the end of
866 * newly freed chain. If the virtqueue was completely used, then
867 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
868 */
869 dp->next = vq_gtoh16(vq, vq->vq_desc_head_idx);
870 vq->vq_desc_head_idx = desc_idx;
871 }
872