1 /* $NetBSD: virtio.c,v 1.82 2024/08/05 19:26:43 riastradh Exp $ */
2
3 /*
4 * Copyright (c) 2020 The NetBSD Foundation, Inc.
5 * Copyright (c) 2012 Stefan Fritsch, Alexander Fiveg.
6 * Copyright (c) 2010 Minoura Makoto.
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 __KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.82 2024/08/05 19:26:43 riastradh Exp $");
32
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/kernel.h>
36 #include <sys/atomic.h>
37 #include <sys/bus.h>
38 #include <sys/device.h>
39 #include <sys/kmem.h>
40 #include <sys/module.h>
41
42 #define VIRTIO_PRIVATE
43
44 #include <dev/pci/virtioreg.h> /* XXX: move to non-pci */
45 #include <dev/pci/virtiovar.h> /* XXX: move to non-pci */
46
47 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */
48
49 /*
50 * The maximum descriptor size is 2^15. Use that value as the end of
51 * descriptor chain terminator since it will never be a valid index
52 * in the descriptor table.
53 */
54 #define VRING_DESC_CHAIN_END 32768
55
56 /* incomplete list */
57 static const char *virtio_device_name[] = {
58 "unknown (0)", /* 0 */
59 "network", /* 1 */
60 "block", /* 2 */
61 "console", /* 3 */
62 "entropy", /* 4 */
63 "memory balloon", /* 5 */
64 "I/O memory", /* 6 */
65 "remote processor messaging", /* 7 */
66 "SCSI", /* 8 */
67 "9P transport", /* 9 */
68 };
69 #define NDEVNAMES __arraycount(virtio_device_name)
70
71 static void virtio_reset_vq(struct virtio_softc *,
72 struct virtqueue *);
73
74 void
virtio_set_status(struct virtio_softc * sc,int status)75 virtio_set_status(struct virtio_softc *sc, int status)
76 {
77 sc->sc_ops->set_status(sc, status);
78 }
79
80 /*
81 * Reset the device.
82 */
83 /*
84 * To reset the device to a known state, do following:
85 * virtio_reset(sc); // this will stop the device activity
86 * <dequeue finished requests>; // virtio_dequeue() still can be called
87 * <revoke pending requests in the vqs if any>;
88 * virtio_reinit_start(sc); // dequeue prohibited
89 * newfeatures = virtio_negotiate_features(sc, requestedfeatures);
90 * <some other initialization>;
91 * virtio_reinit_end(sc); // device activated; enqueue allowed
92 * Once attached, feature negotiation can only be allowed after virtio_reset.
93 */
94 void
virtio_reset(struct virtio_softc * sc)95 virtio_reset(struct virtio_softc *sc)
96 {
97 virtio_device_reset(sc);
98 }
99
100 int
virtio_reinit_start(struct virtio_softc * sc)101 virtio_reinit_start(struct virtio_softc *sc)
102 {
103 int i, r;
104
105 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
106 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
107 for (i = 0; i < sc->sc_nvqs; i++) {
108 int n;
109 struct virtqueue *vq = &sc->sc_vqs[i];
110 n = sc->sc_ops->read_queue_size(sc, vq->vq_index);
111 if (n == 0) /* vq disappeared */
112 continue;
113 if (n != vq->vq_num) {
114 panic("%s: virtqueue size changed, vq index %d\n",
115 device_xname(sc->sc_dev),
116 vq->vq_index);
117 }
118 virtio_reset_vq(sc, vq);
119 sc->sc_ops->setup_queue(sc, vq->vq_index,
120 vq->vq_dmamap->dm_segs[0].ds_addr);
121 }
122
123 r = sc->sc_ops->setup_interrupts(sc, 1);
124 if (r != 0)
125 goto fail;
126
127 return 0;
128
129 fail:
130 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
131
132 return 1;
133 }
134
135 void
virtio_reinit_end(struct virtio_softc * sc)136 virtio_reinit_end(struct virtio_softc *sc)
137 {
138 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
139 }
140
141 /*
142 * Feature negotiation.
143 */
144 void
virtio_negotiate_features(struct virtio_softc * sc,uint64_t guest_features)145 virtio_negotiate_features(struct virtio_softc *sc, uint64_t guest_features)
146 {
147 if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) &&
148 !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */
149 guest_features |= VIRTIO_F_RING_INDIRECT_DESC;
150 sc->sc_ops->neg_features(sc, guest_features);
151 if (sc->sc_active_features & VIRTIO_F_RING_INDIRECT_DESC)
152 sc->sc_indirect = true;
153 else
154 sc->sc_indirect = false;
155 }
156
157
158 /*
159 * Device configuration registers readers/writers
160 */
161 #if 0
162 #define DPRINTFR(n, fmt, val, index, num) \
163 printf("\n%s (", n); \
164 for (int i = 0; i < num; i++) \
165 printf("%02x ", bus_space_read_1(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index+i)); \
166 printf(") -> "); printf(fmt, val); printf("\n");
167 #define DPRINTFR2(n, fmt, val_s, val_n) \
168 printf("%s ", n); \
169 printf("\n stream "); printf(fmt, val_s); printf(" norm "); printf(fmt, val_n); printf("\n");
170 #else
171 #define DPRINTFR(n, fmt, val, index, num)
172 #define DPRINTFR2(n, fmt, val_s, val_n)
173 #endif
174
175
176 uint8_t
virtio_read_device_config_1(struct virtio_softc * sc,int index)177 virtio_read_device_config_1(struct virtio_softc *sc, int index)
178 {
179 bus_space_tag_t iot = sc->sc_devcfg_iot;
180 bus_space_handle_t ioh = sc->sc_devcfg_ioh;
181 uint8_t val;
182
183 val = bus_space_read_1(iot, ioh, index);
184
185 DPRINTFR("read_1", "%02x", val, index, 1);
186 return val;
187 }
188
189 uint16_t
virtio_read_device_config_2(struct virtio_softc * sc,int index)190 virtio_read_device_config_2(struct virtio_softc *sc, int index)
191 {
192 bus_space_tag_t iot = sc->sc_devcfg_iot;
193 bus_space_handle_t ioh = sc->sc_devcfg_ioh;
194 uint16_t val;
195
196 val = bus_space_read_2(iot, ioh, index);
197 if (BYTE_ORDER != sc->sc_bus_endian)
198 val = bswap16(val);
199
200 DPRINTFR("read_2", "%04x", val, index, 2);
201 DPRINTFR2("read_2", "%04x",
202 bus_space_read_stream_2(sc->sc_devcfg_iot, sc->sc_devcfg_ioh,
203 index),
204 bus_space_read_2(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index));
205 return val;
206 }
207
208 uint32_t
virtio_read_device_config_4(struct virtio_softc * sc,int index)209 virtio_read_device_config_4(struct virtio_softc *sc, int index)
210 {
211 bus_space_tag_t iot = sc->sc_devcfg_iot;
212 bus_space_handle_t ioh = sc->sc_devcfg_ioh;
213 uint32_t val;
214
215 val = bus_space_read_4(iot, ioh, index);
216 if (BYTE_ORDER != sc->sc_bus_endian)
217 val = bswap32(val);
218
219 DPRINTFR("read_4", "%08x", val, index, 4);
220 DPRINTFR2("read_4", "%08x",
221 bus_space_read_stream_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh,
222 index),
223 bus_space_read_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index));
224 return val;
225 }
226
227 /*
228 * The Virtio spec explicitly tells that reading and writing 8 bytes are not
229 * considered atomic and no triggers may be connected to reading or writing
230 * it. We access it using two 32 reads. See virtio spec 4.1.3.1.
231 */
232 uint64_t
virtio_read_device_config_8(struct virtio_softc * sc,int index)233 virtio_read_device_config_8(struct virtio_softc *sc, int index)
234 {
235 bus_space_tag_t iot = sc->sc_devcfg_iot;
236 bus_space_handle_t ioh = sc->sc_devcfg_ioh;
237 union {
238 uint64_t u64;
239 uint32_t l[2];
240 } v;
241 uint64_t val;
242
243 v.l[0] = bus_space_read_4(iot, ioh, index);
244 v.l[1] = bus_space_read_4(iot, ioh, index + 4);
245 if (sc->sc_bus_endian != sc->sc_struct_endian) {
246 v.l[0] = bswap32(v.l[0]);
247 v.l[1] = bswap32(v.l[1]);
248 }
249 val = v.u64;
250
251 if (BYTE_ORDER != sc->sc_struct_endian)
252 val = bswap64(val);
253
254 DPRINTFR("read_8", "%08"PRIx64, val, index, 8);
255 DPRINTFR2("read_8 low ", "%08x",
256 bus_space_read_stream_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh,
257 index),
258 bus_space_read_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index));
259 DPRINTFR2("read_8 high ", "%08x",
260 bus_space_read_stream_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh,
261 index + 4),
262 bus_space_read_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index + 4));
263 return val;
264 }
265
266 /*
267 * In the older virtio spec, device config registers are host endian. On newer
268 * they are little endian. Some newer devices however explicitly specify their
269 * register to always be little endian. These functions cater for these.
270 */
271 uint16_t
virtio_read_device_config_le_2(struct virtio_softc * sc,int index)272 virtio_read_device_config_le_2(struct virtio_softc *sc, int index)
273 {
274 bus_space_tag_t iot = sc->sc_devcfg_iot;
275 bus_space_handle_t ioh = sc->sc_devcfg_ioh;
276 uint16_t val;
277
278 val = bus_space_read_2(iot, ioh, index);
279 #if !defined(__aarch64__) && !defined(__arm__)
280 /*
281 * For big-endian aarch64/armv7, bus endian is always LSB, but
282 * byte-order is automatically swapped by bus_space(9) (see also
283 * comments in virtio_pci.c). Therefore, no need to swap here.
284 */
285 if (sc->sc_bus_endian != LITTLE_ENDIAN)
286 val = bswap16(val);
287 #endif
288
289 DPRINTFR("read_le_2", "%04x", val, index, 2);
290 DPRINTFR2("read_le_2", "%04x",
291 bus_space_read_stream_2(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 0),
292 bus_space_read_2(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 0));
293 return val;
294 }
295
296 uint32_t
virtio_read_device_config_le_4(struct virtio_softc * sc,int index)297 virtio_read_device_config_le_4(struct virtio_softc *sc, int index)
298 {
299 bus_space_tag_t iot = sc->sc_devcfg_iot;
300 bus_space_handle_t ioh = sc->sc_devcfg_ioh;
301 uint32_t val;
302
303 val = bus_space_read_4(iot, ioh, index);
304 #if !defined(__aarch64__) && !defined(__arm__)
305 /* See virtio_read_device_config_le_2() above. */
306 if (sc->sc_bus_endian != LITTLE_ENDIAN)
307 val = bswap32(val);
308 #endif
309
310 DPRINTFR("read_le_4", "%08x", val, index, 4);
311 DPRINTFR2("read_le_4", "%08x",
312 bus_space_read_stream_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 0),
313 bus_space_read_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 0));
314 return val;
315 }
316
317 void
virtio_write_device_config_1(struct virtio_softc * sc,int index,uint8_t value)318 virtio_write_device_config_1(struct virtio_softc *sc, int index, uint8_t value)
319 {
320 bus_space_tag_t iot = sc->sc_devcfg_iot;
321 bus_space_handle_t ioh = sc->sc_devcfg_ioh;
322
323 bus_space_write_1(iot, ioh, index, value);
324 }
325
326 void
virtio_write_device_config_2(struct virtio_softc * sc,int index,uint16_t value)327 virtio_write_device_config_2(struct virtio_softc *sc, int index,
328 uint16_t value)
329 {
330 bus_space_tag_t iot = sc->sc_devcfg_iot;
331 bus_space_handle_t ioh = sc->sc_devcfg_ioh;
332
333 if (BYTE_ORDER != sc->sc_bus_endian)
334 value = bswap16(value);
335 bus_space_write_2(iot, ioh, index, value);
336 }
337
338 void
virtio_write_device_config_4(struct virtio_softc * sc,int index,uint32_t value)339 virtio_write_device_config_4(struct virtio_softc *sc, int index,
340 uint32_t value)
341 {
342 bus_space_tag_t iot = sc->sc_devcfg_iot;
343 bus_space_handle_t ioh = sc->sc_devcfg_ioh;
344
345 if (BYTE_ORDER != sc->sc_bus_endian)
346 value = bswap32(value);
347 bus_space_write_4(iot, ioh, index, value);
348 }
349
350 /*
351 * The Virtio spec explicitly tells that reading and writing 8 bytes are not
352 * considered atomic and no triggers may be connected to reading or writing
353 * it. We access it using two 32 bit writes. For good measure it is stated to
354 * always write lsb first just in case of a hypervisor bug. See See virtio
355 * spec 4.1.3.1.
356 */
357 void
virtio_write_device_config_8(struct virtio_softc * sc,int index,uint64_t value)358 virtio_write_device_config_8(struct virtio_softc *sc, int index,
359 uint64_t value)
360 {
361 bus_space_tag_t iot = sc->sc_devcfg_iot;
362 bus_space_handle_t ioh = sc->sc_devcfg_ioh;
363 union {
364 uint64_t u64;
365 uint32_t l[2];
366 } v;
367
368 if (BYTE_ORDER != sc->sc_struct_endian)
369 value = bswap64(value);
370
371 v.u64 = value;
372 if (sc->sc_bus_endian != sc->sc_struct_endian) {
373 v.l[0] = bswap32(v.l[0]);
374 v.l[1] = bswap32(v.l[1]);
375 }
376
377 if (sc->sc_struct_endian == LITTLE_ENDIAN) {
378 bus_space_write_4(iot, ioh, index, v.l[0]);
379 bus_space_write_4(iot, ioh, index + 4, v.l[1]);
380 } else {
381 bus_space_write_4(iot, ioh, index + 4, v.l[1]);
382 bus_space_write_4(iot, ioh, index, v.l[0]);
383 }
384 }
385
386 /*
387 * In the older virtio spec, device config registers are host endian. On newer
388 * they are little endian. Some newer devices however explicitly specify their
389 * register to always be little endian. These functions cater for these.
390 */
391 void
virtio_write_device_config_le_2(struct virtio_softc * sc,int index,uint16_t value)392 virtio_write_device_config_le_2(struct virtio_softc *sc, int index,
393 uint16_t value)
394 {
395 bus_space_tag_t iot = sc->sc_devcfg_iot;
396 bus_space_handle_t ioh = sc->sc_devcfg_ioh;
397
398 if (sc->sc_bus_endian != LITTLE_ENDIAN)
399 value = bswap16(value);
400 bus_space_write_2(iot, ioh, index, value);
401 }
402
403 void
virtio_write_device_config_le_4(struct virtio_softc * sc,int index,uint32_t value)404 virtio_write_device_config_le_4(struct virtio_softc *sc, int index,
405 uint32_t value)
406 {
407 bus_space_tag_t iot = sc->sc_devcfg_iot;
408 bus_space_handle_t ioh = sc->sc_devcfg_ioh;
409
410 if (sc->sc_bus_endian != LITTLE_ENDIAN)
411 value = bswap32(value);
412 bus_space_write_4(iot, ioh, index, value);
413 }
414
415
416 /*
417 * data structures endian helpers
418 */
419 uint16_t
virtio_rw16(struct virtio_softc * sc,uint16_t val)420 virtio_rw16(struct virtio_softc *sc, uint16_t val)
421 {
422 KASSERT(sc);
423 return BYTE_ORDER != sc->sc_struct_endian ? bswap16(val) : val;
424 }
425
426 uint32_t
virtio_rw32(struct virtio_softc * sc,uint32_t val)427 virtio_rw32(struct virtio_softc *sc, uint32_t val)
428 {
429 KASSERT(sc);
430 return BYTE_ORDER != sc->sc_struct_endian ? bswap32(val) : val;
431 }
432
433 uint64_t
virtio_rw64(struct virtio_softc * sc,uint64_t val)434 virtio_rw64(struct virtio_softc *sc, uint64_t val)
435 {
436 KASSERT(sc);
437 return BYTE_ORDER != sc->sc_struct_endian ? bswap64(val) : val;
438 }
439
440
441 /*
442 * Interrupt handler.
443 */
444 static void
virtio_soft_intr(void * arg)445 virtio_soft_intr(void *arg)
446 {
447 struct virtio_softc *sc = arg;
448
449 KASSERT(sc->sc_intrhand != NULL);
450
451 (*sc->sc_intrhand)(sc);
452 }
453
454 /* set to vq->vq_intrhand in virtio_init_vq_vqdone() */
455 static int
virtio_vq_done(void * xvq)456 virtio_vq_done(void *xvq)
457 {
458 struct virtqueue *vq = xvq;
459
460 return vq->vq_done(vq);
461 }
462
463 static int
virtio_vq_intr(struct virtio_softc * sc)464 virtio_vq_intr(struct virtio_softc *sc)
465 {
466 struct virtqueue *vq;
467 int i, r = 0;
468
469 for (i = 0; i < sc->sc_nvqs; i++) {
470 vq = &sc->sc_vqs[i];
471 if (virtio_vq_is_enqueued(sc, vq) == 1) {
472 r |= (*vq->vq_intrhand)(vq->vq_intrhand_arg);
473 }
474 }
475
476 return r;
477 }
478
479 /*
480 * dmamap sync operations for a virtqueue.
481 */
482 static inline void
vq_sync_descs(struct virtio_softc * sc,struct virtqueue * vq,int ops)483 vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops)
484 {
485
486 /* availoffset == sizeof(vring_desc) * vq_num */
487 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset,
488 ops);
489 }
490
491 static inline void
vq_sync_aring_all(struct virtio_softc * sc,struct virtqueue * vq,int ops)492 vq_sync_aring_all(struct virtio_softc *sc, struct virtqueue *vq, int ops)
493 {
494 uint16_t hdrlen = offsetof(struct vring_avail, ring);
495 size_t payloadlen = vq->vq_num * sizeof(uint16_t);
496 size_t usedlen = 0;
497
498 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX)
499 usedlen = sizeof(uint16_t);
500 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
501 vq->vq_availoffset, hdrlen + payloadlen + usedlen, ops);
502 }
503
504 static inline void
vq_sync_aring_header(struct virtio_softc * sc,struct virtqueue * vq,int ops)505 vq_sync_aring_header(struct virtio_softc *sc, struct virtqueue *vq, int ops)
506 {
507 uint16_t hdrlen = offsetof(struct vring_avail, ring);
508
509 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
510 vq->vq_availoffset, hdrlen, ops);
511 }
512
513 static inline void
vq_sync_aring_payload(struct virtio_softc * sc,struct virtqueue * vq,int ops)514 vq_sync_aring_payload(struct virtio_softc *sc, struct virtqueue *vq, int ops)
515 {
516 uint16_t hdrlen = offsetof(struct vring_avail, ring);
517 size_t payloadlen = vq->vq_num * sizeof(uint16_t);
518
519 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
520 vq->vq_availoffset + hdrlen, payloadlen, ops);
521 }
522
523 static inline void
vq_sync_aring_used(struct virtio_softc * sc,struct virtqueue * vq,int ops)524 vq_sync_aring_used(struct virtio_softc *sc, struct virtqueue *vq, int ops)
525 {
526 uint16_t hdrlen = offsetof(struct vring_avail, ring);
527 size_t payloadlen = vq->vq_num * sizeof(uint16_t);
528 size_t usedlen = sizeof(uint16_t);
529
530 if ((sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) == 0)
531 return;
532 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
533 vq->vq_availoffset + hdrlen + payloadlen, usedlen, ops);
534 }
535
536 static inline void
vq_sync_uring_all(struct virtio_softc * sc,struct virtqueue * vq,int ops)537 vq_sync_uring_all(struct virtio_softc *sc, struct virtqueue *vq, int ops)
538 {
539 uint16_t hdrlen = offsetof(struct vring_used, ring);
540 size_t payloadlen = vq->vq_num * sizeof(struct vring_used_elem);
541 size_t availlen = 0;
542
543 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX)
544 availlen = sizeof(uint16_t);
545 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
546 vq->vq_usedoffset, hdrlen + payloadlen + availlen, ops);
547 }
548
549 static inline void
vq_sync_uring_header(struct virtio_softc * sc,struct virtqueue * vq,int ops)550 vq_sync_uring_header(struct virtio_softc *sc, struct virtqueue *vq, int ops)
551 {
552 uint16_t hdrlen = offsetof(struct vring_used, ring);
553
554 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
555 vq->vq_usedoffset, hdrlen, ops);
556 }
557
558 static inline void
vq_sync_uring_payload(struct virtio_softc * sc,struct virtqueue * vq,int ops)559 vq_sync_uring_payload(struct virtio_softc *sc, struct virtqueue *vq, int ops)
560 {
561 uint16_t hdrlen = offsetof(struct vring_used, ring);
562 size_t payloadlen = vq->vq_num * sizeof(struct vring_used_elem);
563
564 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
565 vq->vq_usedoffset + hdrlen, payloadlen, ops);
566 }
567
568 static inline void
vq_sync_uring_avail(struct virtio_softc * sc,struct virtqueue * vq,int ops)569 vq_sync_uring_avail(struct virtio_softc *sc, struct virtqueue *vq, int ops)
570 {
571 uint16_t hdrlen = offsetof(struct vring_used, ring);
572 size_t payloadlen = vq->vq_num * sizeof(struct vring_used_elem);
573 size_t availlen = sizeof(uint16_t);
574
575 if ((sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) == 0)
576 return;
577 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
578 vq->vq_usedoffset + hdrlen + payloadlen, availlen, ops);
579 }
580
581 static inline void
vq_sync_indirect(struct virtio_softc * sc,struct virtqueue * vq,int slot,int ops)582 vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot,
583 int ops)
584 {
585 int offset = vq->vq_indirectoffset +
586 sizeof(struct vring_desc) * vq->vq_maxnsegs * slot;
587
588 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap,
589 offset, sizeof(struct vring_desc) * vq->vq_maxnsegs, ops);
590 }
591
592 bool
virtio_vq_is_enqueued(struct virtio_softc * sc,struct virtqueue * vq)593 virtio_vq_is_enqueued(struct virtio_softc *sc, struct virtqueue *vq)
594 {
595
596 if (vq->vq_queued) {
597 vq->vq_queued = 0;
598 vq_sync_aring_all(sc, vq, BUS_DMASYNC_POSTWRITE);
599 }
600
601 vq_sync_uring_header(sc, vq, BUS_DMASYNC_POSTREAD);
602 if (vq->vq_used_idx == virtio_rw16(sc, vq->vq_used->idx))
603 return 0;
604 vq_sync_uring_payload(sc, vq, BUS_DMASYNC_POSTREAD);
605 return 1;
606 }
607
608 /*
609 * Increase the event index in order to delay interrupts.
610 */
611 int
virtio_postpone_intr(struct virtio_softc * sc,struct virtqueue * vq,uint16_t nslots)612 virtio_postpone_intr(struct virtio_softc *sc, struct virtqueue *vq,
613 uint16_t nslots)
614 {
615 uint16_t idx, nused;
616
617 idx = vq->vq_used_idx + nslots;
618
619 /* set the new event index: avail_ring->used_event = idx */
620 *vq->vq_used_event = virtio_rw16(sc, idx);
621 vq_sync_aring_used(vq->vq_owner, vq, BUS_DMASYNC_PREWRITE);
622 vq->vq_queued++;
623
624 nused = (uint16_t)
625 (virtio_rw16(sc, vq->vq_used->idx) - vq->vq_used_idx);
626 KASSERT(nused <= vq->vq_num);
627
628 return nslots < nused;
629 }
630
631 /*
632 * Postpone interrupt until 3/4 of the available descriptors have been
633 * consumed.
634 */
635 int
virtio_postpone_intr_smart(struct virtio_softc * sc,struct virtqueue * vq)636 virtio_postpone_intr_smart(struct virtio_softc *sc, struct virtqueue *vq)
637 {
638 uint16_t nslots;
639
640 nslots = (uint16_t)
641 (virtio_rw16(sc, vq->vq_avail->idx) - vq->vq_used_idx) * 3 / 4;
642
643 return virtio_postpone_intr(sc, vq, nslots);
644 }
645
646 /*
647 * Postpone interrupt until all of the available descriptors have been
648 * consumed.
649 */
650 int
virtio_postpone_intr_far(struct virtio_softc * sc,struct virtqueue * vq)651 virtio_postpone_intr_far(struct virtio_softc *sc, struct virtqueue *vq)
652 {
653 uint16_t nslots;
654
655 nslots = (uint16_t)
656 (virtio_rw16(sc, vq->vq_avail->idx) - vq->vq_used_idx);
657
658 return virtio_postpone_intr(sc, vq, nslots);
659 }
660
661 /*
662 * Start/stop vq interrupt. No guarantee.
663 */
664 void
virtio_stop_vq_intr(struct virtio_softc * sc,struct virtqueue * vq)665 virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq)
666 {
667
668 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) {
669 /*
670 * No way to disable the interrupt completely with
671 * RingEventIdx. Instead advance used_event by half the
672 * possible value. This won't happen soon and is far enough in
673 * the past to not trigger a spurious interrupt.
674 */
675 *vq->vq_used_event = virtio_rw16(sc, vq->vq_used_idx + 0x8000);
676 vq_sync_aring_used(sc, vq, BUS_DMASYNC_PREWRITE);
677 } else {
678 vq->vq_avail->flags |=
679 virtio_rw16(sc, VRING_AVAIL_F_NO_INTERRUPT);
680 vq_sync_aring_header(sc, vq, BUS_DMASYNC_PREWRITE);
681 }
682 vq->vq_queued++;
683 }
684
685 int
virtio_start_vq_intr(struct virtio_softc * sc,struct virtqueue * vq)686 virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq)
687 {
688
689 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) {
690 /*
691 * If event index feature is negotiated, enabling interrupts
692 * is done through setting the latest consumed index in the
693 * used_event field
694 */
695 *vq->vq_used_event = virtio_rw16(sc, vq->vq_used_idx);
696 vq_sync_aring_used(sc, vq, BUS_DMASYNC_PREWRITE);
697 } else {
698 vq->vq_avail->flags &=
699 ~virtio_rw16(sc, VRING_AVAIL_F_NO_INTERRUPT);
700 vq_sync_aring_header(sc, vq, BUS_DMASYNC_PREWRITE);
701 }
702 vq->vq_queued++;
703
704 vq_sync_uring_header(sc, vq, BUS_DMASYNC_POSTREAD);
705 if (vq->vq_used_idx == virtio_rw16(sc, vq->vq_used->idx))
706 return 0;
707 vq_sync_uring_payload(sc, vq, BUS_DMASYNC_POSTREAD);
708 return 1;
709 }
710
711 /*
712 * Initialize vq structure.
713 */
714 /*
715 * Reset virtqueue parameters
716 */
717 static void
virtio_reset_vq(struct virtio_softc * sc,struct virtqueue * vq)718 virtio_reset_vq(struct virtio_softc *sc, struct virtqueue *vq)
719 {
720 struct vring_desc *vds;
721 int i, j;
722 int vq_size = vq->vq_num;
723
724 memset(vq->vq_vaddr, 0, vq->vq_bytesize);
725
726 /* build the descriptor chain for free slot management */
727 vds = vq->vq_desc;
728 for (i = 0; i < vq_size - 1; i++) {
729 vds[i].next = virtio_rw16(sc, i + 1);
730 }
731 vds[i].next = virtio_rw16(sc, VRING_DESC_CHAIN_END);
732 vq->vq_free_idx = 0;
733
734 /* build the indirect descriptor chain */
735 if (vq->vq_indirect != NULL) {
736 struct vring_desc *vd;
737
738 for (i = 0; i < vq_size; i++) {
739 vd = vq->vq_indirect;
740 vd += vq->vq_maxnsegs * i;
741 for (j = 0; j < vq->vq_maxnsegs - 1; j++) {
742 vd[j].next = virtio_rw16(sc, j + 1);
743 }
744 }
745 }
746
747 /* enqueue/dequeue status */
748 vq->vq_avail_idx = 0;
749 vq->vq_used_idx = 0;
750 vq->vq_queued = 0;
751 vq_sync_uring_all(sc, vq, BUS_DMASYNC_PREREAD);
752 vq->vq_queued++;
753 }
754
755 /* Initialize vq */
756 void
virtio_init_vq_vqdone(struct virtio_softc * sc,struct virtqueue * vq,int index,int (* vq_done)(struct virtqueue *))757 virtio_init_vq_vqdone(struct virtio_softc *sc, struct virtqueue *vq,
758 int index, int (*vq_done)(struct virtqueue *))
759 {
760
761 virtio_init_vq(sc, vq, index, virtio_vq_done, vq);
762 vq->vq_done = vq_done;
763 }
764
765 void
virtio_init_vq(struct virtio_softc * sc,struct virtqueue * vq,int index,int (* func)(void *),void * arg)766 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, int index,
767 int (*func)(void *), void *arg)
768 {
769
770 memset(vq, 0, sizeof(*vq));
771
772 vq->vq_owner = sc;
773 vq->vq_num = sc->sc_ops->read_queue_size(sc, index);
774 vq->vq_index = index;
775 vq->vq_intrhand = func;
776 vq->vq_intrhand_arg = arg;
777 }
778
779 /*
780 * Allocate/free a vq.
781 */
782 int
virtio_alloc_vq(struct virtio_softc * sc,struct virtqueue * vq,int maxsegsize,int maxnsegs,const char * name)783 virtio_alloc_vq(struct virtio_softc *sc, struct virtqueue *vq,
784 int maxsegsize, int maxnsegs, const char *name)
785 {
786 bus_size_t size_desc, size_avail, size_used, size_indirect;
787 bus_size_t allocsize = 0, size_desc_avail;
788 int rsegs, r, hdrlen;
789 unsigned int vq_num;
790 #define VIRTQUEUE_ALIGN(n) roundup(n, VIRTIO_PAGE_SIZE)
791
792 vq_num = vq->vq_num;
793
794 if (vq_num == 0) {
795 aprint_error_dev(sc->sc_dev,
796 "virtqueue not exist, index %d for %s\n",
797 vq->vq_index, name);
798 goto err;
799 }
800
801 hdrlen = sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX ? 3 : 2;
802
803 size_desc = sizeof(vq->vq_desc[0]) * vq_num;
804 size_avail = sizeof(uint16_t) * hdrlen
805 + sizeof(vq->vq_avail[0].ring[0]) * vq_num;
806 size_used = sizeof(uint16_t) *hdrlen
807 + sizeof(vq->vq_used[0].ring[0]) * vq_num;
808 size_indirect = (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT) ?
809 sizeof(struct vring_desc) * maxnsegs * vq_num : 0;
810
811 size_desc_avail = VIRTQUEUE_ALIGN(size_desc + size_avail);
812 size_used = VIRTQUEUE_ALIGN(size_used);
813
814 allocsize = size_desc_avail + size_used + size_indirect;
815
816 /* alloc and map the memory */
817 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0,
818 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_WAITOK);
819 if (r != 0) {
820 aprint_error_dev(sc->sc_dev,
821 "virtqueue %d for %s allocation failed, "
822 "error code %d\n", vq->vq_index, name, r);
823 goto err;
824 }
825
826 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], rsegs, allocsize,
827 &vq->vq_vaddr, BUS_DMA_WAITOK);
828 if (r != 0) {
829 aprint_error_dev(sc->sc_dev,
830 "virtqueue %d for %s map failed, "
831 "error code %d\n", vq->vq_index, name, r);
832 goto err;
833 }
834
835 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0,
836 BUS_DMA_WAITOK, &vq->vq_dmamap);
837 if (r != 0) {
838 aprint_error_dev(sc->sc_dev,
839 "virtqueue %d for %s dmamap creation failed, "
840 "error code %d\n", vq->vq_index, name, r);
841 goto err;
842 }
843
844 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap,
845 vq->vq_vaddr, allocsize, NULL, BUS_DMA_WAITOK);
846 if (r != 0) {
847 aprint_error_dev(sc->sc_dev,
848 "virtqueue %d for %s dmamap load failed, "
849 "error code %d\n", vq->vq_index, name, r);
850 goto err;
851 }
852
853 vq->vq_bytesize = allocsize;
854 vq->vq_maxsegsize = maxsegsize;
855 vq->vq_maxnsegs = maxnsegs;
856
857 #define VIRTIO_PTR(base, offset) (void *)((intptr_t)(base) + (offset))
858 /* initialize vring pointers */
859 vq->vq_desc = VIRTIO_PTR(vq->vq_vaddr, 0);
860 vq->vq_availoffset = size_desc;
861 vq->vq_avail = VIRTIO_PTR(vq->vq_vaddr, vq->vq_availoffset);
862 vq->vq_used_event = VIRTIO_PTR(vq->vq_avail,
863 offsetof(struct vring_avail, ring[vq_num]));
864 vq->vq_usedoffset = size_desc_avail;
865 vq->vq_used = VIRTIO_PTR(vq->vq_vaddr, vq->vq_usedoffset);
866 vq->vq_avail_event = VIRTIO_PTR(vq->vq_used,
867 offsetof(struct vring_used, ring[vq_num]));
868
869 if (size_indirect > 0) {
870 vq->vq_indirectoffset = size_desc_avail + size_used;
871 vq->vq_indirect = VIRTIO_PTR(vq->vq_vaddr,
872 vq->vq_indirectoffset);
873 }
874 #undef VIRTIO_PTR
875
876 vq->vq_descx = kmem_zalloc(sizeof(vq->vq_descx[0]) * vq_num,
877 KM_SLEEP);
878
879 mutex_init(&vq->vq_freedesc_lock, MUTEX_SPIN, sc->sc_ipl);
880 mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl);
881 mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl);
882
883 virtio_reset_vq(sc, vq);
884
885 aprint_verbose_dev(sc->sc_dev,
886 "allocated %" PRIuBUSSIZE " byte for virtqueue %d for %s, "
887 "size %d\n", allocsize, vq->vq_index, name, vq_num);
888 if (size_indirect > 0)
889 aprint_verbose_dev(sc->sc_dev,
890 "using %" PRIuBUSSIZE " byte (%d entries) indirect "
891 "descriptors\n", size_indirect, maxnsegs * vq_num);
892
893 return 0;
894
895 err:
896 sc->sc_ops->setup_queue(sc, vq->vq_index, 0);
897 if (vq->vq_dmamap)
898 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap);
899 if (vq->vq_vaddr)
900 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize);
901 if (vq->vq_segs[0].ds_addr)
902 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1);
903 memset(vq, 0, sizeof(*vq));
904
905 return -1;
906 }
907
908 int
virtio_free_vq(struct virtio_softc * sc,struct virtqueue * vq)909 virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq)
910 {
911 uint16_t s;
912 size_t i;
913
914 if (vq->vq_vaddr == NULL)
915 return 0;
916
917 /* device must be already deactivated */
918 /* confirm the vq is empty */
919 s = vq->vq_free_idx;
920 i = 0;
921 while (s != virtio_rw16(sc, VRING_DESC_CHAIN_END)) {
922 s = vq->vq_desc[s].next;
923 i++;
924 }
925 if (i != vq->vq_num) {
926 printf("%s: freeing non-empty vq, index %d\n",
927 device_xname(sc->sc_dev), vq->vq_index);
928 return EBUSY;
929 }
930
931 /* tell device that there's no virtqueue any longer */
932 sc->sc_ops->setup_queue(sc, vq->vq_index, 0);
933
934 vq_sync_aring_all(sc, vq, BUS_DMASYNC_POSTWRITE);
935
936 kmem_free(vq->vq_descx, sizeof(vq->vq_descx[0]) * vq->vq_num);
937 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap);
938 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap);
939 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize);
940 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1);
941 mutex_destroy(&vq->vq_freedesc_lock);
942 mutex_destroy(&vq->vq_uring_lock);
943 mutex_destroy(&vq->vq_aring_lock);
944 memset(vq, 0, sizeof(*vq));
945
946 return 0;
947 }
948
949 /*
950 * Free descriptor management.
951 */
952 static int
vq_alloc_slot_locked(struct virtio_softc * sc,struct virtqueue * vq,size_t nslots)953 vq_alloc_slot_locked(struct virtio_softc *sc, struct virtqueue *vq,
954 size_t nslots)
955 {
956 struct vring_desc *vd;
957 uint16_t head, tail;
958 size_t i;
959
960 KASSERT(mutex_owned(&vq->vq_freedesc_lock));
961
962 head = tail = virtio_rw16(sc, vq->vq_free_idx);
963 for (i = 0; i < nslots - 1; i++) {
964 if (tail == VRING_DESC_CHAIN_END)
965 return VRING_DESC_CHAIN_END;
966
967 vd = &vq->vq_desc[tail];
968 vd->flags = virtio_rw16(sc, VRING_DESC_F_NEXT);
969 tail = virtio_rw16(sc, vd->next);
970 }
971
972 if (tail == VRING_DESC_CHAIN_END)
973 return VRING_DESC_CHAIN_END;
974
975 vd = &vq->vq_desc[tail];
976 vd->flags = virtio_rw16(sc, 0);
977 vq->vq_free_idx = vd->next;
978
979 return head;
980 }
981 static uint16_t
vq_alloc_slot(struct virtio_softc * sc,struct virtqueue * vq,size_t nslots)982 vq_alloc_slot(struct virtio_softc *sc, struct virtqueue *vq, size_t nslots)
983 {
984 uint16_t rv;
985
986 mutex_enter(&vq->vq_freedesc_lock);
987 rv = vq_alloc_slot_locked(sc, vq, nslots);
988 mutex_exit(&vq->vq_freedesc_lock);
989
990 return rv;
991 }
992
993 static void
vq_free_slot(struct virtio_softc * sc,struct virtqueue * vq,uint16_t slot)994 vq_free_slot(struct virtio_softc *sc, struct virtqueue *vq, uint16_t slot)
995 {
996 struct vring_desc *vd;
997 uint16_t s;
998
999 mutex_enter(&vq->vq_freedesc_lock);
1000 vd = &vq->vq_desc[slot];
1001 while ((vd->flags & virtio_rw16(sc, VRING_DESC_F_NEXT)) != 0) {
1002 s = virtio_rw16(sc, vd->next);
1003 vd = &vq->vq_desc[s];
1004 }
1005 vd->next = vq->vq_free_idx;
1006 vq->vq_free_idx = virtio_rw16(sc, slot);
1007 mutex_exit(&vq->vq_freedesc_lock);
1008 }
1009
1010 /*
1011 * Enqueue several dmamaps as a single request.
1012 */
1013 /*
1014 * Typical usage:
1015 * <queue size> number of followings are stored in arrays
1016 * - command blocks (in dmamem) should be pre-allocated and mapped
1017 * - dmamaps for command blocks should be pre-allocated and loaded
1018 * - dmamaps for payload should be pre-allocated
1019 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot
1020 * if (r) // currently 0 or EAGAIN
1021 * return r;
1022 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..);
1023 * if (r) {
1024 * virtio_enqueue_abort(sc, vq, slot);
1025 * return r;
1026 * }
1027 * r = virtio_enqueue_reserve(sc, vq, slot,
1028 * dmamap_payload[slot]->dm_nsegs + 1);
1029 * // ^ +1 for command
1030 * if (r) { // currently 0 or EAGAIN
1031 * bus_dmamap_unload(dmat, dmamap_payload[slot]);
1032 * return r; // do not call abort()
1033 * }
1034 * <setup and prepare commands>
1035 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE);
1036 * bus_dmamap_sync(dmat, dmamap_payload[slot],...);
1037 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false);
1038 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite);
1039 * virtio_enqueue_commit(sc, vq, slot, true);
1040 */
1041
1042 /*
1043 * enqueue_prep: allocate a slot number
1044 */
1045 int
virtio_enqueue_prep(struct virtio_softc * sc,struct virtqueue * vq,int * slotp)1046 virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp)
1047 {
1048 uint16_t slot;
1049
1050 KASSERT(sc->sc_child_state == VIRTIO_CHILD_ATTACH_FINISHED);
1051 KASSERT(slotp != NULL);
1052
1053 slot = vq_alloc_slot(sc, vq, 1);
1054 if (slot == VRING_DESC_CHAIN_END)
1055 return EAGAIN;
1056
1057 *slotp = slot;
1058
1059 return 0;
1060 }
1061
1062 /*
1063 * enqueue_reserve: allocate remaining slots and build the descriptor chain.
1064 */
1065 int
virtio_enqueue_reserve(struct virtio_softc * sc,struct virtqueue * vq,int slot,int nsegs)1066 virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq,
1067 int slot, int nsegs)
1068 {
1069 struct vring_desc *vd;
1070 struct vring_desc_extra *vdx;
1071 int i;
1072
1073 KASSERT(1 <= nsegs);
1074 KASSERT(nsegs <= vq->vq_num);
1075
1076 vdx = &vq->vq_descx[slot];
1077 vd = &vq->vq_desc[slot];
1078
1079 KASSERT((vd->flags & virtio_rw16(sc, VRING_DESC_F_NEXT)) == 0);
1080
1081 if ((vq->vq_indirect != NULL) &&
1082 (nsegs >= MINSEG_INDIRECT) &&
1083 (nsegs <= vq->vq_maxnsegs))
1084 vdx->use_indirect = true;
1085 else
1086 vdx->use_indirect = false;
1087
1088 if (vdx->use_indirect) {
1089 uint64_t addr;
1090
1091 addr = vq->vq_dmamap->dm_segs[0].ds_addr
1092 + vq->vq_indirectoffset;
1093 addr += sizeof(struct vring_desc)
1094 * vq->vq_maxnsegs * slot;
1095
1096 vd->addr = virtio_rw64(sc, addr);
1097 vd->len = virtio_rw32(sc, sizeof(struct vring_desc) * nsegs);
1098 vd->flags = virtio_rw16(sc, VRING_DESC_F_INDIRECT);
1099
1100 vd = &vq->vq_indirect[vq->vq_maxnsegs * slot];
1101 vdx->desc_base = vd;
1102 vdx->desc_free_idx = 0;
1103
1104 for (i = 0; i < nsegs - 1; i++) {
1105 vd[i].flags = virtio_rw16(sc, VRING_DESC_F_NEXT);
1106 }
1107 vd[i].flags = virtio_rw16(sc, 0);
1108 } else {
1109 if (nsegs > 1) {
1110 uint16_t s;
1111
1112 s = vq_alloc_slot(sc, vq, nsegs - 1);
1113 if (s == VRING_DESC_CHAIN_END) {
1114 vq_free_slot(sc, vq, slot);
1115 return EAGAIN;
1116 }
1117 vd->next = virtio_rw16(sc, s);
1118 vd->flags = virtio_rw16(sc, VRING_DESC_F_NEXT);
1119 }
1120
1121 vdx->desc_base = &vq->vq_desc[0];
1122 vdx->desc_free_idx = slot;
1123 }
1124
1125 return 0;
1126 }
1127
1128 /*
1129 * enqueue: enqueue a single dmamap.
1130 */
1131 int
virtio_enqueue(struct virtio_softc * sc,struct virtqueue * vq,int slot,bus_dmamap_t dmamap,bool write)1132 virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot,
1133 bus_dmamap_t dmamap, bool write)
1134 {
1135 struct vring_desc *vds;
1136 struct vring_desc_extra *vdx;
1137 uint16_t s;
1138 int i;
1139
1140 KASSERT(dmamap->dm_nsegs > 0);
1141
1142 vdx = &vq->vq_descx[slot];
1143 vds = vdx->desc_base;
1144 s = vdx->desc_free_idx;
1145
1146 KASSERT(vds != NULL);
1147
1148 for (i = 0; i < dmamap->dm_nsegs; i++) {
1149 KASSERT(s != VRING_DESC_CHAIN_END);
1150
1151 vds[s].addr = virtio_rw64(sc, dmamap->dm_segs[i].ds_addr);
1152 vds[s].len = virtio_rw32(sc, dmamap->dm_segs[i].ds_len);
1153 if (!write)
1154 vds[s].flags |= virtio_rw16(sc, VRING_DESC_F_WRITE);
1155
1156 if ((vds[s].flags & virtio_rw16(sc, VRING_DESC_F_NEXT)) == 0) {
1157 s = VRING_DESC_CHAIN_END;
1158 } else {
1159 s = virtio_rw16(sc, vds[s].next);
1160 }
1161 }
1162
1163 vdx->desc_free_idx = s;
1164
1165 return 0;
1166 }
1167
1168 int
virtio_enqueue_p(struct virtio_softc * sc,struct virtqueue * vq,int slot,bus_dmamap_t dmamap,bus_addr_t start,bus_size_t len,bool write)1169 virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot,
1170 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len,
1171 bool write)
1172 {
1173 struct vring_desc_extra *vdx;
1174 struct vring_desc *vds;
1175 uint16_t s;
1176
1177 vdx = &vq->vq_descx[slot];
1178 vds = vdx->desc_base;
1179 s = vdx->desc_free_idx;
1180
1181 KASSERT(s != VRING_DESC_CHAIN_END);
1182 KASSERT(vds != NULL);
1183 KASSERT(dmamap->dm_nsegs == 1); /* XXX */
1184 KASSERT(dmamap->dm_segs[0].ds_len > start);
1185 KASSERT(dmamap->dm_segs[0].ds_len >= start + len);
1186
1187 vds[s].addr = virtio_rw64(sc, dmamap->dm_segs[0].ds_addr + start);
1188 vds[s].len = virtio_rw32(sc, len);
1189 if (!write)
1190 vds[s].flags |= virtio_rw16(sc, VRING_DESC_F_WRITE);
1191
1192 if ((vds[s].flags & virtio_rw16(sc, VRING_DESC_F_NEXT)) == 0) {
1193 s = VRING_DESC_CHAIN_END;
1194 } else {
1195 s = virtio_rw16(sc, vds[s].next);
1196 }
1197
1198 vdx->desc_free_idx = s;
1199
1200 return 0;
1201 }
1202
1203 /*
1204 * enqueue_commit: add it to the aring.
1205 */
1206 int
virtio_enqueue_commit(struct virtio_softc * sc,struct virtqueue * vq,int slot,bool notifynow)1207 virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot,
1208 bool notifynow)
1209 {
1210
1211 if (slot < 0) {
1212 mutex_enter(&vq->vq_aring_lock);
1213 goto notify;
1214 }
1215
1216 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE);
1217 if (vq->vq_descx[slot].use_indirect)
1218 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE);
1219
1220 mutex_enter(&vq->vq_aring_lock);
1221 vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] =
1222 virtio_rw16(sc, slot);
1223
1224 notify:
1225 if (notifynow) {
1226 uint16_t o, n, t;
1227 uint16_t flags;
1228
1229 o = virtio_rw16(sc, vq->vq_avail->idx) - 1;
1230 n = vq->vq_avail_idx;
1231
1232 /*
1233 * Prepare for `device->CPU' (host->guest) transfer
1234 * into the buffer. This must happen before we commit
1235 * the vq->vq_avail->idx update to ensure we're not
1236 * still using the buffer in case program-prior loads
1237 * or stores in it get delayed past the store to
1238 * vq->vq_avail->idx.
1239 */
1240 vq_sync_uring_all(sc, vq, BUS_DMASYNC_PREREAD);
1241
1242 /* ensure payload is published, then avail idx */
1243 vq_sync_aring_payload(sc, vq, BUS_DMASYNC_PREWRITE);
1244 vq->vq_avail->idx = virtio_rw16(sc, vq->vq_avail_idx);
1245 vq_sync_aring_header(sc, vq, BUS_DMASYNC_PREWRITE);
1246 vq->vq_queued++;
1247
1248 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) {
1249 vq_sync_uring_avail(sc, vq, BUS_DMASYNC_POSTREAD);
1250 t = virtio_rw16(sc, *vq->vq_avail_event) + 1;
1251 if ((uint16_t) (n - t) < (uint16_t) (n - o))
1252 sc->sc_ops->kick(sc, vq->vq_index);
1253 } else {
1254 vq_sync_uring_header(sc, vq, BUS_DMASYNC_POSTREAD);
1255 flags = virtio_rw16(sc, vq->vq_used->flags);
1256 if (!(flags & VRING_USED_F_NO_NOTIFY))
1257 sc->sc_ops->kick(sc, vq->vq_index);
1258 }
1259 }
1260 mutex_exit(&vq->vq_aring_lock);
1261
1262 return 0;
1263 }
1264
1265 /*
1266 * enqueue_abort: rollback.
1267 */
1268 int
virtio_enqueue_abort(struct virtio_softc * sc,struct virtqueue * vq,int slot)1269 virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot)
1270 {
1271 struct vring_desc_extra *vdx;
1272
1273 vdx = &vq->vq_descx[slot];
1274 vdx->desc_free_idx = VRING_DESC_CHAIN_END;
1275 vdx->desc_base = NULL;
1276
1277 vq_free_slot(sc, vq, slot);
1278
1279 return 0;
1280 }
1281
1282 /*
1283 * Dequeue a request.
1284 */
1285 /*
1286 * dequeue: dequeue a request from uring; dmamap_sync for uring is
1287 * already done in the interrupt handler.
1288 */
1289 int
virtio_dequeue(struct virtio_softc * sc,struct virtqueue * vq,int * slotp,int * lenp)1290 virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq,
1291 int *slotp, int *lenp)
1292 {
1293 uint16_t slot, usedidx;
1294
1295 if (vq->vq_used_idx == virtio_rw16(sc, vq->vq_used->idx))
1296 return ENOENT;
1297 mutex_enter(&vq->vq_uring_lock);
1298 usedidx = vq->vq_used_idx++;
1299 mutex_exit(&vq->vq_uring_lock);
1300 usedidx %= vq->vq_num;
1301 slot = virtio_rw32(sc, vq->vq_used->ring[usedidx].id);
1302
1303 if (vq->vq_descx[slot].use_indirect)
1304 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE);
1305
1306 if (slotp)
1307 *slotp = slot;
1308 if (lenp)
1309 *lenp = virtio_rw32(sc, vq->vq_used->ring[usedidx].len);
1310
1311 return 0;
1312 }
1313
1314 /*
1315 * dequeue_commit: complete dequeue; the slot is recycled for future use.
1316 * if you forget to call this the slot will be leaked.
1317 */
1318 int
virtio_dequeue_commit(struct virtio_softc * sc,struct virtqueue * vq,int slot)1319 virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot)
1320 {
1321 struct vring_desc_extra *vdx;
1322
1323 vdx = &vq->vq_descx[slot];
1324 vdx->desc_base = NULL;
1325 vdx->desc_free_idx = VRING_DESC_CHAIN_END;
1326
1327 vq_free_slot(sc, vq, slot);
1328
1329 return 0;
1330 }
1331
1332 /*
1333 * Attach a child, fill all the members.
1334 */
1335 void
virtio_child_attach_start(struct virtio_softc * sc,device_t child,int ipl,uint64_t req_features,const char * feat_bits)1336 virtio_child_attach_start(struct virtio_softc *sc, device_t child, int ipl,
1337 uint64_t req_features, const char *feat_bits)
1338 {
1339 char buf[1024];
1340
1341 KASSERT(sc->sc_child == NULL);
1342 KASSERT(sc->sc_child_state == VIRTIO_NO_CHILD);
1343
1344 sc->sc_child = child;
1345 sc->sc_ipl = ipl;
1346
1347 virtio_negotiate_features(sc, req_features);
1348 snprintb(buf, sizeof(buf), feat_bits, sc->sc_active_features);
1349 aprint_normal(": features: %s\n", buf);
1350 aprint_naive("\n");
1351 }
1352
1353 int
virtio_child_attach_finish(struct virtio_softc * sc,struct virtqueue * vqs,size_t nvqs,virtio_callback config_change,int req_flags)1354 virtio_child_attach_finish(struct virtio_softc *sc,
1355 struct virtqueue *vqs, size_t nvqs,
1356 virtio_callback config_change,
1357 int req_flags)
1358 {
1359 size_t i;
1360 int r;
1361
1362 #ifdef DIAGNOSTIC
1363 KASSERT(nvqs > 0);
1364 #define VIRTIO_ASSERT_FLAGS (VIRTIO_F_INTR_SOFTINT | VIRTIO_F_INTR_PERVQ)
1365 KASSERT((req_flags & VIRTIO_ASSERT_FLAGS) != VIRTIO_ASSERT_FLAGS);
1366 #undef VIRTIO_ASSERT_FLAGS
1367
1368 for (i = 0; i < nvqs; i++){
1369 KASSERT(vqs[i].vq_index == i);
1370 KASSERT(vqs[i].vq_intrhand != NULL);
1371 KASSERT(vqs[i].vq_done == NULL ||
1372 vqs[i].vq_intrhand == virtio_vq_done);
1373 }
1374 #endif
1375
1376
1377 sc->sc_vqs = vqs;
1378 sc->sc_nvqs = nvqs;
1379 sc->sc_config_change = config_change;
1380 sc->sc_intrhand = virtio_vq_intr;
1381 sc->sc_flags = req_flags;
1382
1383 /* set the vq address */
1384 for (i = 0; i < nvqs; i++) {
1385 sc->sc_ops->setup_queue(sc, vqs[i].vq_index,
1386 vqs[i].vq_dmamap->dm_segs[0].ds_addr);
1387 }
1388
1389 r = sc->sc_ops->alloc_interrupts(sc);
1390 if (r != 0) {
1391 aprint_error_dev(sc->sc_dev,
1392 "failed to allocate interrupts\n");
1393 goto fail;
1394 }
1395
1396 r = sc->sc_ops->setup_interrupts(sc, 0);
1397 if (r != 0) {
1398 aprint_error_dev(sc->sc_dev, "failed to setup interrupts\n");
1399 goto fail;
1400 }
1401
1402 KASSERT(sc->sc_soft_ih == NULL);
1403 if (sc->sc_flags & VIRTIO_F_INTR_SOFTINT) {
1404 u_int flags = SOFTINT_NET;
1405 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE)
1406 flags |= SOFTINT_MPSAFE;
1407
1408 sc->sc_soft_ih = softint_establish(flags, virtio_soft_intr,
1409 sc);
1410 if (sc->sc_soft_ih == NULL) {
1411 sc->sc_ops->free_interrupts(sc);
1412 aprint_error_dev(sc->sc_dev,
1413 "failed to establish soft interrupt\n");
1414 goto fail;
1415 }
1416 }
1417
1418 sc->sc_child_state = VIRTIO_CHILD_ATTACH_FINISHED;
1419 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
1420 return 0;
1421
1422 fail:
1423 if (sc->sc_soft_ih) {
1424 softint_disestablish(sc->sc_soft_ih);
1425 sc->sc_soft_ih = NULL;
1426 }
1427
1428 sc->sc_ops->free_interrupts(sc);
1429
1430 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
1431 return 1;
1432 }
1433
1434 void
virtio_child_detach(struct virtio_softc * sc)1435 virtio_child_detach(struct virtio_softc *sc)
1436 {
1437
1438 /* already detached */
1439 if (sc->sc_child == NULL)
1440 return;
1441
1442
1443 virtio_device_reset(sc);
1444
1445 sc->sc_ops->free_interrupts(sc);
1446
1447 if (sc->sc_soft_ih) {
1448 softint_disestablish(sc->sc_soft_ih);
1449 sc->sc_soft_ih = NULL;
1450 }
1451
1452 sc->sc_vqs = NULL;
1453 sc->sc_child = NULL;
1454 }
1455
1456 void
virtio_child_attach_failed(struct virtio_softc * sc)1457 virtio_child_attach_failed(struct virtio_softc *sc)
1458 {
1459 virtio_child_detach(sc);
1460
1461 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
1462
1463 sc->sc_child_state = VIRTIO_CHILD_ATTACH_FAILED;
1464 }
1465
1466 bus_dma_tag_t
virtio_dmat(struct virtio_softc * sc)1467 virtio_dmat(struct virtio_softc *sc)
1468 {
1469 return sc->sc_dmat;
1470 }
1471
1472 device_t
virtio_child(struct virtio_softc * sc)1473 virtio_child(struct virtio_softc *sc)
1474 {
1475 return sc->sc_child;
1476 }
1477
1478 int
virtio_intrhand(struct virtio_softc * sc)1479 virtio_intrhand(struct virtio_softc *sc)
1480 {
1481 return (*sc->sc_intrhand)(sc);
1482 }
1483
1484 uint64_t
virtio_features(struct virtio_softc * sc)1485 virtio_features(struct virtio_softc *sc)
1486 {
1487 return sc->sc_active_features;
1488 }
1489
1490 int
virtio_attach_failed(struct virtio_softc * sc)1491 virtio_attach_failed(struct virtio_softc *sc)
1492 {
1493 device_t self = sc->sc_dev;
1494
1495 /* no error if its not connected, but its failed */
1496 if (sc->sc_childdevid == 0)
1497 return 1;
1498
1499 if (sc->sc_child == NULL) {
1500 switch (sc->sc_child_state) {
1501 case VIRTIO_CHILD_ATTACH_FAILED:
1502 aprint_error_dev(self,
1503 "virtio configuration failed\n");
1504 break;
1505 case VIRTIO_NO_CHILD:
1506 aprint_error_dev(self,
1507 "no matching child driver; not configured\n");
1508 break;
1509 default:
1510 /* sanity check */
1511 aprint_error_dev(self,
1512 "virtio internal error, "
1513 "child driver is not configured\n");
1514 break;
1515 }
1516
1517 return 1;
1518 }
1519
1520 /* sanity check */
1521 if (sc->sc_child_state != VIRTIO_CHILD_ATTACH_FINISHED) {
1522 aprint_error_dev(self, "virtio internal error, child driver "
1523 "signaled OK but didn't initialize interrupts\n");
1524 return 1;
1525 }
1526
1527 return 0;
1528 }
1529
1530 void
virtio_print_device_type(device_t self,int id,int revision)1531 virtio_print_device_type(device_t self, int id, int revision)
1532 {
1533 aprint_normal_dev(self, "%s device (id %d, rev. 0x%02x)\n",
1534 (id < NDEVNAMES ? virtio_device_name[id] : "Unknown"),
1535 id,
1536 revision);
1537 }
1538
1539
1540 MODULE(MODULE_CLASS_DRIVER, virtio, NULL);
1541
1542 #ifdef _MODULE
1543 #include "ioconf.c"
1544 #endif
1545
1546 static int
virtio_modcmd(modcmd_t cmd,void * opaque)1547 virtio_modcmd(modcmd_t cmd, void *opaque)
1548 {
1549 int error = 0;
1550
1551 #ifdef _MODULE
1552 switch (cmd) {
1553 case MODULE_CMD_INIT:
1554 error = config_init_component(cfdriver_ioconf_virtio,
1555 cfattach_ioconf_virtio, cfdata_ioconf_virtio);
1556 break;
1557 case MODULE_CMD_FINI:
1558 error = config_fini_component(cfdriver_ioconf_virtio,
1559 cfattach_ioconf_virtio, cfdata_ioconf_virtio);
1560 break;
1561 default:
1562 error = ENOTTY;
1563 break;
1564 }
1565 #endif
1566
1567 return error;
1568 }
1569