1 /*-
2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com>
3 * Copyright (c) 2014 The FreeBSD Foundation
4 * All rights reserved.
5 *
6 * This software was developed by SRI International and the University of
7 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
8 * ("CTSRD"), as part of the DARPA CRASH research programme.
9 *
10 * Portions of this software were developed by Andrew Turner
11 * under sponsorship from the FreeBSD Foundation.
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35 /*
36 * VirtIO MMIO interface.
37 * This driver is heavily based on VirtIO PCI interface driver.
38 */
39
40 /*
41 * FDT example:
42 * virtio_block@1000 {
43 * compatible = "virtio,mmio";
44 * reg = <0x1000 0x100>;
45 * interrupts = <63>;
46 * interrupt-parent = <&GIC>;
47 * };
48 */
49
50 #include <sys/cdefs.h>
51 __FBSDID("$FreeBSD$");
52
53 #include <sys/param.h>
54 #include <sys/systm.h>
55 #include <sys/bus.h>
56 #include <sys/kernel.h>
57 #include <sys/module.h>
58 #include <sys/malloc.h>
59 #include <sys/rman.h>
60
61 #include <machine/bus.h>
62 #include <machine/resource.h>
63
64 #include <dev/fdt/fdt_common.h>
65 #include <dev/ofw/openfirm.h>
66 #include <dev/ofw/ofw_bus.h>
67 #include <dev/ofw/ofw_bus_subr.h>
68
69 #include <dev/virtio/virtio.h>
70 #include <dev/virtio/virtqueue.h>
71 #include <dev/virtio/mmio/virtio_mmio.h>
72
73 #include "virtio_mmio_if.h"
74 #include "virtio_bus_if.h"
75 #include "virtio_if.h"
76
77 #define PAGE_SHIFT 12
78
79 struct vtmmio_virtqueue {
80 struct virtqueue *vtv_vq;
81 int vtv_no_intr;
82 };
83
84 struct vtmmio_softc {
85 device_t dev;
86 device_t platform;
87 struct resource *res[2];
88
89 uint64_t vtmmio_features;
90 uint32_t vtmmio_flags;
91
92 /* This "bus" will only ever have one child. */
93 device_t vtmmio_child_dev;
94 struct virtio_feature_desc *vtmmio_child_feat_desc;
95
96 int vtmmio_nvqs;
97 struct vtmmio_virtqueue *vtmmio_vqs;
98 void *ih;
99 };
100
101 static int vtmmio_probe(device_t);
102 static int vtmmio_attach(device_t);
103 static int vtmmio_detach(device_t);
104 static int vtmmio_suspend(device_t);
105 static int vtmmio_resume(device_t);
106 static int vtmmio_shutdown(device_t);
107 static void vtmmio_driver_added(device_t, driver_t *);
108 static void vtmmio_child_detached(device_t, device_t);
109 static int vtmmio_read_ivar(device_t, device_t, int, uintptr_t *);
110 static int vtmmio_write_ivar(device_t, device_t, int, uintptr_t);
111 static uint64_t vtmmio_negotiate_features(device_t, uint64_t);
112 static int vtmmio_with_feature(device_t, uint64_t);
113 static int vtmmio_alloc_virtqueues(device_t, int, int,
114 struct vq_alloc_info *);
115 static int vtmmio_setup_intr(device_t, enum intr_type);
116 static void vtmmio_stop(device_t);
117 static void vtmmio_poll(device_t);
118 static int vtmmio_reinit(device_t, uint64_t);
119 static void vtmmio_reinit_complete(device_t);
120 static void vtmmio_notify_virtqueue(device_t, uint16_t);
121 static uint8_t vtmmio_get_status(device_t);
122 static void vtmmio_set_status(device_t, uint8_t);
123 static void vtmmio_read_dev_config(device_t, bus_size_t, void *, int);
124 static void vtmmio_write_dev_config(device_t, bus_size_t, void *, int);
125 static void vtmmio_describe_features(struct vtmmio_softc *, const char *,
126 uint64_t);
127 static void vtmmio_probe_and_attach_child(struct vtmmio_softc *);
128 static int vtmmio_reinit_virtqueue(struct vtmmio_softc *, int);
129 static void vtmmio_free_interrupts(struct vtmmio_softc *);
130 static void vtmmio_free_virtqueues(struct vtmmio_softc *);
131 static void vtmmio_release_child_resources(struct vtmmio_softc *);
132 static void vtmmio_reset(struct vtmmio_softc *);
133 static void vtmmio_select_virtqueue(struct vtmmio_softc *, int);
134 static void vtmmio_vq_intr(void *);
135
136 /*
137 * I/O port read/write wrappers.
138 */
139 #define vtmmio_write_config_1(sc, o, v) \
140 do { \
141 if (sc->platform != NULL) \
142 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \
143 bus_write_1((sc)->res[0], (o), (v)); \
144 if (sc->platform != NULL) \
145 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \
146 } while (0)
147 #define vtmmio_write_config_2(sc, o, v) \
148 do { \
149 if (sc->platform != NULL) \
150 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \
151 bus_write_2((sc)->res[0], (o), (v)); \
152 if (sc->platform != NULL) \
153 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \
154 } while (0)
155 #define vtmmio_write_config_4(sc, o, v) \
156 do { \
157 if (sc->platform != NULL) \
158 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \
159 bus_write_4((sc)->res[0], (o), (v)); \
160 if (sc->platform != NULL) \
161 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \
162 } while (0)
163
164 #define vtmmio_read_config_1(sc, o) \
165 bus_read_1((sc)->res[0], (o))
166 #define vtmmio_read_config_2(sc, o) \
167 bus_read_2((sc)->res[0], (o))
168 #define vtmmio_read_config_4(sc, o) \
169 bus_read_4((sc)->res[0], (o))
170
171 static device_method_t vtmmio_methods[] = {
172 /* Device interface. */
173 DEVMETHOD(device_probe, vtmmio_probe),
174 DEVMETHOD(device_attach, vtmmio_attach),
175 DEVMETHOD(device_detach, vtmmio_detach),
176 DEVMETHOD(device_suspend, vtmmio_suspend),
177 DEVMETHOD(device_resume, vtmmio_resume),
178 DEVMETHOD(device_shutdown, vtmmio_shutdown),
179
180 /* Bus interface. */
181 DEVMETHOD(bus_driver_added, vtmmio_driver_added),
182 DEVMETHOD(bus_child_detached, vtmmio_child_detached),
183 DEVMETHOD(bus_read_ivar, vtmmio_read_ivar),
184 DEVMETHOD(bus_write_ivar, vtmmio_write_ivar),
185
186 /* VirtIO bus interface. */
187 DEVMETHOD(virtio_bus_negotiate_features, vtmmio_negotiate_features),
188 DEVMETHOD(virtio_bus_with_feature, vtmmio_with_feature),
189 DEVMETHOD(virtio_bus_alloc_virtqueues, vtmmio_alloc_virtqueues),
190 DEVMETHOD(virtio_bus_setup_intr, vtmmio_setup_intr),
191 DEVMETHOD(virtio_bus_stop, vtmmio_stop),
192 DEVMETHOD(virtio_bus_poll, vtmmio_poll),
193 DEVMETHOD(virtio_bus_reinit, vtmmio_reinit),
194 DEVMETHOD(virtio_bus_reinit_complete, vtmmio_reinit_complete),
195 DEVMETHOD(virtio_bus_notify_vq, vtmmio_notify_virtqueue),
196 DEVMETHOD(virtio_bus_read_device_config, vtmmio_read_dev_config),
197 DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config),
198
199 DEVMETHOD_END
200 };
201
202 static driver_t vtmmio_driver = {
203 "virtio_mmio",
204 vtmmio_methods,
205 sizeof(struct vtmmio_softc)
206 };
207
208 devclass_t vtmmio_devclass;
209
210 DRIVER_MODULE(virtio_mmio, simplebus, vtmmio_driver, vtmmio_devclass, 0, 0);
211 DRIVER_MODULE(virtio_mmio, ofwbus, vtmmio_driver, vtmmio_devclass, 0, 0);
212 MODULE_VERSION(virtio_mmio, 1);
213 MODULE_DEPEND(virtio_mmio, simplebus, 1, 1, 1);
214 MODULE_DEPEND(virtio_mmio, virtio, 1, 1, 1);
215
216 static int
vtmmio_setup_intr(device_t dev,enum intr_type type)217 vtmmio_setup_intr(device_t dev, enum intr_type type)
218 {
219 struct vtmmio_softc *sc;
220 int rid;
221 int err;
222
223 sc = device_get_softc(dev);
224
225 if (sc->platform != NULL) {
226 err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev,
227 vtmmio_vq_intr, sc);
228 if (err == 0) {
229 /* Okay we have backend-specific interrupts */
230 return (0);
231 }
232 }
233
234 rid = 0;
235 sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
236 RF_ACTIVE);
237 if (!sc->res[1]) {
238 device_printf(dev, "Can't allocate interrupt\n");
239 return (ENXIO);
240 }
241
242 if (bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE,
243 NULL, vtmmio_vq_intr, sc, &sc->ih)) {
244 device_printf(dev, "Can't setup the interrupt\n");
245 return (ENXIO);
246 }
247
248 return (0);
249 }
250
251 static int
vtmmio_probe(device_t dev)252 vtmmio_probe(device_t dev)
253 {
254
255 if (!ofw_bus_status_okay(dev))
256 return (ENXIO);
257
258 if (!ofw_bus_is_compatible(dev, "virtio,mmio"))
259 return (ENXIO);
260
261 device_set_desc(dev, "VirtIO MMIO adapter");
262 return (BUS_PROBE_DEFAULT);
263 }
264
265 static int
vtmmio_setup_platform(struct vtmmio_softc * sc)266 vtmmio_setup_platform(struct vtmmio_softc *sc)
267 {
268 phandle_t platform_node;
269 struct fdt_ic *ic;
270 phandle_t xref;
271 phandle_t node;
272
273 sc->platform = NULL;
274
275 if ((node = ofw_bus_get_node(sc->dev)) == -1)
276 return (ENXIO);
277
278 if (OF_searchencprop(node, "platform", &xref,
279 sizeof(xref)) == -1) {
280 return (ENXIO);
281 }
282
283 platform_node = OF_node_from_xref(xref);
284
285 SLIST_FOREACH(ic, &fdt_ic_list_head, fdt_ics) {
286 if (ic->iph == platform_node) {
287 sc->platform = ic->dev;
288 break;
289 }
290 }
291
292 if (sc->platform == NULL) {
293 /* No platform-specific device. Ignore it. */
294 }
295
296 return (0);
297 }
298
299 static int
vtmmio_attach(device_t dev)300 vtmmio_attach(device_t dev)
301 {
302 struct vtmmio_softc *sc;
303 device_t child;
304 int rid;
305
306 sc = device_get_softc(dev);
307 sc->dev = dev;
308
309 vtmmio_setup_platform(sc);
310
311 rid = 0;
312 sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
313 RF_ACTIVE);
314 if (!sc->res[0]) {
315 device_printf(dev, "Cannot allocate memory window.\n");
316 return (ENXIO);
317 }
318
319 vtmmio_reset(sc);
320
321 /* Tell the host we've noticed this device. */
322 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
323
324 if ((child = device_add_child(dev, NULL, -1)) == NULL) {
325 device_printf(dev, "Cannot create child device.\n");
326 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
327 vtmmio_detach(dev);
328 return (ENOMEM);
329 }
330
331 sc->vtmmio_child_dev = child;
332 vtmmio_probe_and_attach_child(sc);
333
334 return (0);
335 }
336
337 static int
vtmmio_detach(device_t dev)338 vtmmio_detach(device_t dev)
339 {
340 struct vtmmio_softc *sc;
341 device_t child;
342 int error;
343
344 sc = device_get_softc(dev);
345
346 if ((child = sc->vtmmio_child_dev) != NULL) {
347 error = device_delete_child(dev, child);
348 if (error)
349 return (error);
350 sc->vtmmio_child_dev = NULL;
351 }
352
353 vtmmio_reset(sc);
354
355 if (sc->res[0] != NULL) {
356 bus_release_resource(dev, SYS_RES_MEMORY, 0,
357 sc->res[0]);
358 sc->res[0] = NULL;
359 }
360
361 return (0);
362 }
363
364 static int
vtmmio_suspend(device_t dev)365 vtmmio_suspend(device_t dev)
366 {
367
368 return (bus_generic_suspend(dev));
369 }
370
371 static int
vtmmio_resume(device_t dev)372 vtmmio_resume(device_t dev)
373 {
374
375 return (bus_generic_resume(dev));
376 }
377
378 static int
vtmmio_shutdown(device_t dev)379 vtmmio_shutdown(device_t dev)
380 {
381
382 (void) bus_generic_shutdown(dev);
383
384 /* Forcibly stop the host device. */
385 vtmmio_stop(dev);
386
387 return (0);
388 }
389
390 static void
vtmmio_driver_added(device_t dev,driver_t * driver)391 vtmmio_driver_added(device_t dev, driver_t *driver)
392 {
393 struct vtmmio_softc *sc;
394
395 sc = device_get_softc(dev);
396
397 vtmmio_probe_and_attach_child(sc);
398 }
399
400 static void
vtmmio_child_detached(device_t dev,device_t child)401 vtmmio_child_detached(device_t dev, device_t child)
402 {
403 struct vtmmio_softc *sc;
404
405 sc = device_get_softc(dev);
406
407 vtmmio_reset(sc);
408 vtmmio_release_child_resources(sc);
409 }
410
411 static int
vtmmio_read_ivar(device_t dev,device_t child,int index,uintptr_t * result)412 vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
413 {
414 struct vtmmio_softc *sc;
415
416 sc = device_get_softc(dev);
417
418 if (sc->vtmmio_child_dev != child)
419 return (ENOENT);
420
421 switch (index) {
422 case VIRTIO_IVAR_DEVTYPE:
423 case VIRTIO_IVAR_SUBDEVICE:
424 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID);
425 break;
426 case VIRTIO_IVAR_VENDOR:
427 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID);
428 break;
429 default:
430 return (ENOENT);
431 }
432
433 return (0);
434 }
435
436 static int
vtmmio_write_ivar(device_t dev,device_t child,int index,uintptr_t value)437 vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
438 {
439 struct vtmmio_softc *sc;
440
441 sc = device_get_softc(dev);
442
443 if (sc->vtmmio_child_dev != child)
444 return (ENOENT);
445
446 switch (index) {
447 case VIRTIO_IVAR_FEATURE_DESC:
448 sc->vtmmio_child_feat_desc = (void *) value;
449 break;
450 default:
451 return (ENOENT);
452 }
453
454 return (0);
455 }
456
457 static uint64_t
vtmmio_negotiate_features(device_t dev,uint64_t child_features)458 vtmmio_negotiate_features(device_t dev, uint64_t child_features)
459 {
460 struct vtmmio_softc *sc;
461 uint64_t host_features, features;
462
463 sc = device_get_softc(dev);
464
465 host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES);
466 vtmmio_describe_features(sc, "host", host_features);
467
468 /*
469 * Limit negotiated features to what the driver, virtqueue, and
470 * host all support.
471 */
472 features = host_features & child_features;
473 features = virtqueue_filter_features(features);
474 sc->vtmmio_features = features;
475
476 vtmmio_describe_features(sc, "negotiated", features);
477 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features);
478
479 return (features);
480 }
481
482 static int
vtmmio_with_feature(device_t dev,uint64_t feature)483 vtmmio_with_feature(device_t dev, uint64_t feature)
484 {
485 struct vtmmio_softc *sc;
486
487 sc = device_get_softc(dev);
488
489 return ((sc->vtmmio_features & feature) != 0);
490 }
491
492 static int
vtmmio_alloc_virtqueues(device_t dev,int flags,int nvqs,struct vq_alloc_info * vq_info)493 vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs,
494 struct vq_alloc_info *vq_info)
495 {
496 struct vtmmio_virtqueue *vqx;
497 struct vq_alloc_info *info;
498 struct vtmmio_softc *sc;
499 struct virtqueue *vq;
500 uint32_t size;
501 int idx, error;
502
503 sc = device_get_softc(dev);
504
505 if (sc->vtmmio_nvqs != 0)
506 return (EALREADY);
507 if (nvqs <= 0)
508 return (EINVAL);
509
510 sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue),
511 M_DEVBUF, M_NOWAIT | M_ZERO);
512 if (sc->vtmmio_vqs == NULL)
513 return (ENOMEM);
514
515 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, 1 << PAGE_SHIFT);
516
517 for (idx = 0; idx < nvqs; idx++) {
518 vqx = &sc->vtmmio_vqs[idx];
519 info = &vq_info[idx];
520
521 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
522
523 vtmmio_select_virtqueue(sc, idx);
524 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
525
526 error = virtqueue_alloc(dev, idx, size,
527 VIRTIO_MMIO_VRING_ALIGN, 0xFFFFFFFFUL, info, &vq);
528 if (error) {
529 device_printf(dev,
530 "cannot allocate virtqueue %d: %d\n",
531 idx, error);
532 break;
533 }
534
535 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size);
536 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN,
537 VIRTIO_MMIO_VRING_ALIGN);
538 #if 0
539 device_printf(dev, "virtqueue paddr 0x%08lx\n",
540 (uint64_t)virtqueue_paddr(vq));
541 #endif
542 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
543 virtqueue_paddr(vq) >> PAGE_SHIFT);
544
545 vqx->vtv_vq = *info->vqai_vq = vq;
546 vqx->vtv_no_intr = info->vqai_intr == NULL;
547
548 sc->vtmmio_nvqs++;
549 }
550
551 if (error)
552 vtmmio_free_virtqueues(sc);
553
554 return (error);
555 }
556
557 static void
vtmmio_stop(device_t dev)558 vtmmio_stop(device_t dev)
559 {
560
561 vtmmio_reset(device_get_softc(dev));
562 }
563
564 static void
vtmmio_poll(device_t dev)565 vtmmio_poll(device_t dev)
566 {
567 struct vtmmio_softc *sc;
568
569 sc = device_get_softc(dev);
570
571 if (sc->platform != NULL)
572 VIRTIO_MMIO_POLL(sc->platform);
573 }
574
575 static int
vtmmio_reinit(device_t dev,uint64_t features)576 vtmmio_reinit(device_t dev, uint64_t features)
577 {
578 struct vtmmio_softc *sc;
579 int idx, error;
580
581 sc = device_get_softc(dev);
582
583 if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
584 vtmmio_stop(dev);
585
586 /*
587 * Quickly drive the status through ACK and DRIVER. The device
588 * does not become usable again until vtmmio_reinit_complete().
589 */
590 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
591 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
592
593 vtmmio_negotiate_features(dev, features);
594
595 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
596 error = vtmmio_reinit_virtqueue(sc, idx);
597 if (error)
598 return (error);
599 }
600
601 return (0);
602 }
603
604 static void
vtmmio_reinit_complete(device_t dev)605 vtmmio_reinit_complete(device_t dev)
606 {
607
608 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
609 }
610
611 static void
vtmmio_notify_virtqueue(device_t dev,uint16_t queue)612 vtmmio_notify_virtqueue(device_t dev, uint16_t queue)
613 {
614 struct vtmmio_softc *sc;
615
616 sc = device_get_softc(dev);
617
618 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue);
619 }
620
621 static uint8_t
vtmmio_get_status(device_t dev)622 vtmmio_get_status(device_t dev)
623 {
624 struct vtmmio_softc *sc;
625
626 sc = device_get_softc(dev);
627
628 return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS));
629 }
630
631 static void
vtmmio_set_status(device_t dev,uint8_t status)632 vtmmio_set_status(device_t dev, uint8_t status)
633 {
634 struct vtmmio_softc *sc;
635
636 sc = device_get_softc(dev);
637
638 if (status != VIRTIO_CONFIG_STATUS_RESET)
639 status |= vtmmio_get_status(dev);
640
641 vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status);
642 }
643
644 static void
vtmmio_read_dev_config(device_t dev,bus_size_t offset,void * dst,int length)645 vtmmio_read_dev_config(device_t dev, bus_size_t offset,
646 void *dst, int length)
647 {
648 struct vtmmio_softc *sc;
649 bus_size_t off;
650 uint8_t *d;
651 int size;
652
653 sc = device_get_softc(dev);
654 off = VIRTIO_MMIO_CONFIG + offset;
655
656 for (d = dst; length > 0; d += size, off += size, length -= size) {
657 #ifdef ALLOW_WORD_ALIGNED_ACCESS
658 if (length >= 4) {
659 size = 4;
660 *(uint32_t *)d = vtmmio_read_config_4(sc, off);
661 } else if (length >= 2) {
662 size = 2;
663 *(uint16_t *)d = vtmmio_read_config_2(sc, off);
664 } else
665 #endif
666 {
667 size = 1;
668 *d = vtmmio_read_config_1(sc, off);
669 }
670 }
671 }
672
673 static void
vtmmio_write_dev_config(device_t dev,bus_size_t offset,void * src,int length)674 vtmmio_write_dev_config(device_t dev, bus_size_t offset,
675 void *src, int length)
676 {
677 struct vtmmio_softc *sc;
678 bus_size_t off;
679 uint8_t *s;
680 int size;
681
682 sc = device_get_softc(dev);
683 off = VIRTIO_MMIO_CONFIG + offset;
684
685 for (s = src; length > 0; s += size, off += size, length -= size) {
686 #ifdef ALLOW_WORD_ALIGNED_ACCESS
687 if (length >= 4) {
688 size = 4;
689 vtmmio_write_config_4(sc, off, *(uint32_t *)s);
690 } else if (length >= 2) {
691 size = 2;
692 vtmmio_write_config_2(sc, off, *(uint16_t *)s);
693 } else
694 #endif
695 {
696 size = 1;
697 vtmmio_write_config_1(sc, off, *s);
698 }
699 }
700 }
701
702 static void
vtmmio_describe_features(struct vtmmio_softc * sc,const char * msg,uint64_t features)703 vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg,
704 uint64_t features)
705 {
706 device_t dev, child;
707
708 dev = sc->dev;
709 child = sc->vtmmio_child_dev;
710
711 if (device_is_attached(child) || bootverbose == 0)
712 return;
713
714 virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc);
715 }
716
717 static void
vtmmio_probe_and_attach_child(struct vtmmio_softc * sc)718 vtmmio_probe_and_attach_child(struct vtmmio_softc *sc)
719 {
720 device_t dev, child;
721
722 dev = sc->dev;
723 child = sc->vtmmio_child_dev;
724
725 if (child == NULL)
726 return;
727
728 if (device_get_state(child) != DS_NOTPRESENT) {
729 return;
730 }
731
732 if (device_probe(child) != 0) {
733 return;
734 }
735
736 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
737 if (device_attach(child) != 0) {
738 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
739 vtmmio_reset(sc);
740 vtmmio_release_child_resources(sc);
741 /* Reset status for future attempt. */
742 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
743 } else {
744 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
745 VIRTIO_ATTACH_COMPLETED(child);
746 }
747 }
748
749 static int
vtmmio_reinit_virtqueue(struct vtmmio_softc * sc,int idx)750 vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx)
751 {
752 struct vtmmio_virtqueue *vqx;
753 struct virtqueue *vq;
754 int error;
755 uint16_t size;
756
757 vqx = &sc->vtmmio_vqs[idx];
758 vq = vqx->vtv_vq;
759
760 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
761
762 vtmmio_select_virtqueue(sc, idx);
763 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX);
764
765 error = virtqueue_reinit(vq, size);
766 if (error)
767 return (error);
768
769 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN,
770 virtqueue_paddr(vq) >> PAGE_SHIFT);
771
772 return (0);
773 }
774
775 static void
vtmmio_free_interrupts(struct vtmmio_softc * sc)776 vtmmio_free_interrupts(struct vtmmio_softc *sc)
777 {
778
779 if (sc->ih != NULL)
780 bus_teardown_intr(sc->dev, sc->res[1], sc->ih);
781
782 if (sc->res[1] != NULL)
783 bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]);
784 }
785
786 static void
vtmmio_free_virtqueues(struct vtmmio_softc * sc)787 vtmmio_free_virtqueues(struct vtmmio_softc *sc)
788 {
789 struct vtmmio_virtqueue *vqx;
790 int idx;
791
792 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
793 vqx = &sc->vtmmio_vqs[idx];
794
795 vtmmio_select_virtqueue(sc, idx);
796 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0);
797
798 virtqueue_free(vqx->vtv_vq);
799 vqx->vtv_vq = NULL;
800 }
801
802 free(sc->vtmmio_vqs, M_DEVBUF);
803 sc->vtmmio_vqs = NULL;
804 sc->vtmmio_nvqs = 0;
805 }
806
807 static void
vtmmio_release_child_resources(struct vtmmio_softc * sc)808 vtmmio_release_child_resources(struct vtmmio_softc *sc)
809 {
810
811 vtmmio_free_interrupts(sc);
812 vtmmio_free_virtqueues(sc);
813 }
814
815 static void
vtmmio_reset(struct vtmmio_softc * sc)816 vtmmio_reset(struct vtmmio_softc *sc)
817 {
818
819 /*
820 * Setting the status to RESET sets the host device to
821 * the original, uninitialized state.
822 */
823 vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET);
824 }
825
826 static void
vtmmio_select_virtqueue(struct vtmmio_softc * sc,int idx)827 vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx)
828 {
829
830 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx);
831 }
832
833 static void
vtmmio_vq_intr(void * arg)834 vtmmio_vq_intr(void *arg)
835 {
836 struct vtmmio_virtqueue *vqx;
837 struct vtmmio_softc *sc;
838 struct virtqueue *vq;
839 uint32_t status;
840 int idx;
841
842 sc = arg;
843
844 status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS);
845 vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status);
846
847 /* The config changed */
848 if (status & VIRTIO_MMIO_INT_CONFIG)
849 if (sc->vtmmio_child_dev != NULL)
850 VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev);
851
852 /* Notify all virtqueues. */
853 if (status & VIRTIO_MMIO_INT_VRING) {
854 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) {
855 vqx = &sc->vtmmio_vqs[idx];
856 if (vqx->vtv_no_intr == 0) {
857 vq = vqx->vtv_vq;
858 virtqueue_intr(vq);
859 }
860 }
861 }
862 }
863