1 /* $NetBSD: bus_space.c,v 1.47 2022/07/17 08:33:48 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Charles M. Hannum and by Jason R. Thorpe of the Numerical Aerospace
9 * Simulation Facility, NASA Ames Research Center.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 #include <sys/cdefs.h>
34 __KERNEL_RCSID(0, "$NetBSD: bus_space.c,v 1.47 2022/07/17 08:33:48 riastradh Exp $");
35
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/malloc.h>
39 #include <sys/extent.h>
40 #include <sys/kmem.h>
41
42 #include <uvm/uvm_extern.h>
43
44 #include <dev/isa/isareg.h>
45
46 #include <sys/bus.h>
47 #include <machine/pio.h>
48 #include <machine/isa_machdep.h>
49
50 #ifdef XEN
51 #include <xen/hypervisor.h>
52 #endif
53
54 /*
55 * Macros for sanity-checking the aligned-ness of pointers passed to
56 * bus space ops. These are not strictly necessary on the x86, but
57 * could lead to performance improvements, and help catch problems
58 * with drivers that would creep up on other architectures.
59 */
60 #ifdef BUS_SPACE_DEBUG
61 #define BUS_SPACE_ALIGNED_ADDRESS(p, t) \
62 ((((u_long)(p)) & (sizeof(t)-1)) == 0)
63
64 #define BUS_SPACE_ADDRESS_SANITY(p, t, d) \
65 ({ \
66 if (BUS_SPACE_ALIGNED_ADDRESS((p), t) == 0) { \
67 printf("%s 0x%lx not aligned to %zu bytes %s:%d\n", \
68 d, (u_long)(p), sizeof(t), __FILE__, __LINE__); \
69 } \
70 (void) 0; \
71 })
72 #else
73 #define BUS_SPACE_ADDRESS_SANITY(p,t,d) (void) 0
74 #endif /* BUS_SPACE_DEBUG */
75
76 /*
77 * Extent maps to manage I/O and memory space. Allocate
78 * storage for 8 regions in each, initially. Later, ioport_malloc_safe
79 * will indicate that it's safe to use malloc() to dynamically allocate
80 * region descriptors.
81 *
82 * N.B. At least two regions are _always_ allocated from the iomem
83 * extent map; (0 -> ISA hole) and (end of ISA hole -> end of RAM).
84 *
85 * The extent maps are not static! Machine-dependent ISA and EISA
86 * routines need access to them for bus address space allocation.
87 */
88 static long ioport_ex_storage[EXTENT_FIXED_STORAGE_SIZE(16) / sizeof(long)];
89 static long iomem_ex_storage[EXTENT_FIXED_STORAGE_SIZE(64) / sizeof(long)];
90 struct extent *ioport_ex;
91 struct extent *iomem_ex;
92 static int ioport_malloc_safe;
93
94 static struct bus_space_tag x86_io = { .bst_type = X86_BUS_SPACE_IO };
95 static struct bus_space_tag x86_mem = { .bst_type = X86_BUS_SPACE_MEM };
96
97 bus_space_tag_t x86_bus_space_io = &x86_io;
98 bus_space_tag_t x86_bus_space_mem = &x86_mem;
99
100 int x86_mem_add_mapping(bus_addr_t, bus_size_t,
101 int, bus_space_handle_t *);
102
103 static inline bool
x86_bus_space_is_io(bus_space_tag_t t)104 x86_bus_space_is_io(bus_space_tag_t t)
105 {
106 return t->bst_type == X86_BUS_SPACE_IO;
107 }
108
109 static inline bool
x86_bus_space_is_mem(bus_space_tag_t t)110 x86_bus_space_is_mem(bus_space_tag_t t)
111 {
112 return t->bst_type == X86_BUS_SPACE_MEM;
113 }
114
115 void
x86_bus_space_init(void)116 x86_bus_space_init(void)
117 {
118 /*
119 * Initialize the I/O port and I/O mem extent maps.
120 * Note: we don't have to check the return value since
121 * creation of a fixed extent map will never fail (since
122 * descriptor storage has already been allocated).
123 *
124 * N.B. The iomem extent manages _all_ physical addresses
125 * on the machine. When the amount of RAM is found, the two
126 * extents of RAM are allocated from the map (0 -> ISA hole
127 * and end of ISA hole -> end of RAM).
128 */
129 ioport_ex = extent_create("ioport", 0x0, 0xffff,
130 (void *)ioport_ex_storage, sizeof(ioport_ex_storage),
131 EX_NOCOALESCE|EX_NOWAIT);
132 iomem_ex = extent_create("iomem", 0x0, MAXIOMEM,
133 (void *)iomem_ex_storage, sizeof(iomem_ex_storage),
134 EX_NOCOALESCE|EX_NOWAIT);
135
136 #ifdef XENPV
137 /* We are privileged guest os - should have IO privileges. */
138 if (xendomain_is_privileged()) {
139 struct physdev_set_iopl set_iopl;
140 memset(&set_iopl, 0, sizeof(set_iopl));
141 set_iopl.iopl = 1;
142 if (HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl) != 0)
143 panic("Unable to obtain IOPL, "
144 "despite being SIF_PRIVILEGED");
145 }
146 #endif /* XENPV */
147 }
148
149 void
x86_bus_space_mallocok(void)150 x86_bus_space_mallocok(void)
151 {
152
153 ioport_malloc_safe = 1;
154 }
155
156 int
bus_space_map(bus_space_tag_t t,bus_addr_t bpa,bus_size_t size,int flags,bus_space_handle_t * bshp)157 bus_space_map(bus_space_tag_t t, bus_addr_t bpa, bus_size_t size,
158 int flags, bus_space_handle_t *bshp)
159 {
160 bus_space_reservation_t bsr;
161 bus_space_tag_t it;
162 int error;
163
164 if ((t->bst_exists & BUS_SPACE_OVERRIDE_MAP) == 0)
165 ; /* skip override */
166 else for (it = t; it != NULL; it = it->bst_super) {
167 if ((it->bst_present & BUS_SPACE_OVERRIDE_MAP) == 0)
168 continue;
169 return (*it->bst_ov->ov_space_map)(it->bst_ctx, t, bpa, size,
170 flags, bshp);
171 }
172
173 error = bus_space_reserve(t, bpa, size, flags, &bsr);
174 if (error != 0)
175 return error;
176
177 error = bus_space_reservation_map(t, &bsr, flags, bshp);
178 if (error != 0)
179 bus_space_release(t, &bsr);
180
181 return error;
182 }
183
184 int
bus_space_reservation_map(bus_space_tag_t t,bus_space_reservation_t * bsr,int flags,bus_space_handle_t * bshp)185 bus_space_reservation_map(bus_space_tag_t t, bus_space_reservation_t *bsr,
186 int flags, bus_space_handle_t *bshp)
187 {
188 bus_addr_t bpa;
189 bus_size_t size;
190 bus_space_tag_t it;
191
192 if ((t->bst_exists & BUS_SPACE_OVERRIDE_RESERVATION_MAP) == 0)
193 ; /* skip override */
194 else for (it = t; it != NULL; it = it->bst_super) {
195 if ((it->bst_present & BUS_SPACE_OVERRIDE_RESERVATION_MAP) == 0)
196 continue;
197 return (*it->bst_ov->ov_space_reservation_map)(it->bst_ctx, t,
198 bsr, flags, bshp);
199 }
200
201 bpa = bus_space_reservation_addr(bsr);
202 size = bus_space_reservation_size(bsr);
203
204 /*
205 * For I/O space, that's all she wrote.
206 */
207 if (x86_bus_space_is_io(t)) {
208 *bshp = bpa;
209 return 0;
210 }
211
212 #ifndef XENPV
213 if (bpa >= IOM_BEGIN && (bpa + size) != 0 && (bpa + size) <= IOM_END) {
214 *bshp = (bus_space_handle_t)ISA_HOLE_VADDR(bpa);
215 return 0;
216 }
217 #endif /* !XENPV */
218
219 /*
220 * For memory space, map the bus physical address to
221 * a kernel virtual address.
222 */
223 return x86_mem_add_mapping(bpa, size, flags, bshp);
224 }
225
226 int
_x86_memio_map(bus_space_tag_t t,bus_addr_t bpa,bus_size_t size,int flags,bus_space_handle_t * bshp)227 _x86_memio_map(bus_space_tag_t t, bus_addr_t bpa, bus_size_t size,
228 int flags, bus_space_handle_t *bshp)
229 {
230
231 /*
232 * For I/O space, just fill in the handle.
233 */
234 if (x86_bus_space_is_io(t)) {
235 if (flags & BUS_SPACE_MAP_LINEAR)
236 return (EOPNOTSUPP);
237 *bshp = bpa;
238 return (0);
239 }
240
241 /*
242 * For memory space, map the bus physical address to
243 * a kernel virtual address.
244 */
245 return x86_mem_add_mapping(bpa, size, flags, bshp);
246 }
247
248 int
bus_space_reserve(bus_space_tag_t t,bus_addr_t bpa,bus_size_t size,int flags,bus_space_reservation_t * bsrp)249 bus_space_reserve(bus_space_tag_t t,
250 bus_addr_t bpa,
251 bus_size_t size,
252 int flags, bus_space_reservation_t *bsrp)
253 {
254 struct extent *ex;
255 int error;
256 bus_space_tag_t it;
257
258 if ((t->bst_exists & BUS_SPACE_OVERRIDE_RESERVE) == 0)
259 ; /* skip override */
260 else for (it = t; it != NULL; it = it->bst_super) {
261 if ((it->bst_present & BUS_SPACE_OVERRIDE_RESERVE) == 0)
262 continue;
263 return (*it->bst_ov->ov_space_reserve)(it->bst_ctx, t,
264 bpa, size, flags, bsrp);
265 }
266
267 /*
268 * Pick the appropriate extent map.
269 */
270 if (x86_bus_space_is_io(t)) {
271 if (flags & BUS_SPACE_MAP_LINEAR)
272 return (EOPNOTSUPP);
273 ex = ioport_ex;
274 } else if (x86_bus_space_is_mem(t))
275 ex = iomem_ex;
276 else
277 panic("x86_memio_alloc: bad bus space tag");
278
279 /*
280 * Before we go any further, let's make sure that this
281 * region is available.
282 */
283 error = extent_alloc_region(ex, bpa, size,
284 EX_NOWAIT | (ioport_malloc_safe ? EX_MALLOCOK : 0));
285
286 if (error != 0)
287 return error;
288
289 bus_space_reservation_init(bsrp, bpa, size);
290
291 return 0;
292 }
293
294 int
bus_space_reserve_subregion(bus_space_tag_t t,bus_addr_t rstart,bus_addr_t rend,const bus_size_t size,const bus_size_t alignment,const bus_size_t boundary,const int flags,bus_space_reservation_t * bsrp)295 bus_space_reserve_subregion(bus_space_tag_t t,
296 bus_addr_t rstart, bus_addr_t rend,
297 const bus_size_t size, const bus_size_t alignment,
298 const bus_size_t boundary,
299 const int flags, bus_space_reservation_t *bsrp)
300 {
301 bus_space_reservation_t bsr;
302 struct extent *ex;
303 u_long bpa;
304 int error;
305 bus_space_tag_t it;
306
307 if ((t->bst_exists & BUS_SPACE_OVERRIDE_RESERVE_SUBREGION) == 0)
308 ; /* skip override */
309 else for (it = t; it != NULL; it = it->bst_super) {
310 if ((it->bst_present & BUS_SPACE_OVERRIDE_RESERVE_SUBREGION) ==
311 0)
312 continue;
313 return (*it->bst_ov->ov_space_reserve_subregion)(it->bst_ctx, t,
314 rstart, rend, size, alignment, boundary, flags, bsrp);
315 }
316
317 /*
318 * Pick the appropriate extent map.
319 */
320 if (x86_bus_space_is_io(t)) {
321 if (flags & BUS_SPACE_MAP_LINEAR)
322 return (EOPNOTSUPP);
323 ex = ioport_ex;
324 } else if (x86_bus_space_is_mem(t))
325 ex = iomem_ex;
326 else
327 panic("x86_memio_alloc: bad bus space tag");
328
329 /*
330 * Sanity check the allocation against the extent's boundaries.
331 */
332 rstart = MAX(rstart, ex->ex_start);
333 rend = MIN(rend, ex->ex_end);
334 if (rstart >= rend)
335 panic("x86_memio_alloc: bad region start/end");
336
337 /*
338 * Do the requested allocation.
339 */
340 error = extent_alloc_subregion(ex, rstart, rend, size, alignment,
341 boundary,
342 EX_FAST | EX_NOWAIT | (ioport_malloc_safe ? EX_MALLOCOK : 0),
343 &bpa);
344
345 if (error)
346 return (error);
347
348 bus_space_reservation_init(&bsr, bpa, size);
349
350 *bsrp = bsr;
351
352 return 0;
353 }
354
355 void
bus_space_release(bus_space_tag_t t,bus_space_reservation_t * bsr)356 bus_space_release(bus_space_tag_t t, bus_space_reservation_t *bsr)
357 {
358 struct extent *ex;
359 bus_space_tag_t it;
360
361 if ((t->bst_exists & BUS_SPACE_OVERRIDE_RELEASE) == 0)
362 ; /* skip override */
363 else for (it = t; it != NULL; it = it->bst_super) {
364 if ((it->bst_present & BUS_SPACE_OVERRIDE_RELEASE) == 0)
365 continue;
366 (*it->bst_ov->ov_space_release)(it->bst_ctx, t, bsr);
367 return;
368 }
369
370 /*
371 * Pick the appropriate extent map.
372 */
373 if (x86_bus_space_is_io(t)) {
374 ex = ioport_ex;
375 } else if (x86_bus_space_is_mem(t))
376 ex = iomem_ex;
377 else
378 panic("x86_memio_alloc: bad bus space tag");
379
380 if (extent_free(ex, bus_space_reservation_addr(bsr),
381 bus_space_reservation_size(bsr), EX_NOWAIT |
382 (ioport_malloc_safe ? EX_MALLOCOK : 0))) {
383 printf("%s: pa 0x%jx, size 0x%jx\n", __func__,
384 (uintmax_t)bus_space_reservation_addr(bsr),
385 (uintmax_t)bus_space_reservation_size(bsr));
386 printf("%s: can't free region\n", __func__);
387 }
388 }
389
390 int
bus_space_alloc(bus_space_tag_t t,bus_addr_t rstart,bus_addr_t rend,bus_size_t size,bus_size_t alignment,bus_size_t boundary,int flags,bus_addr_t * bpap,bus_space_handle_t * bshp)391 bus_space_alloc(bus_space_tag_t t, bus_addr_t rstart, bus_addr_t rend,
392 bus_size_t size, bus_size_t alignment, bus_size_t boundary,
393 int flags, bus_addr_t *bpap, bus_space_handle_t *bshp)
394 {
395 bus_space_reservation_t bsr;
396 bus_space_tag_t it;
397 int error;
398
399 if ((t->bst_exists & BUS_SPACE_OVERRIDE_ALLOC) == 0)
400 ; /* skip override */
401 else for (it = t; it != NULL; it = it->bst_super) {
402 if ((it->bst_present & BUS_SPACE_OVERRIDE_ALLOC) == 0)
403 continue;
404 return (*it->bst_ov->ov_space_alloc)(it->bst_ctx, t,
405 rstart, rend, size, alignment, boundary, flags, bpap, bshp);
406 }
407
408 /*
409 * Do the requested allocation.
410 */
411 error = bus_space_reserve_subregion(t, rstart, rend, size, alignment,
412 boundary, flags, &bsr);
413
414 if (error != 0)
415 return error;
416
417 error = bus_space_reservation_map(t, &bsr, flags, bshp);
418 if (error != 0)
419 bus_space_release(t, &bsr);
420
421 *bpap = bus_space_reservation_addr(&bsr);
422
423 return error;
424 }
425
426 int
x86_mem_add_mapping(bus_addr_t bpa,bus_size_t size,int flags,bus_space_handle_t * bshp)427 x86_mem_add_mapping(bus_addr_t bpa, bus_size_t size,
428 int flags, bus_space_handle_t *bshp)
429 {
430 paddr_t pa, endpa;
431 vaddr_t va, sva;
432 u_int pmapflags;
433
434 pa = x86_trunc_page(bpa);
435 endpa = x86_round_page(bpa + size);
436
437 pmapflags = PMAP_NOCACHE;
438 if ((flags & BUS_SPACE_MAP_CACHEABLE) != 0)
439 pmapflags = 0;
440 else if (flags & BUS_SPACE_MAP_PREFETCHABLE)
441 pmapflags = PMAP_WRITE_COMBINE;
442
443 #ifdef DIAGNOSTIC
444 if (endpa != 0 && endpa <= pa)
445 panic("x86_mem_add_mapping: overflow");
446 #endif
447
448 #ifdef XENPV
449 if (bpa >= IOM_BEGIN && (bpa + size) != 0 && (bpa + size) <= IOM_END) {
450 sva = (vaddr_t)ISA_HOLE_VADDR(pa);
451 } else
452 #endif /* XENPV */
453 {
454 sva = uvm_km_alloc(kernel_map, endpa - pa, 0,
455 UVM_KMF_VAONLY | UVM_KMF_NOWAIT);
456 if (sva == 0)
457 return (ENOMEM);
458 }
459
460 *bshp = (bus_space_handle_t)(sva + (bpa & PGOFSET));
461
462 for (va = sva; pa != endpa; pa += PAGE_SIZE, va += PAGE_SIZE) {
463 pmap_kenter_ma(va, pa, VM_PROT_READ | VM_PROT_WRITE, pmapflags);
464 }
465 pmap_update(pmap_kernel());
466
467 return 0;
468 }
469
470 bool
bus_space_is_equal(bus_space_tag_t t1,bus_space_tag_t t2)471 bus_space_is_equal(bus_space_tag_t t1, bus_space_tag_t t2)
472 {
473 if (t1 == NULL || t2 == NULL)
474 return false;
475 return t1->bst_type == t2->bst_type;
476 }
477
478 /*
479 * void _x86_memio_unmap(bus_space_tag bst, bus_space_handle bsh,
480 * bus_size_t size, bus_addr_t *adrp)
481 *
482 * This function unmaps memory- or io-space mapped by the function
483 * _x86_memio_map(). This function works nearly as same as
484 * x86_memio_unmap(), but this function does not ask kernel
485 * built-in extents and returns physical address of the bus space,
486 * for the convenience of the extra extent manager.
487 */
488 void
_x86_memio_unmap(bus_space_tag_t t,bus_space_handle_t bsh,bus_size_t size,bus_addr_t * adrp)489 _x86_memio_unmap(bus_space_tag_t t, bus_space_handle_t bsh,
490 bus_size_t size, bus_addr_t *adrp)
491 {
492 u_long va, endva;
493 bus_addr_t bpa;
494
495 /*
496 * Find the correct extent and bus physical address.
497 */
498 if (x86_bus_space_is_io(t)) {
499 bpa = bsh;
500 } else if (x86_bus_space_is_mem(t)) {
501 if (bsh >= atdevbase && (bsh + size) != 0 &&
502 (bsh + size) <= (atdevbase + IOM_SIZE)) {
503 bpa = (bus_addr_t)ISA_PHYSADDR(bsh);
504 } else {
505
506 va = x86_trunc_page(bsh);
507 endva = x86_round_page(bsh + size);
508
509 #ifdef DIAGNOSTIC
510 if (endva <= va) {
511 panic("_x86_memio_unmap: overflow");
512 }
513 #endif
514
515 if (pmap_extract_ma(pmap_kernel(), va, &bpa) == FALSE) {
516 panic("_x86_memio_unmap:"
517 " wrong virtual address");
518 }
519 bpa += (bsh & PGOFSET);
520 pmap_kremove(va, endva - va);
521 pmap_update(pmap_kernel());
522
523 /*
524 * Free the kernel virtual mapping.
525 */
526 uvm_km_free(kernel_map, va, endva - va, UVM_KMF_VAONLY);
527 }
528 } else {
529 panic("_x86_memio_unmap: bad bus space tag");
530 }
531
532 if (adrp != NULL) {
533 *adrp = bpa;
534 }
535 }
536
537 static void
bus_space_reservation_unmap1(bus_space_tag_t t,const bus_space_handle_t bsh,const bus_size_t size,bus_addr_t * bpap)538 bus_space_reservation_unmap1(bus_space_tag_t t, const bus_space_handle_t bsh,
539 const bus_size_t size, bus_addr_t *bpap)
540 {
541 u_long va, endva;
542 bus_addr_t bpa;
543
544 /*
545 * Find the correct extent and bus physical address.
546 */
547 if (x86_bus_space_is_io(t)) {
548 bpa = bsh;
549 } else if (x86_bus_space_is_mem(t)) {
550 if (bsh >= atdevbase && (bsh + size) != 0 &&
551 (bsh + size) <= (atdevbase + IOM_SIZE)) {
552 bpa = (bus_addr_t)ISA_PHYSADDR(bsh);
553 goto ok;
554 }
555
556 va = x86_trunc_page(bsh);
557 endva = x86_round_page(bsh + size);
558
559 #ifdef DIAGNOSTIC
560 if (endva <= va)
561 panic("x86_memio_unmap: overflow");
562 #endif
563
564 (void) pmap_extract_ma(pmap_kernel(), va, &bpa);
565 bpa += (bsh & PGOFSET);
566
567 pmap_kremove(va, endva - va);
568 pmap_update(pmap_kernel());
569
570 /*
571 * Free the kernel virtual mapping.
572 */
573 uvm_km_free(kernel_map, va, endva - va, UVM_KMF_VAONLY);
574 } else
575 panic("x86_memio_unmap: bad bus space tag");
576 ok:
577 if (bpap != NULL)
578 *bpap = bpa;
579 }
580
581 void
bus_space_reservation_unmap(bus_space_tag_t t,const bus_space_handle_t bsh,const bus_size_t size)582 bus_space_reservation_unmap(bus_space_tag_t t, const bus_space_handle_t bsh,
583 const bus_size_t size)
584 {
585 bus_space_tag_t it;
586
587 if ((t->bst_exists & BUS_SPACE_OVERRIDE_RESERVATION_UNMAP) == 0)
588 ; /* skip override */
589 else for (it = t; it != NULL; it = it->bst_super) {
590 if ((it->bst_present & BUS_SPACE_OVERRIDE_RESERVATION_UNMAP) ==
591 0)
592 continue;
593 (*it->bst_ov->ov_space_reservation_unmap)(it->bst_ctx,
594 t, bsh, size);
595 return;
596 }
597
598 bus_space_reservation_unmap1(t, bsh, size, NULL);
599 }
600
601 void
bus_space_unmap(bus_space_tag_t t,const bus_space_handle_t bsh,const bus_size_t size)602 bus_space_unmap(bus_space_tag_t t, const bus_space_handle_t bsh,
603 const bus_size_t size)
604 {
605 bus_addr_t addr;
606 bus_space_reservation_t bsr;
607 bus_space_tag_t it;
608
609 if ((t->bst_exists & BUS_SPACE_OVERRIDE_UNMAP) == 0)
610 ; /* skip override */
611 else for (it = t; it != NULL; it = it->bst_super) {
612 if ((it->bst_present & BUS_SPACE_OVERRIDE_UNMAP) == 0)
613 continue;
614 (*it->bst_ov->ov_space_unmap)(it->bst_ctx, t, bsh, size);
615 return;
616 }
617
618 bus_space_reservation_unmap1(t, bsh, size, &addr);
619
620 bus_space_reservation_init(&bsr, addr, size);
621 bus_space_release(t, &bsr);
622 }
623
624 void
bus_space_free(bus_space_tag_t t,bus_space_handle_t bsh,bus_size_t size)625 bus_space_free(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t size)
626 {
627 bus_space_tag_t it;
628
629 if ((t->bst_exists & BUS_SPACE_OVERRIDE_FREE) == 0)
630 ; /* skip override */
631 else for (it = t; it != NULL; it = it->bst_super) {
632 if ((it->bst_present & BUS_SPACE_OVERRIDE_FREE) == 0)
633 continue;
634 (*it->bst_ov->ov_space_free)(it->bst_ctx, t, bsh, size);
635 return;
636 }
637 /* bus_space_unmap() does all that we need to do. */
638 bus_space_unmap(t, bsh, size);
639 }
640
641 int
bus_space_subregion(bus_space_tag_t t,bus_space_handle_t bsh,bus_size_t offset,bus_size_t size,bus_space_handle_t * nbshp)642 bus_space_subregion(bus_space_tag_t t, bus_space_handle_t bsh,
643 bus_size_t offset, bus_size_t size, bus_space_handle_t *nbshp)
644 {
645
646 *nbshp = bsh + offset;
647 return (0);
648 }
649
650 paddr_t
bus_space_mmap(bus_space_tag_t t,bus_addr_t addr,off_t off,int prot,int flags)651 bus_space_mmap(bus_space_tag_t t, bus_addr_t addr, off_t off, int prot,
652 int flags)
653 {
654 paddr_t pflags = 0;
655
656 /* Can't mmap I/O space. */
657 if (x86_bus_space_is_io(t))
658 return (-1);
659
660 /*
661 * "addr" is the base address of the device we're mapping.
662 * "off" is the offset into that device.
663 *
664 * Note we are called for each "page" in the device that
665 * the upper layers want to map.
666 */
667 if (flags & BUS_SPACE_MAP_PREFETCHABLE)
668 pflags |= X86_MMAP_FLAG_PREFETCH;
669
670 return x86_btop(addr + off) | (pflags << X86_MMAP_FLAG_SHIFT);
671 }
672
673 void
bus_space_set_multi_1(bus_space_tag_t t,bus_space_handle_t h,bus_size_t o,uint8_t v,size_t c)674 bus_space_set_multi_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
675 uint8_t v, size_t c)
676 {
677 vaddr_t addr = h + o;
678
679 if (x86_bus_space_is_io(t))
680 while (c--)
681 outb(addr, v);
682 else
683 while (c--)
684 *(volatile uint8_t *)(addr) = v;
685 }
686
687 void
bus_space_set_multi_2(bus_space_tag_t t,bus_space_handle_t h,bus_size_t o,uint16_t v,size_t c)688 bus_space_set_multi_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
689 uint16_t v, size_t c)
690 {
691 vaddr_t addr = h + o;
692
693 BUS_SPACE_ADDRESS_SANITY(addr, uint16_t, "bus addr");
694
695 if (x86_bus_space_is_io(t))
696 while (c--)
697 outw(addr, v);
698 else
699 while (c--)
700 *(volatile uint16_t *)(addr) = v;
701 }
702
703 void
bus_space_set_multi_4(bus_space_tag_t t,bus_space_handle_t h,bus_size_t o,uint32_t v,size_t c)704 bus_space_set_multi_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
705 uint32_t v, size_t c)
706 {
707 vaddr_t addr = h + o;
708
709 BUS_SPACE_ADDRESS_SANITY(addr, uint32_t, "bus addr");
710
711 if (x86_bus_space_is_io(t))
712 while (c--)
713 outl(addr, v);
714 else
715 while (c--)
716 *(volatile uint32_t *)(addr) = v;
717 }
718
719 void
bus_space_set_region_1(bus_space_tag_t t,bus_space_handle_t h,bus_size_t o,uint8_t v,size_t c)720 bus_space_set_region_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
721 uint8_t v, size_t c)
722 {
723 vaddr_t addr = h + o;
724
725 if (x86_bus_space_is_io(t))
726 for (; c != 0; c--, addr++)
727 outb(addr, v);
728 else
729 for (; c != 0; c--, addr++)
730 *(volatile uint8_t *)(addr) = v;
731 }
732
733 void
bus_space_set_region_2(bus_space_tag_t t,bus_space_handle_t h,bus_size_t o,uint16_t v,size_t c)734 bus_space_set_region_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
735 uint16_t v, size_t c)
736 {
737 vaddr_t addr = h + o;
738
739 BUS_SPACE_ADDRESS_SANITY(addr, uint16_t, "bus addr");
740
741 if (x86_bus_space_is_io(t))
742 for (; c != 0; c--, addr += 2)
743 outw(addr, v);
744 else
745 for (; c != 0; c--, addr += 2)
746 *(volatile uint16_t *)(addr) = v;
747 }
748
749 void
bus_space_set_region_4(bus_space_tag_t t,bus_space_handle_t h,bus_size_t o,uint32_t v,size_t c)750 bus_space_set_region_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
751 uint32_t v, size_t c)
752 {
753 vaddr_t addr = h + o;
754
755 BUS_SPACE_ADDRESS_SANITY(addr, uint32_t, "bus addr");
756
757 if (x86_bus_space_is_io(t))
758 for (; c != 0; c--, addr += 4)
759 outl(addr, v);
760 else
761 for (; c != 0; c--, addr += 4)
762 *(volatile uint32_t *)(addr) = v;
763 }
764
765 void
bus_space_copy_region_1(bus_space_tag_t t,bus_space_handle_t h1,bus_size_t o1,bus_space_handle_t h2,bus_size_t o2,size_t c)766 bus_space_copy_region_1(bus_space_tag_t t, bus_space_handle_t h1,
767 bus_size_t o1, bus_space_handle_t h2,
768 bus_size_t o2, size_t c)
769 {
770 vaddr_t addr1 = h1 + o1;
771 vaddr_t addr2 = h2 + o2;
772
773 if (x86_bus_space_is_io(t)) {
774 if (addr1 >= addr2) {
775 /* src after dest: copy forward */
776 for (; c != 0; c--, addr1++, addr2++)
777 outb(addr2, inb(addr1));
778 } else {
779 /* dest after src: copy backwards */
780 for (addr1 += (c - 1), addr2 += (c - 1);
781 c != 0; c--, addr1--, addr2--)
782 outb(addr2, inb(addr1));
783 }
784 } else {
785 if (addr1 >= addr2) {
786 /* src after dest: copy forward */
787 for (; c != 0; c--, addr1++, addr2++)
788 *(volatile uint8_t *)(addr2) =
789 *(volatile uint8_t *)(addr1);
790 } else {
791 /* dest after src: copy backwards */
792 for (addr1 += (c - 1), addr2 += (c - 1);
793 c != 0; c--, addr1--, addr2--)
794 *(volatile uint8_t *)(addr2) =
795 *(volatile uint8_t *)(addr1);
796 }
797 }
798 }
799
800 void
bus_space_copy_region_2(bus_space_tag_t t,bus_space_handle_t h1,bus_size_t o1,bus_space_handle_t h2,bus_size_t o2,size_t c)801 bus_space_copy_region_2(bus_space_tag_t t, bus_space_handle_t h1,
802 bus_size_t o1, bus_space_handle_t h2,
803 bus_size_t o2, size_t c)
804 {
805 vaddr_t addr1 = h1 + o1;
806 vaddr_t addr2 = h2 + o2;
807
808 BUS_SPACE_ADDRESS_SANITY(addr1, uint16_t, "bus addr 1");
809 BUS_SPACE_ADDRESS_SANITY(addr2, uint16_t, "bus addr 2");
810
811 if (x86_bus_space_is_io(t)) {
812 if (addr1 >= addr2) {
813 /* src after dest: copy forward */
814 for (; c != 0; c--, addr1 += 2, addr2 += 2)
815 outw(addr2, inw(addr1));
816 } else {
817 /* dest after src: copy backwards */
818 for (addr1 += 2 * (c - 1), addr2 += 2 * (c - 1);
819 c != 0; c--, addr1 -= 2, addr2 -= 2)
820 outw(addr2, inw(addr1));
821 }
822 } else {
823 if (addr1 >= addr2) {
824 /* src after dest: copy forward */
825 for (; c != 0; c--, addr1 += 2, addr2 += 2)
826 *(volatile uint16_t *)(addr2) =
827 *(volatile uint16_t *)(addr1);
828 } else {
829 /* dest after src: copy backwards */
830 for (addr1 += 2 * (c - 1), addr2 += 2 * (c - 1);
831 c != 0; c--, addr1 -= 2, addr2 -= 2)
832 *(volatile uint16_t *)(addr2) =
833 *(volatile uint16_t *)(addr1);
834 }
835 }
836 }
837
838 void
bus_space_copy_region_4(bus_space_tag_t t,bus_space_handle_t h1,bus_size_t o1,bus_space_handle_t h2,bus_size_t o2,size_t c)839 bus_space_copy_region_4(bus_space_tag_t t, bus_space_handle_t h1,
840 bus_size_t o1, bus_space_handle_t h2,
841 bus_size_t o2, size_t c)
842 {
843 vaddr_t addr1 = h1 + o1;
844 vaddr_t addr2 = h2 + o2;
845
846 BUS_SPACE_ADDRESS_SANITY(addr1, uint32_t, "bus addr 1");
847 BUS_SPACE_ADDRESS_SANITY(addr2, uint32_t, "bus addr 2");
848
849 if (x86_bus_space_is_io(t)) {
850 if (addr1 >= addr2) {
851 /* src after dest: copy forward */
852 for (; c != 0; c--, addr1 += 4, addr2 += 4)
853 outl(addr2, inl(addr1));
854 } else {
855 /* dest after src: copy backwards */
856 for (addr1 += 4 * (c - 1), addr2 += 4 * (c - 1);
857 c != 0; c--, addr1 -= 4, addr2 -= 4)
858 outl(addr2, inl(addr1));
859 }
860 } else {
861 if (addr1 >= addr2) {
862 /* src after dest: copy forward */
863 for (; c != 0; c--, addr1 += 4, addr2 += 4)
864 *(volatile uint32_t *)(addr2) =
865 *(volatile uint32_t *)(addr1);
866 } else {
867 /* dest after src: copy backwards */
868 for (addr1 += 4 * (c - 1), addr2 += 4 * (c - 1);
869 c != 0; c--, addr1 -= 4, addr2 -= 4)
870 *(volatile uint32_t *)(addr2) =
871 *(volatile uint32_t *)(addr1);
872 }
873 }
874 }
875
876 void
bus_space_barrier(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,bus_size_t len,int flags)877 bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t bsh,
878 bus_size_t offset, bus_size_t len, int flags)
879 {
880
881 /* I/O instructions always happen in program order. */
882 if (x86_bus_space_is_io(tag))
883 return;
884
885 /*
886 * For default mappings, which are mapped with UC-type memory
887 * regions, all loads and stores are issued in program order.
888 *
889 * For BUS_SPACE_MAP_PREFETCHABLE mappings, which are mapped
890 * with WC-type memory regions, loads and stores may be issued
891 * out of order, potentially requiring any of the three x86
892 * fences -- LFENCE, SFENCE, MFENCE.
893 *
894 * For BUS_SPACE_MAP_CACHEABLE mappings, which are mapped with
895 * WB-type memory regions (like normal memory), store/load may
896 * be reordered to load/store, potentially requiring MFENCE.
897 *
898 * We can't easily tell here how the region was mapped (without
899 * consulting the page tables), so just issue the fence
900 * unconditionally. Chances are either it's necessary or the
901 * cost is small in comparison to device register I/O.
902 *
903 * Reference:
904 *
905 * AMD64 Architecture Programmer's Manual, Volume 2:
906 * System Programming, 24593--Rev. 3.38--November 2021,
907 * Sec. 7.4.2 Memory Barrier Interaction with Memory
908 * Types, Table 7-3, p. 196.
909 * https://web.archive.org/web/20220625040004/https://www.amd.com/system/files/TechDocs/24593.pdf#page=256
910 */
911 switch (flags) {
912 case 0:
913 break;
914 case BUS_SPACE_BARRIER_READ:
915 x86_lfence();
916 break;
917 case BUS_SPACE_BARRIER_WRITE:
918 x86_sfence();
919 break;
920 case BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE:
921 x86_mfence();
922 break;
923 default:
924 panic("unknown bus space barrier: 0x%x", (unsigned)flags);
925 }
926 }
927
928 void *
bus_space_vaddr(bus_space_tag_t tag,bus_space_handle_t bsh)929 bus_space_vaddr(bus_space_tag_t tag, bus_space_handle_t bsh)
930 {
931
932 return x86_bus_space_is_mem(tag) ? (void *)bsh : NULL;
933 }
934
935 static const void *
bit_to_function_pointer(const struct bus_space_overrides * ov,uint64_t bit)936 bit_to_function_pointer(const struct bus_space_overrides *ov, uint64_t bit)
937 {
938 switch (bit) {
939 case BUS_SPACE_OVERRIDE_MAP:
940 return ov->ov_space_map;
941 case BUS_SPACE_OVERRIDE_UNMAP:
942 return ov->ov_space_unmap;
943 case BUS_SPACE_OVERRIDE_ALLOC:
944 return ov->ov_space_alloc;
945 case BUS_SPACE_OVERRIDE_FREE:
946 return ov->ov_space_free;
947 case BUS_SPACE_OVERRIDE_RESERVE:
948 return ov->ov_space_reserve;
949 case BUS_SPACE_OVERRIDE_RELEASE:
950 return ov->ov_space_release;
951 case BUS_SPACE_OVERRIDE_RESERVATION_MAP:
952 return ov->ov_space_reservation_map;
953 case BUS_SPACE_OVERRIDE_RESERVATION_UNMAP:
954 return ov->ov_space_reservation_unmap;
955 case BUS_SPACE_OVERRIDE_RESERVE_SUBREGION:
956 return ov->ov_space_reserve_subregion;
957 default:
958 return NULL;
959 }
960 }
961
962 void
bus_space_tag_destroy(bus_space_tag_t bst)963 bus_space_tag_destroy(bus_space_tag_t bst)
964 {
965 kmem_free(bst, sizeof(struct bus_space_tag));
966 }
967
968 int
bus_space_tag_create(bus_space_tag_t obst,const uint64_t present,const uint64_t extpresent,const struct bus_space_overrides * ov,void * ctx,bus_space_tag_t * bstp)969 bus_space_tag_create(bus_space_tag_t obst, const uint64_t present,
970 const uint64_t extpresent, const struct bus_space_overrides *ov, void *ctx,
971 bus_space_tag_t *bstp)
972 {
973 uint64_t bit, bits, nbits;
974 bus_space_tag_t bst;
975 const void *fp;
976
977 if (ov == NULL || present == 0 || extpresent != 0)
978 return EINVAL;
979
980 bst = kmem_alloc(sizeof(struct bus_space_tag), KM_SLEEP);
981 bst->bst_super = obst;
982 bst->bst_type = obst->bst_type;
983
984 for (bits = present; bits != 0; bits = nbits) {
985 nbits = bits & (bits - 1);
986 bit = nbits ^ bits;
987 if ((fp = bit_to_function_pointer(ov, bit)) == NULL) {
988 printf("%s: missing bit %" PRIx64 "\n", __func__, bit);
989 goto einval;
990 }
991 }
992
993 bst->bst_ov = ov;
994 bst->bst_exists = obst->bst_exists | present;
995 bst->bst_present = present;
996 bst->bst_ctx = ctx;
997
998 *bstp = bst;
999
1000 return 0;
1001 einval:
1002 kmem_free(bst, sizeof(struct bus_space_tag));
1003 return EINVAL;
1004 }
1005